public void Create(Stream target, FileFilter filter, string basePath, string packageFormat) { Stream archive = new MemoryStream(); string cleanUp = null; try { switch (packageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: { Console.WriteLine("Writing package in tar/gzip format..."); break; } case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { Console.WriteLine("Writing package in tar/lzma format..."); break; } } switch (packageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { var state = _deduplicator.CreateState(); Console.Write("Deduplicating files in package..."); var progressHelper = new DedupProgressRenderer(filter.Count()); var current = 0; foreach (var kv in filter.OrderBy(kv => kv.Value)) { if (kv.Value.EndsWith("/")) { // Directory _deduplicator.AddDirectory(state, kv.Value); } else { // File var realFile = Path.Combine(basePath, kv.Key); var realFileInfo = new FileInfo(realFile); _deduplicator.AddFile(state, realFileInfo, kv.Value); } current++; progressHelper.SetProgress(current); } Console.WriteLine(); Console.WriteLine("Adding files to package..."); try { using (var writer = new tar_cs.TarWriter(archive)) { _deduplicator.PushToTar(state, writer); } } catch (OutOfMemoryException ex) { // It's possible the archive is too large to store in memory. Fall // back to using a temporary file on disk. cleanUp = Path.GetTempFileName(); Console.WriteLine( "WARNING: Out-of-memory while creating TAR file, falling back to storing " + "temporary file on disk at " + cleanUp + " during package creation."); archive.Dispose(); archive = new FileStream(cleanUp, FileMode.Create, FileAccess.ReadWrite); using (var writer = new tar_cs.TarWriter(archive)) { _deduplicator.PushToTar(state, writer); } } break; } } archive.Seek(0, SeekOrigin.Begin); switch (packageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: { Console.WriteLine("Compressing package..."); using (var compress = new GZipStream(target, CompressionMode.Compress)) { archive.CopyTo(compress); } break; } case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { Console.Write("Compressing package..."); var progressHelper = new CompressProgressRenderer(archive.Length); LZMA.LzmaHelper.Compress(archive, target, progressHelper); Console.WriteLine(); break; } } } finally { if (cleanUp != null) { try { File.Delete(cleanUp); } catch { Console.WriteLine("WARNING: Unable to clean up temporary package file at " + cleanUp); } } } }
public int Execute(Execution execution) { if (!Directory.Exists(execution.PackageSourceFolder)) { throw new InvalidOperationException("The source folder " + execution.PackageSourceFolder + " does not exist."); } var allowAutopackage = true; var moduleExpectedPath = Path.Combine(execution.PackageSourceFolder, "Build", "Module.xml"); ModuleInfo rootModule = null; if (!File.Exists(moduleExpectedPath)) { if (execution.PackageFilterFile == null) { Console.WriteLine( "There is no module in the path '" + execution.PackageSourceFolder + "' (expected to " + "find a Build\\Module.xml file within that directory)."); return 1; } else { // We allow this mode if the user has provided a filter file and are constructing // the package manually. allowAutopackage = false; } } else { rootModule = ModuleInfo.Load(moduleExpectedPath); } var customDirectives = new Dictionary<string, Action<FileFilter>>() { { "autopackage", f => { if (allowAutopackage && rootModule != null) { this.m_AutomaticProjectPackager.Autopackage( f, execution, rootModule, execution.PackageSourceFolder, execution.PackagePlatform); } else { Console.WriteLine( "WARNING: There is no module in the path '" + execution.PackageSourceFolder + "' (expected to " + "find a Build\\Module.xml file within that directory). Ignoring the 'autopackage' directive."); } } } }; Console.WriteLine("Starting package creation for " + execution.PackagePlatform); var filter = new FileFilter(GetRecursiveFilesInPath(execution.PackageSourceFolder)); if (execution.PackageFilterFile != null) { using (var reader = new StreamReader(execution.PackageFilterFile)) { var contents = reader.ReadToEnd(); contents = contents.Replace("%PLATFORM%", execution.PackagePlatform); using (var inputStream = new MemoryStream(Encoding.ASCII.GetBytes(contents))) { this.m_FileFilterParser.ParseAndApply(filter, inputStream, customDirectives); } } } else { customDirectives["autopackage"](filter); } if (File.Exists(execution.PackageDestinationFile)) { Console.WriteLine("The destination file " + execution.PackageDestinationFile + " already exists; it will be overwritten."); File.Delete(execution.PackageDestinationFile); } filter.ImplyDirectories(); var filterDictionary = filter.ToDictionary(k => k.Key, v => v.Value); if (!filterDictionary.ContainsValue("Build/")) { Console.WriteLine("ERROR: The Build directory does not exist in the source folder."); if (execution.PackageFilterFile != null) { this.PrintFilterMappings(filterDictionary); } return 1; } if (!filterDictionary.ContainsValue("Build/Projects/")) { Console.WriteLine("ERROR: The Build\\Projects directory does not exist in the source folder."); if (execution.PackageFilterFile != null) { this.PrintFilterMappings(filterDictionary); } return 1; } if (!filterDictionary.ContainsValue("Build/Module.xml")) { Console.WriteLine("ERROR: The Build\\Module.xml file does not exist in the source folder."); if (execution.PackageFilterFile != null) { this.PrintFilterMappings(filterDictionary); } return 1; } if (filterDictionary.ContainsValue("Protobuild.exe")) { Console.WriteLine("ERROR: The Protobuild.exe file should not be included in the package file."); if (execution.PackageFilterFile != null) { this.PrintFilterMappings(filterDictionary); } return 1; } using (var target = new FileStream(execution.PackageDestinationFile, FileMode.CreateNew, FileAccess.Write, FileShare.None)) { var archive = new MemoryStream(); switch (execution.PackageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: { Console.WriteLine("Writing package in tar/gzip format..."); break; } case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { Console.WriteLine("Writing package in tar/lzma format..."); break; } } switch (execution.PackageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { var state = this.m_Deduplicator.CreateState(); Console.Write("Deduplicating files in package..."); var progressHelper = new DedupProgressRenderer(filter.Count()); var current = 0; foreach (var kv in filter.OrderBy(kv => kv.Value)) { if (kv.Value.EndsWith("/")) { // Directory this.m_Deduplicator.AddDirectory(state, kv.Value); } else { // File var realFile = Path.Combine(execution.PackageSourceFolder, kv.Key); var realFileInfo = new FileInfo(realFile); this.m_Deduplicator.AddFile(state, realFileInfo, kv.Value); } current++; progressHelper.SetProgress(current); } Console.WriteLine(); Console.WriteLine("Adding files to package..."); using (var writer = new tar_cs.TarWriter(archive)) { this.m_Deduplicator.PushToTar(state, writer); } break; } } archive.Seek(0, SeekOrigin.Begin); switch (execution.PackageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: { Console.WriteLine("Compressing package..."); using (var compress = new GZipStream(target, CompressionMode.Compress)) { archive.CopyTo(compress); } break; } case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { Console.Write("Compressing package..."); var progressHelper = new CompressProgressRenderer(archive.Length); LZMA.LzmaHelper.Compress(archive, target, progressHelper); Console.WriteLine(); break; } } } Console.WriteLine("\rPackage written to " + execution.PackageDestinationFile + " successfully."); return 0; }
public void Create(Stream target, FileFilter filter, string basePath, string packageFormat) { Stream archive = new MemoryStream(); string cleanUp = null; try { switch (packageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: { Console.WriteLine("Writing package in tar/gzip format..."); break; } case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { Console.WriteLine("Writing package in tar/lzma format..."); break; } } switch (packageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { var state = _deduplicator.CreateState(); Console.Write("Deduplicating files in package..."); var progressHelper = new DedupProgressRenderer(filter.Count()); var current = 0; foreach (var kv in filter.OrderBy(kv => kv.Value)) { if (kv.Value.EndsWith("/")) { // Directory _deduplicator.AddDirectory(state, kv.Value); } else { // File var realFile = Path.Combine(basePath, kv.Key); var realFileInfo = new FileInfo(realFile); _deduplicator.AddFile(state, realFileInfo, kv.Value); } current++; progressHelper.SetProgress(current); } progressHelper.Finalize(); Console.WriteLine("Adding files to package..."); try { using (var writer = new tar_cs.TarWriter(archive)) { _deduplicator.PushToTar(state, writer); } } catch (OutOfMemoryException ex) { // It's possible the archive is too large to store in memory. Fall // back to using a temporary file on disk. cleanUp = Path.GetTempFileName(); Console.WriteLine( "WARNING: Out-of-memory while creating TAR file, falling back to storing " + "temporary file on disk at " + cleanUp + " during package creation."); archive.Dispose(); archive = new FileStream(cleanUp, FileMode.Create, FileAccess.ReadWrite); using (var writer = new tar_cs.TarWriter(archive)) { _deduplicator.PushToTar(state, writer); } } break; } } archive.Seek(0, SeekOrigin.Begin); switch (packageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: { Console.WriteLine("Compressing package..."); using (var compress = new GZipStream(target, CompressionMode.Compress)) { archive.CopyTo(compress); } break; } case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { Console.Write("Compressing package..."); var progressHelper = new CompressProgressRenderer(archive.Length); LZMA.LzmaHelper.Compress(archive, target, progressHelper); progressHelper.Finalize(); break; } } } finally { if (cleanUp != null) { try { File.Delete(cleanUp); } catch { Console.WriteLine("WARNING: Unable to clean up temporary package file at " + cleanUp); } } } }
public void Create(Stream target, FileFilter filter, string basePath, string packageFormat, string platform, string excludedPath) { Stream archive = new MemoryStream(); string cleanUp = null; try { switch (packageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: RedirectableConsole.WriteLine("Writing package in tar/gzip format..."); break; case PackageManager.ARCHIVE_FORMAT_NUGET_ZIP: RedirectableConsole.WriteLine("Writing package in NuGet ZIP format... (this is experimental!)"); break; case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: RedirectableConsole.WriteLine("Writing package in tar/lzma format..."); break; } switch (packageFormat) { case PackageManager.ARCHIVE_FORMAT_NUGET_ZIP: { if (platform == "Unified") { // With unified packages, the contents have already been deduplicated in the // individual packages we're combining. Therefore we don't deduplicate again // because we already have the deduplication index calculated as part of the // package contents. Just add all the files to the ZIP instead. Action <ZipStorer> addFilesToZip = zip => { RedirectableConsole.WriteLine("Adding files to package..."); var progressHelper = new AddFilesProgressRenderer(filter.CountTargetPaths()); var current = 0; var uniqueFiles = new HashSet <string>(); var entries = filter.GetExpandedEntries(); foreach (var kv in entries.OrderBy(kv => kv.Value)) { if (kv.Value.EndsWith("/")) { // Directory - do nothing } else { var realFile = Path.Combine(basePath, kv.Key); if (uniqueFiles.Contains(kv.Value)) { // Already exists - do nothing } else { if (excludedPath != null) { var realFileInfo = new FileInfo(realFile); var excludedPathInfo = new FileInfo(excludedPath); if (excludedPathInfo.FullName == realFileInfo.FullName) { // Skip this file because we force it to be excluded. continue; } } zip.AddFile( ZipStorer.Compression.Deflate, realFile, kv.Value, string.Empty); uniqueFiles.Add(kv.Value); } } current++; progressHelper.SetProgress(current); } progressHelper.FinalizeRendering(); }; try { using (var zip = ZipStorer.Create(target, string.Empty, true)) { addFilesToZip(zip); } } catch (OutOfMemoryException) { // It's possible the archive is too large to store in memory. Fall // back to using a temporary file on disk. cleanUp = Path.GetTempFileName(); RedirectableConsole.WriteLine( "WARNING: Out-of-memory while creating ZIP file, falling back to storing " + "temporary file on disk at " + cleanUp + " during package creation."); archive.Dispose(); archive = new FileStream(cleanUp, FileMode.Create, FileAccess.ReadWrite); using (var zip = ZipStorer.Create(archive, string.Empty, true)) { addFilesToZip(zip); } } } else { // Deduplicate the contents of the ZIP package. var state = _deduplicator.CreateState(); RedirectableConsole.Write("Deduplicating files in package..."); var progressHelper = new DedupProgressRenderer(filter.CountTargetPaths()); var current = 0; var entries = filter.GetExpandedEntries(); foreach (var kv in entries.OrderBy(kv => kv.Value)) { if (kv.Value.EndsWith("/")) { // Directory _deduplicator.AddDirectory(state, kv.Value); } else { // File var realFile = Path.Combine(basePath, kv.Key); var realFileInfo = new FileInfo(realFile); if (excludedPath != null) { var excludedPathInfo = new FileInfo(excludedPath); if (excludedPathInfo.FullName == realFileInfo.FullName) { // Skip this file because we force it to be excluded. continue; } } _deduplicator.AddFile(state, realFileInfo, kv.Value); } current++; progressHelper.SetProgress(current); } progressHelper.FinalizeRendering(); RedirectableConsole.WriteLine("Adding files to package..."); try { using (var zip = ZipStorer.Create(target, string.Empty, true)) { _deduplicator.PushToZip(state, zip); } } catch (OutOfMemoryException) { // It's possible the archive is too large to store in memory. Fall // back to using a temporary file on disk. cleanUp = Path.GetTempFileName(); RedirectableConsole.WriteLine( "WARNING: Out-of-memory while creating ZIP file, falling back to storing " + "temporary file on disk at " + cleanUp + " during package creation."); archive.Dispose(); archive = new FileStream(cleanUp, FileMode.Create, FileAccess.ReadWrite); using (var zip = ZipStorer.Create(archive, string.Empty, true)) { _deduplicator.PushToZip(state, zip); } } } break; } case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { var state = _deduplicator.CreateState(); RedirectableConsole.Write("Deduplicating files in package..."); var progressHelper = new DedupProgressRenderer(filter.CountTargetPaths()); var current = 0; var entries = filter.GetExpandedEntries(); foreach (var kv in entries.OrderBy(kv => kv.Value)) { if (kv.Value.EndsWith("/")) { // Directory _deduplicator.AddDirectory(state, kv.Value); } else { // File var realFile = Path.Combine(basePath, kv.Key); var realFileInfo = new FileInfo(realFile); if (excludedPath != null) { var excludedPathInfo = new FileInfo(excludedPath); if (excludedPathInfo.FullName == realFileInfo.FullName) { // Skip this file because we force it to be excluded. continue; } } _deduplicator.AddFile(state, realFileInfo, kv.Value); } current++; progressHelper.SetProgress(current); } progressHelper.FinalizeRendering(); RedirectableConsole.WriteLine("Adding files to package..."); try { using (var writer = new tar_cs.TarWriter(archive)) { _deduplicator.PushToTar(state, writer); } } catch (OutOfMemoryException) { // It's possible the archive is too large to store in memory. Fall // back to using a temporary file on disk. cleanUp = Path.GetTempFileName(); RedirectableConsole.WriteLine( "WARNING: Out-of-memory while creating TAR file, falling back to storing " + "temporary file on disk at " + cleanUp + " during package creation."); archive.Dispose(); archive = new FileStream(cleanUp, FileMode.Create, FileAccess.ReadWrite); using (var writer = new tar_cs.TarWriter(archive)) { _deduplicator.PushToTar(state, writer); } } break; } } archive.Seek(0, SeekOrigin.Begin); switch (packageFormat) { case PackageManager.ARCHIVE_FORMAT_NUGET_ZIP: // No need to compress anything further. break; case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: { RedirectableConsole.WriteLine("Compressing package..."); using (var compress = new GZipStream(target, CompressionMode.Compress)) { archive.CopyTo(compress); } break; } case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { RedirectableConsole.Write("Compressing package..."); var progressHelper = new CompressProgressRenderer(archive.Length); LZMA.LzmaHelper.Compress(archive, target, progressHelper); progressHelper.FinalizeRendering(); break; } } } finally { if (cleanUp != null) { try { File.Delete(cleanUp); } catch { RedirectableConsole.WriteLine("WARNING: Unable to clean up temporary package file at " + cleanUp); } } } }
public int Execute(Execution execution) { if (!Directory.Exists(execution.PackageSourceFolder)) { throw new InvalidOperationException("The source folder " + execution.PackageSourceFolder + " does not exist."); } var allowAutopackage = true; var moduleExpectedPath = Path.Combine(execution.PackageSourceFolder, "Build", "Module.xml"); ModuleInfo rootModule = null; if (!File.Exists(moduleExpectedPath)) { if (execution.PackageFilterFile == null) { Console.WriteLine( "There is no module in the path '" + execution.PackageSourceFolder + "' (expected to " + "find a Build\\Module.xml file within that directory)."); return(1); } else { // We allow this mode if the user has provided a filter file and are constructing // the package manually. allowAutopackage = false; } } else { rootModule = ModuleInfo.Load(moduleExpectedPath); } var customDirectives = new Dictionary <string, Action <FileFilter> >() { { "autopackage", f => { if (allowAutopackage && rootModule != null) { this.m_AutomaticProjectPackager.Autopackage( f, execution, rootModule, execution.PackageSourceFolder, execution.PackagePlatform); } else { Console.WriteLine( "WARNING: There is no module in the path '" + execution.PackageSourceFolder + "' (expected to " + "find a Build\\Module.xml file within that directory). Ignoring the 'autopackage' directive."); } } } }; Console.WriteLine("Starting package creation for " + execution.PackagePlatform); var filter = new FileFilter(GetRecursiveFilesInPath(execution.PackageSourceFolder)); if (execution.PackageFilterFile != null) { using (var reader = new StreamReader(execution.PackageFilterFile)) { var contents = reader.ReadToEnd(); contents = contents.Replace("%PLATFORM%", execution.PackagePlatform); using (var inputStream = new MemoryStream(Encoding.ASCII.GetBytes(contents))) { this.m_FileFilterParser.ParseAndApply(filter, inputStream, customDirectives); } } } else { customDirectives["autopackage"](filter); } if (File.Exists(execution.PackageDestinationFile)) { Console.WriteLine("The destination file " + execution.PackageDestinationFile + " already exists; it will be overwritten."); File.Delete(execution.PackageDestinationFile); } filter.ImplyDirectories(); var filterDictionary = filter.ToDictionary(k => k.Key, v => v.Value); if (!filterDictionary.ContainsValue("Build/")) { Console.WriteLine("ERROR: The Build directory does not exist in the source folder."); if (execution.PackageFilterFile != null) { this.PrintFilterMappings(filterDictionary); } return(1); } if (!filterDictionary.ContainsValue("Build/Projects/")) { Console.WriteLine("ERROR: The Build\\Projects directory does not exist in the source folder."); if (execution.PackageFilterFile != null) { this.PrintFilterMappings(filterDictionary); } return(1); } if (!filterDictionary.ContainsValue("Build/Module.xml")) { Console.WriteLine("ERROR: The Build\\Module.xml file does not exist in the source folder."); if (execution.PackageFilterFile != null) { this.PrintFilterMappings(filterDictionary); } return(1); } if (filterDictionary.ContainsValue("Protobuild.exe")) { Console.WriteLine("ERROR: The Protobuild.exe file should not be included in the package file."); if (execution.PackageFilterFile != null) { this.PrintFilterMappings(filterDictionary); } return(1); } using (var target = new FileStream(execution.PackageDestinationFile, FileMode.CreateNew, FileAccess.Write, FileShare.None)) { var archive = new MemoryStream(); switch (execution.PackageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: { Console.WriteLine("Writing package in tar/gzip format..."); break; } case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { Console.WriteLine("Writing package in tar/lzma format..."); break; } } switch (execution.PackageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { var state = this.m_Deduplicator.CreateState(); Console.Write("Deduplicating files in package..."); var progressHelper = new DedupProgressRenderer(filter.Count()); var current = 0; foreach (var kv in filter.OrderBy(kv => kv.Value)) { if (kv.Value.EndsWith("/")) { // Directory this.m_Deduplicator.AddDirectory(state, kv.Value); } else { // File var realFile = Path.Combine(execution.PackageSourceFolder, kv.Key); var realFileInfo = new FileInfo(realFile); this.m_Deduplicator.AddFile(state, realFileInfo, kv.Value); } current++; progressHelper.SetProgress(current); } Console.WriteLine(); Console.WriteLine("Adding files to package..."); using (var writer = new tar_cs.TarWriter(archive)) { this.m_Deduplicator.PushToTar(state, writer); } break; } } archive.Seek(0, SeekOrigin.Begin); switch (execution.PackageFormat) { case PackageManager.ARCHIVE_FORMAT_TAR_GZIP: { Console.WriteLine("Compressing package..."); using (var compress = new GZipStream(target, CompressionMode.Compress)) { archive.CopyTo(compress); } break; } case PackageManager.ARCHIVE_FORMAT_TAR_LZMA: default: { Console.Write("Compressing package..."); var progressHelper = new CompressProgressRenderer(archive.Length); LZMA.LzmaHelper.Compress(archive, target, progressHelper); Console.WriteLine(); break; } } } Console.WriteLine("\rPackage written to " + execution.PackageDestinationFile + " successfully."); return(0); }