public void Create(Stream target, FileFilter filter, string basePath, string packageFormat)
{
Stream archive = new MemoryStream();
string cleanUp = null;
try
{
switch (packageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
{
Console.WriteLine("Writing package in tar/gzip format...");
break;
}
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
Console.WriteLine("Writing package in tar/lzma format...");
break;
}
}
switch (packageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
var state = _deduplicator.CreateState();
Console.Write("Deduplicating files in package...");
var progressHelper = new DedupProgressRenderer(filter.Count());
var current = 0;
foreach (var kv in filter.OrderBy(kv => kv.Value))
{
if (kv.Value.EndsWith("/"))
{
// Directory
_deduplicator.AddDirectory(state, kv.Value);
}
else
{
// File
var realFile = Path.Combine(basePath, kv.Key);
var realFileInfo = new FileInfo(realFile);
_deduplicator.AddFile(state, realFileInfo, kv.Value);
}
current++;
progressHelper.SetProgress(current);
}
Console.WriteLine();
Console.WriteLine("Adding files to package...");
try
{
using (var writer = new tar_cs.TarWriter(archive))
{
_deduplicator.PushToTar(state, writer);
}
}
catch (OutOfMemoryException ex)
{
// It's possible the archive is too large to store in memory. Fall
// back to using a temporary file on disk.
cleanUp = Path.GetTempFileName();
Console.WriteLine(
"WARNING: Out-of-memory while creating TAR file, falling back to storing " +
"temporary file on disk at " + cleanUp + " during package creation.");
archive.Dispose();
archive = new FileStream(cleanUp, FileMode.Create, FileAccess.ReadWrite);
using (var writer = new tar_cs.TarWriter(archive))
{
_deduplicator.PushToTar(state, writer);
}
}
break;
}
}
archive.Seek(0, SeekOrigin.Begin);
switch (packageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
{
Console.WriteLine("Compressing package...");
using (var compress = new GZipStream(target, CompressionMode.Compress))
{
archive.CopyTo(compress);
}
break;
}
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
Console.Write("Compressing package...");
var progressHelper = new CompressProgressRenderer(archive.Length);
LZMA.LzmaHelper.Compress(archive, target, progressHelper);
Console.WriteLine();
break;
}
}
}
finally
{
if (cleanUp != null)
{
try
{
File.Delete(cleanUp);
}
catch
{
Console.WriteLine("WARNING: Unable to clean up temporary package file at " + cleanUp);
}
}
}
}
public int Execute(Execution execution)
{
if (!Directory.Exists(execution.PackageSourceFolder))
{
throw new InvalidOperationException("The source folder " + execution.PackageSourceFolder + " does not exist.");
}
var allowAutopackage = true;
var moduleExpectedPath = Path.Combine(execution.PackageSourceFolder, "Build", "Module.xml");
ModuleInfo rootModule = null;
if (!File.Exists(moduleExpectedPath))
{
if (execution.PackageFilterFile == null)
{
Console.WriteLine(
"There is no module in the path '" + execution.PackageSourceFolder + "' (expected to " +
"find a Build\\Module.xml file within that directory).");
return 1;
}
else
{
// We allow this mode if the user has provided a filter file and are constructing
// the package manually.
allowAutopackage = false;
}
}
else
{
rootModule = ModuleInfo.Load(moduleExpectedPath);
}
var customDirectives = new Dictionary<string, Action<FileFilter>>()
{
{
"autopackage",
f =>
{
if (allowAutopackage && rootModule != null)
{
this.m_AutomaticProjectPackager.Autopackage(
f,
execution,
rootModule,
execution.PackageSourceFolder,
execution.PackagePlatform);
}
else
{
Console.WriteLine(
"WARNING: There is no module in the path '" + execution.PackageSourceFolder + "' (expected to " +
"find a Build\\Module.xml file within that directory). Ignoring the 'autopackage' directive.");
}
}
}
};
Console.WriteLine("Starting package creation for " + execution.PackagePlatform);
var filter = new FileFilter(GetRecursiveFilesInPath(execution.PackageSourceFolder));
if (execution.PackageFilterFile != null)
{
using (var reader = new StreamReader(execution.PackageFilterFile))
{
var contents = reader.ReadToEnd();
contents = contents.Replace("%PLATFORM%", execution.PackagePlatform);
using (var inputStream = new MemoryStream(Encoding.ASCII.GetBytes(contents)))
{
this.m_FileFilterParser.ParseAndApply(filter, inputStream, customDirectives);
}
}
}
else
{
customDirectives["autopackage"](filter);
}
if (File.Exists(execution.PackageDestinationFile))
{
Console.WriteLine("The destination file " + execution.PackageDestinationFile + " already exists; it will be overwritten.");
File.Delete(execution.PackageDestinationFile);
}
filter.ImplyDirectories();
var filterDictionary = filter.ToDictionary(k => k.Key, v => v.Value);
if (!filterDictionary.ContainsValue("Build/"))
{
Console.WriteLine("ERROR: The Build directory does not exist in the source folder.");
if (execution.PackageFilterFile != null)
{
this.PrintFilterMappings(filterDictionary);
}
return 1;
}
if (!filterDictionary.ContainsValue("Build/Projects/"))
{
Console.WriteLine("ERROR: The Build\\Projects directory does not exist in the source folder.");
if (execution.PackageFilterFile != null)
{
this.PrintFilterMappings(filterDictionary);
}
return 1;
}
if (!filterDictionary.ContainsValue("Build/Module.xml"))
{
Console.WriteLine("ERROR: The Build\\Module.xml file does not exist in the source folder.");
if (execution.PackageFilterFile != null)
{
this.PrintFilterMappings(filterDictionary);
}
return 1;
}
if (filterDictionary.ContainsValue("Protobuild.exe"))
{
Console.WriteLine("ERROR: The Protobuild.exe file should not be included in the package file.");
if (execution.PackageFilterFile != null)
{
this.PrintFilterMappings(filterDictionary);
}
return 1;
}
using (var target = new FileStream(execution.PackageDestinationFile, FileMode.CreateNew, FileAccess.Write, FileShare.None))
{
var archive = new MemoryStream();
switch (execution.PackageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
{
Console.WriteLine("Writing package in tar/gzip format...");
break;
}
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
Console.WriteLine("Writing package in tar/lzma format...");
break;
}
}
switch (execution.PackageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
var state = this.m_Deduplicator.CreateState();
Console.Write("Deduplicating files in package...");
var progressHelper = new DedupProgressRenderer(filter.Count());
var current = 0;
foreach (var kv in filter.OrderBy(kv => kv.Value))
{
if (kv.Value.EndsWith("/"))
{
// Directory
this.m_Deduplicator.AddDirectory(state, kv.Value);
}
else
{
// File
var realFile = Path.Combine(execution.PackageSourceFolder, kv.Key);
var realFileInfo = new FileInfo(realFile);
this.m_Deduplicator.AddFile(state, realFileInfo, kv.Value);
}
current++;
progressHelper.SetProgress(current);
}
Console.WriteLine();
Console.WriteLine("Adding files to package...");
using (var writer = new tar_cs.TarWriter(archive))
{
this.m_Deduplicator.PushToTar(state, writer);
}
break;
}
}
archive.Seek(0, SeekOrigin.Begin);
switch (execution.PackageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
{
Console.WriteLine("Compressing package...");
using (var compress = new GZipStream(target, CompressionMode.Compress))
{
archive.CopyTo(compress);
}
break;
}
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
Console.Write("Compressing package...");
var progressHelper = new CompressProgressRenderer(archive.Length);
LZMA.LzmaHelper.Compress(archive, target, progressHelper);
Console.WriteLine();
break;
}
}
}
Console.WriteLine("\rPackage written to " + execution.PackageDestinationFile + " successfully.");
return 0;
}
public void Create(Stream target, FileFilter filter, string basePath, string packageFormat)
{
Stream archive = new MemoryStream();
string cleanUp = null;
try
{
switch (packageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
{
Console.WriteLine("Writing package in tar/gzip format...");
break;
}
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
Console.WriteLine("Writing package in tar/lzma format...");
break;
}
}
switch (packageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
var state = _deduplicator.CreateState();
Console.Write("Deduplicating files in package...");
var progressHelper = new DedupProgressRenderer(filter.Count());
var current = 0;
foreach (var kv in filter.OrderBy(kv => kv.Value))
{
if (kv.Value.EndsWith("/"))
{
// Directory
_deduplicator.AddDirectory(state, kv.Value);
}
else
{
// File
var realFile = Path.Combine(basePath, kv.Key);
var realFileInfo = new FileInfo(realFile);
_deduplicator.AddFile(state, realFileInfo, kv.Value);
}
current++;
progressHelper.SetProgress(current);
}
progressHelper.Finalize();
Console.WriteLine("Adding files to package...");
try
{
using (var writer = new tar_cs.TarWriter(archive))
{
_deduplicator.PushToTar(state, writer);
}
}
catch (OutOfMemoryException ex)
{
// It's possible the archive is too large to store in memory. Fall
// back to using a temporary file on disk.
cleanUp = Path.GetTempFileName();
Console.WriteLine(
"WARNING: Out-of-memory while creating TAR file, falling back to storing " +
"temporary file on disk at " + cleanUp + " during package creation.");
archive.Dispose();
archive = new FileStream(cleanUp, FileMode.Create, FileAccess.ReadWrite);
using (var writer = new tar_cs.TarWriter(archive))
{
_deduplicator.PushToTar(state, writer);
}
}
break;
}
}
archive.Seek(0, SeekOrigin.Begin);
switch (packageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
{
Console.WriteLine("Compressing package...");
using (var compress = new GZipStream(target, CompressionMode.Compress))
{
archive.CopyTo(compress);
}
break;
}
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
Console.Write("Compressing package...");
var progressHelper = new CompressProgressRenderer(archive.Length);
LZMA.LzmaHelper.Compress(archive, target, progressHelper);
progressHelper.Finalize();
break;
}
}
}
finally
{
if (cleanUp != null)
{
try
{
File.Delete(cleanUp);
}
catch
{
Console.WriteLine("WARNING: Unable to clean up temporary package file at " + cleanUp);
}
}
}
}
public void Create(Stream target, FileFilter filter, string basePath, string packageFormat, string platform, string excludedPath)
{
Stream archive = new MemoryStream();
string cleanUp = null;
try
{
switch (packageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
RedirectableConsole.WriteLine("Writing package in tar/gzip format...");
break;
case PackageManager.ARCHIVE_FORMAT_NUGET_ZIP:
RedirectableConsole.WriteLine("Writing package in NuGet ZIP format... (this is experimental!)");
break;
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
RedirectableConsole.WriteLine("Writing package in tar/lzma format...");
break;
}
switch (packageFormat)
{
case PackageManager.ARCHIVE_FORMAT_NUGET_ZIP:
{
if (platform == "Unified")
{
// With unified packages, the contents have already been deduplicated in the
// individual packages we're combining. Therefore we don't deduplicate again
// because we already have the deduplication index calculated as part of the
// package contents. Just add all the files to the ZIP instead.
Action <ZipStorer> addFilesToZip = zip =>
{
RedirectableConsole.WriteLine("Adding files to package...");
var progressHelper = new AddFilesProgressRenderer(filter.CountTargetPaths());
var current = 0;
var uniqueFiles = new HashSet <string>();
var entries = filter.GetExpandedEntries();
foreach (var kv in entries.OrderBy(kv => kv.Value))
{
if (kv.Value.EndsWith("/"))
{
// Directory - do nothing
}
else
{
var realFile = Path.Combine(basePath, kv.Key);
if (uniqueFiles.Contains(kv.Value))
{
// Already exists - do nothing
}
else
{
if (excludedPath != null)
{
var realFileInfo = new FileInfo(realFile);
var excludedPathInfo = new FileInfo(excludedPath);
if (excludedPathInfo.FullName == realFileInfo.FullName)
{
// Skip this file because we force it to be excluded.
continue;
}
}
zip.AddFile(
ZipStorer.Compression.Deflate,
realFile,
kv.Value,
string.Empty);
uniqueFiles.Add(kv.Value);
}
}
current++;
progressHelper.SetProgress(current);
}
progressHelper.FinalizeRendering();
};
try
{
using (var zip = ZipStorer.Create(target, string.Empty, true))
{
addFilesToZip(zip);
}
}
catch (OutOfMemoryException)
{
// It's possible the archive is too large to store in memory. Fall
// back to using a temporary file on disk.
cleanUp = Path.GetTempFileName();
RedirectableConsole.WriteLine(
"WARNING: Out-of-memory while creating ZIP file, falling back to storing " +
"temporary file on disk at " + cleanUp + " during package creation.");
archive.Dispose();
archive = new FileStream(cleanUp, FileMode.Create, FileAccess.ReadWrite);
using (var zip = ZipStorer.Create(archive, string.Empty, true))
{
addFilesToZip(zip);
}
}
}
else
{
// Deduplicate the contents of the ZIP package.
var state = _deduplicator.CreateState();
RedirectableConsole.Write("Deduplicating files in package...");
var progressHelper = new DedupProgressRenderer(filter.CountTargetPaths());
var current = 0;
var entries = filter.GetExpandedEntries();
foreach (var kv in entries.OrderBy(kv => kv.Value))
{
if (kv.Value.EndsWith("/"))
{
// Directory
_deduplicator.AddDirectory(state, kv.Value);
}
else
{
// File
var realFile = Path.Combine(basePath, kv.Key);
var realFileInfo = new FileInfo(realFile);
if (excludedPath != null)
{
var excludedPathInfo = new FileInfo(excludedPath);
if (excludedPathInfo.FullName == realFileInfo.FullName)
{
// Skip this file because we force it to be excluded.
continue;
}
}
_deduplicator.AddFile(state, realFileInfo, kv.Value);
}
current++;
progressHelper.SetProgress(current);
}
progressHelper.FinalizeRendering();
RedirectableConsole.WriteLine("Adding files to package...");
try
{
using (var zip = ZipStorer.Create(target, string.Empty, true))
{
_deduplicator.PushToZip(state, zip);
}
}
catch (OutOfMemoryException)
{
// It's possible the archive is too large to store in memory. Fall
// back to using a temporary file on disk.
cleanUp = Path.GetTempFileName();
RedirectableConsole.WriteLine(
"WARNING: Out-of-memory while creating ZIP file, falling back to storing " +
"temporary file on disk at " + cleanUp + " during package creation.");
archive.Dispose();
archive = new FileStream(cleanUp, FileMode.Create, FileAccess.ReadWrite);
using (var zip = ZipStorer.Create(archive, string.Empty, true))
{
_deduplicator.PushToZip(state, zip);
}
}
}
break;
}
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
var state = _deduplicator.CreateState();
RedirectableConsole.Write("Deduplicating files in package...");
var progressHelper = new DedupProgressRenderer(filter.CountTargetPaths());
var current = 0;
var entries = filter.GetExpandedEntries();
foreach (var kv in entries.OrderBy(kv => kv.Value))
{
if (kv.Value.EndsWith("/"))
{
// Directory
_deduplicator.AddDirectory(state, kv.Value);
}
else
{
// File
var realFile = Path.Combine(basePath, kv.Key);
var realFileInfo = new FileInfo(realFile);
if (excludedPath != null)
{
var excludedPathInfo = new FileInfo(excludedPath);
if (excludedPathInfo.FullName == realFileInfo.FullName)
{
// Skip this file because we force it to be excluded.
continue;
}
}
_deduplicator.AddFile(state, realFileInfo, kv.Value);
}
current++;
progressHelper.SetProgress(current);
}
progressHelper.FinalizeRendering();
RedirectableConsole.WriteLine("Adding files to package...");
try
{
using (var writer = new tar_cs.TarWriter(archive))
{
_deduplicator.PushToTar(state, writer);
}
}
catch (OutOfMemoryException)
{
// It's possible the archive is too large to store in memory. Fall
// back to using a temporary file on disk.
cleanUp = Path.GetTempFileName();
RedirectableConsole.WriteLine(
"WARNING: Out-of-memory while creating TAR file, falling back to storing " +
"temporary file on disk at " + cleanUp + " during package creation.");
archive.Dispose();
archive = new FileStream(cleanUp, FileMode.Create, FileAccess.ReadWrite);
using (var writer = new tar_cs.TarWriter(archive))
{
_deduplicator.PushToTar(state, writer);
}
}
break;
}
}
archive.Seek(0, SeekOrigin.Begin);
switch (packageFormat)
{
case PackageManager.ARCHIVE_FORMAT_NUGET_ZIP:
// No need to compress anything further.
break;
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
{
RedirectableConsole.WriteLine("Compressing package...");
using (var compress = new GZipStream(target, CompressionMode.Compress))
{
archive.CopyTo(compress);
}
break;
}
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
RedirectableConsole.Write("Compressing package...");
var progressHelper = new CompressProgressRenderer(archive.Length);
LZMA.LzmaHelper.Compress(archive, target, progressHelper);
progressHelper.FinalizeRendering();
break;
}
}
}
finally
{
if (cleanUp != null)
{
try
{
File.Delete(cleanUp);
}
catch
{
RedirectableConsole.WriteLine("WARNING: Unable to clean up temporary package file at " + cleanUp);
}
}
}
}
public int Execute(Execution execution)
{
if (!Directory.Exists(execution.PackageSourceFolder))
{
throw new InvalidOperationException("The source folder " + execution.PackageSourceFolder + " does not exist.");
}
var allowAutopackage = true;
var moduleExpectedPath = Path.Combine(execution.PackageSourceFolder, "Build", "Module.xml");
ModuleInfo rootModule = null;
if (!File.Exists(moduleExpectedPath))
{
if (execution.PackageFilterFile == null)
{
Console.WriteLine(
"There is no module in the path '" + execution.PackageSourceFolder + "' (expected to " +
"find a Build\\Module.xml file within that directory).");
return(1);
}
else
{
// We allow this mode if the user has provided a filter file and are constructing
// the package manually.
allowAutopackage = false;
}
}
else
{
rootModule = ModuleInfo.Load(moduleExpectedPath);
}
var customDirectives = new Dictionary <string, Action <FileFilter> >()
{
{
"autopackage",
f =>
{
if (allowAutopackage && rootModule != null)
{
this.m_AutomaticProjectPackager.Autopackage(
f,
execution,
rootModule,
execution.PackageSourceFolder,
execution.PackagePlatform);
}
else
{
Console.WriteLine(
"WARNING: There is no module in the path '" + execution.PackageSourceFolder + "' (expected to " +
"find a Build\\Module.xml file within that directory). Ignoring the 'autopackage' directive.");
}
}
}
};
Console.WriteLine("Starting package creation for " + execution.PackagePlatform);
var filter = new FileFilter(GetRecursiveFilesInPath(execution.PackageSourceFolder));
if (execution.PackageFilterFile != null)
{
using (var reader = new StreamReader(execution.PackageFilterFile))
{
var contents = reader.ReadToEnd();
contents = contents.Replace("%PLATFORM%", execution.PackagePlatform);
using (var inputStream = new MemoryStream(Encoding.ASCII.GetBytes(contents)))
{
this.m_FileFilterParser.ParseAndApply(filter, inputStream, customDirectives);
}
}
}
else
{
customDirectives["autopackage"](filter);
}
if (File.Exists(execution.PackageDestinationFile))
{
Console.WriteLine("The destination file " + execution.PackageDestinationFile + " already exists; it will be overwritten.");
File.Delete(execution.PackageDestinationFile);
}
filter.ImplyDirectories();
var filterDictionary = filter.ToDictionary(k => k.Key, v => v.Value);
if (!filterDictionary.ContainsValue("Build/"))
{
Console.WriteLine("ERROR: The Build directory does not exist in the source folder.");
if (execution.PackageFilterFile != null)
{
this.PrintFilterMappings(filterDictionary);
}
return(1);
}
if (!filterDictionary.ContainsValue("Build/Projects/"))
{
Console.WriteLine("ERROR: The Build\\Projects directory does not exist in the source folder.");
if (execution.PackageFilterFile != null)
{
this.PrintFilterMappings(filterDictionary);
}
return(1);
}
if (!filterDictionary.ContainsValue("Build/Module.xml"))
{
Console.WriteLine("ERROR: The Build\\Module.xml file does not exist in the source folder.");
if (execution.PackageFilterFile != null)
{
this.PrintFilterMappings(filterDictionary);
}
return(1);
}
if (filterDictionary.ContainsValue("Protobuild.exe"))
{
Console.WriteLine("ERROR: The Protobuild.exe file should not be included in the package file.");
if (execution.PackageFilterFile != null)
{
this.PrintFilterMappings(filterDictionary);
}
return(1);
}
using (var target = new FileStream(execution.PackageDestinationFile, FileMode.CreateNew, FileAccess.Write, FileShare.None))
{
var archive = new MemoryStream();
switch (execution.PackageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
{
Console.WriteLine("Writing package in tar/gzip format...");
break;
}
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
Console.WriteLine("Writing package in tar/lzma format...");
break;
}
}
switch (execution.PackageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
var state = this.m_Deduplicator.CreateState();
Console.Write("Deduplicating files in package...");
var progressHelper = new DedupProgressRenderer(filter.Count());
var current = 0;
foreach (var kv in filter.OrderBy(kv => kv.Value))
{
if (kv.Value.EndsWith("/"))
{
// Directory
this.m_Deduplicator.AddDirectory(state, kv.Value);
}
else
{
// File
var realFile = Path.Combine(execution.PackageSourceFolder, kv.Key);
var realFileInfo = new FileInfo(realFile);
this.m_Deduplicator.AddFile(state, realFileInfo, kv.Value);
}
current++;
progressHelper.SetProgress(current);
}
Console.WriteLine();
Console.WriteLine("Adding files to package...");
using (var writer = new tar_cs.TarWriter(archive))
{
this.m_Deduplicator.PushToTar(state, writer);
}
break;
}
}
archive.Seek(0, SeekOrigin.Begin);
switch (execution.PackageFormat)
{
case PackageManager.ARCHIVE_FORMAT_TAR_GZIP:
{
Console.WriteLine("Compressing package...");
using (var compress = new GZipStream(target, CompressionMode.Compress))
{
archive.CopyTo(compress);
}
break;
}
case PackageManager.ARCHIVE_FORMAT_TAR_LZMA:
default:
{
Console.Write("Compressing package...");
var progressHelper = new CompressProgressRenderer(archive.Length);
LZMA.LzmaHelper.Compress(archive, target, progressHelper);
Console.WriteLine();
break;
}
}
}
Console.WriteLine("\rPackage written to " + execution.PackageDestinationFile + " successfully.");
return(0);
}