private Dictionary <string, byte[]> UnpackZip(ZipStorer zip, string folder, Func <string, bool> filterOutputPaths, Func <string, string> mutateOutputPaths, bool toMemory)
{
var results = new Dictionary <string, byte[]>();
const string DedupPrefix = "_DedupFiles/";
var hashesToStreams = new Dictionary <string, Stream>();
var entries = zip.ReadCentralDir();
if (entries.Any(x => x.FilenameInZip == "_DedupIndex.txt"))
{
// Extract and reduplicate the contents of the package.
var indexFile = entries.First(x => x.FilenameInZip == "_DedupIndex.txt");
using (var indexStream = new MemoryStream())
{
zip.ExtractFile(indexFile, indexStream);
indexStream.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReader(indexStream, Encoding.UTF8, true, 4096, true))
{
while (!reader.EndOfStream)
{
var components = reader.ReadLine().Split(new[] { '?' }, 2);
if (components.Length != 2 || (components.Length >= 1 && string.IsNullOrWhiteSpace(components[0])))
{
Console.WriteLine("WARNING: Malformed index entry in deduplication index.");
continue;
}
var outputFilename = components[0];
var filenameInZip = components[1];
if (filterOutputPaths != null && !filterOutputPaths(outputFilename))
{
// Path was filtered out.
continue;
}
if (mutateOutputPaths != null)
{
outputFilename = mutateOutputPaths(outputFilename);
}
if (filenameInZip == "<DIRECTORY>")
{
// Explicitly create this directory.
var buildUpDir = string.Empty;
foreach (var dirComponent in outputFilename.Replace('\\', '/').Split(new[] { '/' }))
{
var localDir = buildUpDir + dirComponent + "/";
buildUpDir = localDir;
if (toMemory)
{
if (!results.ContainsKey(localDir))
{
results.Add(localDir, null);
}
}
else
{
Directory.CreateDirectory(folder.TrimEnd(new[] { '/', '\\' }) + '/' + NormalizeName(localDir));
}
}
continue;
}
if (!entries.Any(x => x.FilenameInZip == filenameInZip))
{
Console.WriteLine("WARNING: Unable to locate deduplication data file in ZIP: " + filenameInZip);
continue;
}
var baseName = Path.GetDirectoryName(outputFilename);
var buildUp = string.Empty;
foreach (var dirComponent in baseName.Replace('\\', '/').Split(new[] { '/' }))
{
var localDir = buildUp + dirComponent + "/";
buildUp = localDir;
if (toMemory)
{
if (!results.ContainsKey(localDir))
{
results.Add(localDir, null);
}
}
else
{
Directory.CreateDirectory(folder.TrimEnd(new[] { '/', '\\' }) + '/' + NormalizeName(localDir));
}
}
if (toMemory)
{
using (var stream = new MemoryStream())
{
zip.ExtractFile(entries.First(x => x.FilenameInZip == filenameInZip), stream);
var bytes = new byte[stream.Position];
stream.Seek(0, SeekOrigin.Begin);
stream.Read(bytes, 0, bytes.Length);
results.Add(outputFilename, bytes);
}
}
else
{
var fullOutputName = folder.TrimEnd(new[] { '/', '\\' }) + '/' + NormalizeName(outputFilename);
zip.ExtractFile(entries.First(x => x.FilenameInZip == filenameInZip), fullOutputName);
results.Add(outputFilename, null);
// If on a UNIX (MacOS or Linux) system, chmod all files to have an executable bit, otherwise things like
// scripts and macOS application bundles won't work.
PathUtils.MakePathExecutable(fullOutputName, false);
}
}
}
}
}
else
{
// Extract as-is.
foreach (var entry in entries.OrderBy(x => x.FilenameInZip))
{
var outputFilename = NormalizeName(entry.FilenameInZip);
if (filterOutputPaths != null && !filterOutputPaths(outputFilename))
{
// Path was filtered out.
continue;
}
if (mutateOutputPaths != null)
{
outputFilename = mutateOutputPaths(outputFilename);
}
var baseName = Path.GetDirectoryName(outputFilename);
var buildUp = string.Empty;
foreach (var dirComponent in baseName.Replace('\\', '/').Split(new[] { '/' }))
{
var localDir = buildUp + dirComponent + "/";
buildUp = localDir;
if (toMemory)
{
if (!results.ContainsKey(localDir))
{
results.Add(localDir, null);
}
}
else
{
Directory.CreateDirectory(folder.TrimEnd(new[] { '/', '\\' }) + '/' + NormalizeName(localDir));
}
}
if (toMemory)
{
using (var stream = new MemoryStream())
{
zip.ExtractFile(entry, stream);
var bytes = new byte[stream.Position];
stream.Seek(0, SeekOrigin.Begin);
stream.Read(bytes, 0, bytes.Length);
results.Add(outputFilename, bytes);
}
}
else
{
zip.ExtractFile(entry, folder.TrimEnd(new[] { '/', '\\' }) + '/' + outputFilename);
results.Add(outputFilename, null);
}
}
}
return(results);
}
private void ResolveLibraryBinary(string workingDirectory, ICachableBinaryPackageMetadata protobuildMetadata, string folder, bool forceUpgrade, Func <byte[]> getBinaryPackage)
{
var platformFolder = Path.Combine(folder, protobuildMetadata.Platform);
if (File.Exists(Path.Combine(platformFolder, ".pkg")))
{
if (!forceUpgrade)
{
RedirectableConsole.WriteLine("Protobuild binary package already present at " + platformFolder);
return;
}
}
RedirectableConsole.WriteLine("Creating and emptying " + platformFolder);
if (File.Exists(Path.Combine(folder, ".pkg")))
{
if (Directory.Exists(platformFolder))
{
// Only clear out the target's folder if the reference folder
// already contains binary packages (for other platforms)
PathUtils.AggressiveDirectoryDelete(platformFolder);
}
}
else
{
// The reference folder is holding source code, so clear it
// out entirely.
PathUtils.AggressiveDirectoryDelete(folder);
}
Directory.CreateDirectory(platformFolder);
RedirectableConsole.WriteLine("Marking " + folder + " as ignored for Git");
GitUtils.MarkIgnored(folder);
var package = getBinaryPackage();
if (package == null)
{
return;
}
ExtractTo(workingDirectory, protobuildMetadata.PackageName, protobuildMetadata.BinaryFormat, package, platformFolder, protobuildMetadata.Platform);
// Only copy ourselves to the binary folder if both "Build/Module.xml" and
// "Build/Projects" exist in the binary package's folder. This prevents us
// from triggering the "create new module?" logic if the package hasn't been
// setup correctly.
if (Directory.Exists(Path.Combine(platformFolder, "Build", "Projects")) &&
File.Exists(Path.Combine(platformFolder, "Build", "Module.xml")))
{
var sourceProtobuild = Assembly.GetEntryAssembly().Location;
File.Copy(sourceProtobuild, Path.Combine(platformFolder, "Protobuild.exe"), true);
PathUtils.MakePathExecutable(Path.Combine(platformFolder, "Protobuild.exe"), true);
}
var file = File.Create(Path.Combine(platformFolder, ".pkg"));
file.Close();
file = File.Create(Path.Combine(folder, ".pkg"));
file.Close();
RedirectableConsole.WriteLine("Binary resolution complete");
}
public Tuple <int, string, string> RunProtobuild(ModuleInfo module, string args, bool capture = false)
{
var invokeInline = false;
var myHash = ExecEnvironment.GetProgramHash();
var targetHash = ExecEnvironment.GetProgramHash(Path.Combine(module.Path, "Protobuild.exe"));
if (myHash == targetHash)
{
invokeInline = true;
}
if (ExecEnvironment.RunProtobuildInProcess || invokeInline)
{
var oldBuffer = RedirectableConsole.TargetBuffer;
var ourBuffer = new RedirectableBuffer();
RedirectableConsole.TargetBuffer = ourBuffer;
var needsEndSelfInvoke = true;
try
{
var exitCode = ExecEnvironment.InvokeSelf(module.Path, args.SplitCommandLine().ToArray());
RedirectableConsole.TargetBuffer = oldBuffer;
needsEndSelfInvoke = false;
if (capture)
{
return(new Tuple <int, string, string>(
exitCode,
ourBuffer.Stdout,
ourBuffer.Stderr));
}
else
{
return(new Tuple <int, string, string>(
exitCode,
string.Empty,
string.Empty));
}
}
finally
{
if (needsEndSelfInvoke)
{
RedirectableConsole.TargetBuffer = oldBuffer;
}
}
}
var protobuildPath = Path.Combine(module.Path, "Protobuild.exe");
PathUtils.MakePathExecutable(protobuildPath, true);
var stdout = string.Empty;
var stderr = string.Empty;
for (var attempt = 0; attempt < 3; attempt++)
{
if (File.Exists(protobuildPath))
{
var pi = new ProcessStartInfo
{
FileName = protobuildPath,
Arguments = args,
WorkingDirectory = module.Path,
CreateNoWindow = capture,
RedirectStandardError = capture,
RedirectStandardInput = capture,
RedirectStandardOutput = capture,
UseShellExecute = false
};
var p = new Process {
StartInfo = pi
};
if (capture)
{
p.OutputDataReceived += (sender, eventArgs) =>
{
if (!string.IsNullOrEmpty(eventArgs.Data))
{
if (capture)
{
stdout += eventArgs.Data + "\n";
}
else
{
RedirectableConsole.WriteLine(eventArgs.Data);
}
}
};
p.ErrorDataReceived += (sender, eventArgs) =>
{
if (!string.IsNullOrEmpty(eventArgs.Data))
{
if (capture)
{
stderr += eventArgs.Data + "\n";
}
else
{
RedirectableConsole.ErrorWriteLine(eventArgs.Data);
}
}
};
}
try
{
p.Start();
}
catch (System.ComponentModel.Win32Exception ex)
{
if (ex.Message.Contains("Cannot find the specified file"))
{
// Mono sometimes throws this error even though the
// file does exist on disk. The best guess is there's
// a race condition between performing chmod on the
// file and Mono actually seeing it as an executable file.
// Show a warning and sleep for a bit before retrying.
if (attempt != 2)
{
RedirectableConsole.WriteLine("WARNING: Unable to execute Protobuild.exe, will retry again...");
System.Threading.Thread.Sleep(2000);
continue;
}
else
{
RedirectableConsole.WriteLine("ERROR: Still unable to execute Protobuild.exe.");
throw;
}
}
}
if (capture)
{
p.BeginOutputReadLine();
p.BeginErrorReadLine();
}
p.WaitForExit();
return(new Tuple <int, string, string>(p.ExitCode, stdout, stderr));
}
}
return(new Tuple <int, string, string>(1, string.Empty, string.Empty));
}