private string PackFilesPreProcessed(List <CompilationTask> taskList) { byte[] package; using (var memoryStream = new MemoryStream()) { using (var archive = new TarOutputStream(memoryStream)) { List <string> addedEntries = new List <string>(); //precompiled headers are not used in preprocessed build // Package build batch TextWriter batch = new StreamWriter("golembuild.bat", false); // CD to the directory the batch file is in batch.WriteLine("cd %~DP0"); // Create output folder batch.WriteLine("mkdir output"); List <CompilerArg> compilerArgs = new List <CompilerArg>(); foreach (CompilationTask task in taskList) { bool found = false; string args = task.CompilerArgs; foreach (CompilerArg compilerArg in compilerArgs) { bool includesMatch = task.IncludeDirs.Count == compilerArg.includeDirs.Count; if (includesMatch) { for (int i = 0; i < task.IncludeDirs.Count; ++i) { if (!compilerArg.includeDirs[i].Equals(task.IncludeDirs[i])) { includesMatch = false; break; } } } if (compilerArg.compiler == task.Compiler && compilerArg.args == args && includesMatch) { compilerArg.files.Add(task.FilePath); found = true; break; } } if (found) { continue; } CompilerArg newCompilerArg = new CompilerArg(); newCompilerArg.compiler = task.Compiler; newCompilerArg.args = args; newCompilerArg.files.Add(task.FilePath); foreach (string e in task.IncludeDirs) { newCompilerArg.includeDirs.Add(e); } compilerArgs.Add(newCompilerArg); } string tempFolder = "iGolemBuild" + Peer.NodeId; Directory.CreateDirectory(tempFolder); //foreach compilation task, preprocess the cpp file into a temporary folder foreach (CompilerArg compilerArg in compilerArgs) { //preprocess file, grab output, write the file as file to compile on external machine Process proc = new Process(); string args = compilerArg.args; //add includes foreach (string inc in compilerArg.includeDirs) { args += " /I\"" + inc + "\" "; } //add preprocessing flag args += "/P /Fi" + tempFolder + "\\ "; args += "/MP" + TaskCapacity; //add source files foreach (string srcFile in compilerArg.files) { args += " " + srcFile; } proc.StartInfo.Arguments = args; proc.StartInfo.FileName = compilerArg.compiler; proc.StartInfo.WindowStyle = ProcessWindowStyle.Hidden; proc.StartInfo.UseShellExecute = false; proc.StartInfo.RedirectStandardInput = false; proc.StartInfo.RedirectStandardOutput = true; proc.StartInfo.RedirectStandardError = true; proc.StartInfo.CreateNoWindow = true; proc.Start(); System.Text.StringBuilder outputStr = new System.Text.StringBuilder(); proc.OutputDataReceived += (sender, e) => { if (e.Data != null) { string output = e.Data; Logger.LogMessage(output); } }; proc.ErrorDataReceived += (sender, e) => { if (e.Data != null) { outputStr.AppendLine(e.Data); } }; proc.BeginOutputReadLine(); proc.BeginErrorReadLine(); proc.WaitForExit(); Logger.LogMessage(outputStr.ToString()); if (proc.ExitCode == 0) { //now read back the files and add them to tar foreach (string srcFile in compilerArg.files) { //TODO: this might be inside a folder string precompiledFile = tempFolder + "\\" + Path.GetFileNameWithoutExtension(srcFile) + ".i"; TarEntry entry = TarEntry.CreateEntryFromFile(precompiledFile); entry.Name = Path.GetFileName(srcFile); archive.PutNextEntry(entry); using (Stream inputStream = File.OpenRead(precompiledFile)) { writeStreamToTar(archive, inputStream); archive.CloseEntry(); } } } else { Logger.LogError($"Preprocessing of file package failed"); } } Directory.Delete(tempFolder, true); // Add compilation commands, once per CompilerArg foreach (CompilerArg compilerArg in compilerArgs) { //remove precompiled header args /Yu /Fp Match match = Regex.Match(compilerArg.args, "/Yu\".+?\""); if (match.Success) { compilerArg.args = compilerArg.args.Remove(match.Index, match.Length); } match = Regex.Match(compilerArg.args, "/Fp\".+?\""); if (match.Success) { compilerArg.args = compilerArg.args.Remove(match.Index, match.Length); } compilerArg.args += " /FS"; compilerArg.args += " /Fo\"output/\""; compilerArg.args += " /Fd\"output/" + Path.GetFileNameWithoutExtension(compilerArg.files[0]) + ".pdb\""; compilerArg.args += " /MP" + TaskCapacity; batch.Write("\"../" + Path.GetFileName(compilerArg.compiler) + "\" " + compilerArg.args); foreach (string file in compilerArg.files) { batch.Write(" " + Path.GetFileName(file)); } batch.WriteLine(); } // Zip output folder batch.WriteLine("powershell.exe -nologo -noprofile -command \"& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::CreateFromDirectory('output', 'output.zip'); }\""); // stop the service batch.WriteLine("\"../mspdbsrv.exe\" -stop"); batch.WriteLine("exit 0");//assume no error batch.Close(); TarEntry batchEntry = TarEntry.CreateEntryFromFile("golembuild.bat"); batchEntry.Name = "golembuild.bat"; using (Stream inputStream = File.OpenRead("golembuild.bat")) { batchEntry.Size = inputStream.Length; archive.PutNextEntry(batchEntry); writeStreamToTar(archive, inputStream); archive.CloseEntry(); } } package = memoryStream.ToArray(); string hash = GolemCache.RegisterTasksPackage(package); FileStream debug = new FileStream(hash + ".tar", FileMode.Create); debug.Write(package, 0, package.Length); debug.Close(); return(hash); } }
private string PackFiles(List <CompilationTask> taskList) { byte[] package; using (var memoryStream = new MemoryStream()) { using (var archive = TarArchive.CreateOutputTarArchive(memoryStream)) { List <string> addedEntries = new List <string>(); // Package precompiled header if used foreach (CompilationTask task in taskList) { if (task.PrecompiledHeader.Length > 0) { TarEntry entry = TarEntry.CreateEntryFromFile(task.PrecompiledHeader); entry.Name = Path.GetFileName(task.PrecompiledHeader); archive.WriteEntry(entry, false); break; } } foreach (CompilationTask task in taskList) { // Package sourcefiles // not needed since this file is already in the task.Includes // TODO: should it be like this? { TarEntry entry = TarEntry.CreateEntryFromFile(task.FilePath); entry.Name = Path.GetFileName(task.FilePath); archive.WriteEntry(entry, false); } // Package includes string projectPath = task.ProjectPath; string dstLibIncludePath = "includes"; string dstProjectIncludePath = ""; foreach (string include in task.Includes) { string dstFilePath = null; if (include.StartsWith(projectPath)) { string relative = include.Replace(projectPath, "").TrimStart('\\', '/'); dstFilePath = Path.Combine(dstProjectIncludePath, relative); } else { for (int i = 0; i < task.IncludeDirs.Count; ++i) { string srcIncludePath = task.IncludeDirs[i]; if (include.StartsWith(srcIncludePath)) { string relative = include.Replace(srcIncludePath, "").TrimStart('\\', '/'); dstFilePath = Path.Combine(dstLibIncludePath + i.ToString(), relative); break; } } } AddFileToTar(archive, include, dstFilePath, addedEntries); } } // Package build batch TextWriter batch = new StreamWriter("golembuild.bat", false); // CD to the directory the batch file is in batch.WriteLine("cd %~DP0"); // Create output folder batch.WriteLine("mkdir output"); int numberOfIncludeDirs = 0; List <CompilerArg> compilerArgs = new List <CompilerArg>(); foreach (CompilationTask task in taskList) { bool found = false; foreach (CompilerArg compilerArg in compilerArgs) { if (compilerArg.compiler == task.Compiler && compilerArg.args == task.CompilerArgs) { compilerArg.files.Add(Path.GetFileName(task.FilePath)); found = true; break; } } if (found) { continue; } numberOfIncludeDirs = Math.Max(numberOfIncludeDirs, task.IncludeDirs.Count); CompilerArg newCompilerArg = new CompilerArg(); newCompilerArg.compiler = task.Compiler; newCompilerArg.args = task.CompilerArgs; newCompilerArg.files.Add(Path.GetFileName(task.FilePath)); compilerArgs.Add(newCompilerArg); } // Add compilation commands, once per CompilerArg foreach (CompilerArg compilerArg in compilerArgs) { for (int i = 0; i < numberOfIncludeDirs; ++i) { compilerArg.args += " /I\"includes" + i.ToString() + "\" /FS"; } compilerArg.args += " /Fo\"output/\""; compilerArg.args += " /Fd\"output/" + Path.GetFileNameWithoutExtension(compilerArg.files[0]) + ".pdb\""; compilerArg.args += " /MP" + TaskCapacity; batch.Write("\"../" + Path.GetFileName(compilerArg.compiler) + "\" " + compilerArg.args); foreach (string file in compilerArg.files) { batch.Write(" " + file); } batch.WriteLine(); } // Zip output folder batch.WriteLine("powershell.exe -nologo -noprofile -command \"& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::CreateFromDirectory('output', 'output.zip'); }\""); // stop the service batch.WriteLine("\"../mspdbsrv.exe\" -stop"); batch.WriteLine("exit 0");//assume no error batch.Close(); TarEntry batchEntry = TarEntry.CreateEntryFromFile("golembuild.bat"); batchEntry.Name = "golembuild.bat"; archive.WriteEntry(batchEntry, false); } package = memoryStream.ToArray(); string hash = GolemCache.RegisterTasksPackage(package); FileStream debug = new FileStream(hash + ".tar", FileMode.Create); debug.Write(package, 0, package.Length); debug.Close(); return(hash); } }