private async Task TaskDispatcher(System.Threading.CancellationToken token) { try { //we need to distribute work from the queue while (true) { while (taskQueue.Count > 0 && !token.IsCancellationRequested) { //get number of available tasks (this can only increase from another thread, so should be fine) int taskQueueSize = taskQueue.Count; //get first available worker GolemWorker worker = null; if (workerPool.TryDequeue(out worker)) { System.Threading.Interlocked.Increment(ref runningWorkers); Logger.LogMessage("Worker " + worker.Peer.NodeId + " started"); worker.ClearTasks(); List <string> compilersUsed = new List <string>(); //get the number of tasks to process int taskCount = Math.Min(worker.TaskCapacity, taskQueueSize); for (int i = 0; i < taskCount; ++i) { CompilationTask task = null; if (taskQueue.TryDequeue(out task)) { if (!compilersUsed.Contains(task.Compiler)) { compilersUsed.Add(task.Compiler); } worker.AddTask(task); } } string hash = GolemCache.GetCompilerPackageHash(compilersUsed); DeploymentSpecImage specImg = new DeploymentSpecImage("SHA1:" + hash, "http://" + myIP + ":" + ServerPort + "/requestID/compiler/" + hash); //create deployment DeploymentSpec spec = new DeploymentSpec(EnvType.Hd, specImg, "Compiler", new List <string>() { }); worker.Dispatch(golemApi, spec, () => { workerPool.Enqueue(worker); Logger.LogMessage("Worker " + worker.Peer.NodeId + " finished"); System.Threading.Interlocked.Decrement(ref runningWorkers); }); } } await Task.Delay(1000, token); } } catch (TaskCanceledException) { } }
public bool WaitTasks() { while (!taskQueue.IsEmpty || runningWorkers > 0) { System.Threading.Thread.Sleep(100); } GolemCache.Reset(); return(compilationSuccessful); }
private string PackFilesPreProcessed(List <CompilationTask> taskList) { byte[] package; using (var memoryStream = new MemoryStream()) { using (var archive = new TarOutputStream(memoryStream)) { List <string> addedEntries = new List <string>(); //precompiled headers are not used in preprocessed build // Package build batch TextWriter batch = new StreamWriter("golembuild.bat", false); // CD to the directory the batch file is in batch.WriteLine("cd %~DP0"); // Create output folder batch.WriteLine("mkdir output"); List <CompilerArg> compilerArgs = new List <CompilerArg>(); foreach (CompilationTask task in taskList) { bool found = false; string args = task.CompilerArgs; foreach (CompilerArg compilerArg in compilerArgs) { bool includesMatch = task.IncludeDirs.Count == compilerArg.includeDirs.Count; if (includesMatch) { for (int i = 0; i < task.IncludeDirs.Count; ++i) { if (!compilerArg.includeDirs[i].Equals(task.IncludeDirs[i])) { includesMatch = false; break; } } } if (compilerArg.compiler == task.Compiler && compilerArg.args == args && includesMatch) { compilerArg.files.Add(task.FilePath); found = true; break; } } if (found) { continue; } CompilerArg newCompilerArg = new CompilerArg(); newCompilerArg.compiler = task.Compiler; newCompilerArg.args = args; newCompilerArg.files.Add(task.FilePath); foreach (string e in task.IncludeDirs) { newCompilerArg.includeDirs.Add(e); } compilerArgs.Add(newCompilerArg); } string tempFolder = "iGolemBuild" + Peer.NodeId; Directory.CreateDirectory(tempFolder); //foreach compilation task, preprocess the cpp file into a temporary folder foreach (CompilerArg compilerArg in compilerArgs) { //preprocess file, grab output, write the file as file to compile on external machine Process proc = new Process(); string args = compilerArg.args; //add includes foreach (string inc in compilerArg.includeDirs) { args += " /I\"" + inc + "\" "; } //add preprocessing flag args += "/P /Fi" + tempFolder + "\\ "; args += "/MP" + TaskCapacity; //add source files foreach (string srcFile in compilerArg.files) { args += " " + srcFile; } proc.StartInfo.Arguments = args; proc.StartInfo.FileName = compilerArg.compiler; proc.StartInfo.WindowStyle = ProcessWindowStyle.Hidden; proc.StartInfo.UseShellExecute = false; proc.StartInfo.RedirectStandardInput = false; proc.StartInfo.RedirectStandardOutput = true; proc.StartInfo.RedirectStandardError = true; proc.StartInfo.CreateNoWindow = true; proc.Start(); System.Text.StringBuilder outputStr = new System.Text.StringBuilder(); proc.OutputDataReceived += (sender, e) => { if (e.Data != null) { string output = e.Data; Logger.LogMessage(output); } }; proc.ErrorDataReceived += (sender, e) => { if (e.Data != null) { outputStr.AppendLine(e.Data); } }; proc.BeginOutputReadLine(); proc.BeginErrorReadLine(); proc.WaitForExit(); Logger.LogMessage(outputStr.ToString()); if (proc.ExitCode == 0) { //now read back the files and add them to tar foreach (string srcFile in compilerArg.files) { //TODO: this might be inside a folder string precompiledFile = tempFolder + "\\" + Path.GetFileNameWithoutExtension(srcFile) + ".i"; TarEntry entry = TarEntry.CreateEntryFromFile(precompiledFile); entry.Name = Path.GetFileName(srcFile); archive.PutNextEntry(entry); using (Stream inputStream = File.OpenRead(precompiledFile)) { writeStreamToTar(archive, inputStream); archive.CloseEntry(); } } } else { Logger.LogError($"Preprocessing of file package failed"); } } Directory.Delete(tempFolder, true); // Add compilation commands, once per CompilerArg foreach (CompilerArg compilerArg in compilerArgs) { //remove precompiled header args /Yu /Fp Match match = Regex.Match(compilerArg.args, "/Yu\".+?\""); if (match.Success) { compilerArg.args = compilerArg.args.Remove(match.Index, match.Length); } match = Regex.Match(compilerArg.args, "/Fp\".+?\""); if (match.Success) { compilerArg.args = compilerArg.args.Remove(match.Index, match.Length); } compilerArg.args += " /FS"; compilerArg.args += " /Fo\"output/\""; compilerArg.args += " /Fd\"output/" + Path.GetFileNameWithoutExtension(compilerArg.files[0]) + ".pdb\""; compilerArg.args += " /MP" + TaskCapacity; batch.Write("\"../" + Path.GetFileName(compilerArg.compiler) + "\" " + compilerArg.args); foreach (string file in compilerArg.files) { batch.Write(" " + Path.GetFileName(file)); } batch.WriteLine(); } // Zip output folder batch.WriteLine("powershell.exe -nologo -noprofile -command \"& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::CreateFromDirectory('output', 'output.zip'); }\""); // stop the service batch.WriteLine("\"../mspdbsrv.exe\" -stop"); batch.WriteLine("exit 0");//assume no error batch.Close(); TarEntry batchEntry = TarEntry.CreateEntryFromFile("golembuild.bat"); batchEntry.Name = "golembuild.bat"; using (Stream inputStream = File.OpenRead("golembuild.bat")) { batchEntry.Size = inputStream.Length; archive.PutNextEntry(batchEntry); writeStreamToTar(archive, inputStream); archive.CloseEntry(); } } package = memoryStream.ToArray(); string hash = GolemCache.RegisterTasksPackage(package); FileStream debug = new FileStream(hash + ".tar", FileMode.Create); debug.Write(package, 0, package.Length); debug.Close(); return(hash); } }
private string PackFiles(List <CompilationTask> taskList) { byte[] package; using (var memoryStream = new MemoryStream()) { using (var archive = TarArchive.CreateOutputTarArchive(memoryStream)) { List <string> addedEntries = new List <string>(); // Package precompiled header if used foreach (CompilationTask task in taskList) { if (task.PrecompiledHeader.Length > 0) { TarEntry entry = TarEntry.CreateEntryFromFile(task.PrecompiledHeader); entry.Name = Path.GetFileName(task.PrecompiledHeader); archive.WriteEntry(entry, false); break; } } foreach (CompilationTask task in taskList) { // Package sourcefiles // not needed since this file is already in the task.Includes // TODO: should it be like this? { TarEntry entry = TarEntry.CreateEntryFromFile(task.FilePath); entry.Name = Path.GetFileName(task.FilePath); archive.WriteEntry(entry, false); } // Package includes string projectPath = task.ProjectPath; string dstLibIncludePath = "includes"; string dstProjectIncludePath = ""; foreach (string include in task.Includes) { string dstFilePath = null; if (include.StartsWith(projectPath)) { string relative = include.Replace(projectPath, "").TrimStart('\\', '/'); dstFilePath = Path.Combine(dstProjectIncludePath, relative); } else { for (int i = 0; i < task.IncludeDirs.Count; ++i) { string srcIncludePath = task.IncludeDirs[i]; if (include.StartsWith(srcIncludePath)) { string relative = include.Replace(srcIncludePath, "").TrimStart('\\', '/'); dstFilePath = Path.Combine(dstLibIncludePath + i.ToString(), relative); break; } } } AddFileToTar(archive, include, dstFilePath, addedEntries); } } // Package build batch TextWriter batch = new StreamWriter("golembuild.bat", false); // CD to the directory the batch file is in batch.WriteLine("cd %~DP0"); // Create output folder batch.WriteLine("mkdir output"); int numberOfIncludeDirs = 0; List <CompilerArg> compilerArgs = new List <CompilerArg>(); foreach (CompilationTask task in taskList) { bool found = false; foreach (CompilerArg compilerArg in compilerArgs) { if (compilerArg.compiler == task.Compiler && compilerArg.args == task.CompilerArgs) { compilerArg.files.Add(Path.GetFileName(task.FilePath)); found = true; break; } } if (found) { continue; } numberOfIncludeDirs = Math.Max(numberOfIncludeDirs, task.IncludeDirs.Count); CompilerArg newCompilerArg = new CompilerArg(); newCompilerArg.compiler = task.Compiler; newCompilerArg.args = task.CompilerArgs; newCompilerArg.files.Add(Path.GetFileName(task.FilePath)); compilerArgs.Add(newCompilerArg); } // Add compilation commands, once per CompilerArg foreach (CompilerArg compilerArg in compilerArgs) { for (int i = 0; i < numberOfIncludeDirs; ++i) { compilerArg.args += " /I\"includes" + i.ToString() + "\" /FS"; } compilerArg.args += " /Fo\"output/\""; compilerArg.args += " /Fd\"output/" + Path.GetFileNameWithoutExtension(compilerArg.files[0]) + ".pdb\""; compilerArg.args += " /MP" + TaskCapacity; batch.Write("\"../" + Path.GetFileName(compilerArg.compiler) + "\" " + compilerArg.args); foreach (string file in compilerArg.files) { batch.Write(" " + file); } batch.WriteLine(); } // Zip output folder batch.WriteLine("powershell.exe -nologo -noprofile -command \"& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::CreateFromDirectory('output', 'output.zip'); }\""); // stop the service batch.WriteLine("\"../mspdbsrv.exe\" -stop"); batch.WriteLine("exit 0");//assume no error batch.Close(); TarEntry batchEntry = TarEntry.CreateEntryFromFile("golembuild.bat"); batchEntry.Name = "golembuild.bat"; archive.WriteEntry(batchEntry, false); } package = memoryStream.ToArray(); string hash = GolemCache.RegisterTasksPackage(package); FileStream debug = new FileStream(hash + ".tar", FileMode.Create); debug.Write(package, 0, package.Length); debug.Close(); return(hash); } }
private void ProcessRequest(HttpListenerContext context) { HttpListenerRequest request = context.Request; // Are they trying to upload a file? if (request.HttpMethod == "PUT") { System.IO.Stream input = request.InputStream; string test = GolemBuildService.Instance.BuildPath; string test2 = Path.GetFileNameWithoutExtension(request.RawUrl) + ".zip"; string zipName = Path.Combine(test, test2); FileStream fileStream = File.Create(zipName); input.CopyTo(fileStream); fileStream.Close(); input.Close(); // Send back OK HttpListenerResponse response = context.Response; response.Headers.Clear(); response.SendChunked = false; response.StatusCode = 201; response.AddHeader("Content-Location", zipName); //response.AddHeader("Server", String.Empty); //response.AddHeader("Date", String.Empty); response.Close(); ZipFile.ExtractToDirectory(zipName, Path.GetDirectoryName(zipName)); } else // They are trying to download a file { // Obtain a response object. HttpListenerResponse response = context.Response; //let's check ranges long offset = 0; long size = -1; foreach (string header in request.Headers.AllKeys) { if (header == "Range") { string[] values = request.Headers.GetValues(header); string[] tokens = values[0].Split('=', '-'); offset = int.Parse(tokens[1]); size = (int)(int.Parse(tokens[2]) - offset + 1); } } try { // Are they requesting a CompilerPackage? if (request.RawUrl.StartsWith("/requestID/compiler/")) { string compilerHash = request.RawUrl.Replace("/requestID/compiler/", ""); byte[] data; if (GolemCache.GetCompilerPackageData(compilerHash, out data)) { response.AddHeader("ETag", "SHA1:" + compilerHash); if (size == -1) { size = data.Length; } response.ContentLength64 = size; Stream output = response.OutputStream; output.Write(data, (int)offset, (int)size); output.Close(); } } // Or are they requesting a tasks package? else if (request.RawUrl.StartsWith("/requestID/tasks/")) { string tasksPackageHash = request.RawUrl.Replace("/requestID/tasks/", ""); byte[] data; if (GolemCache.GetTasksPackage(tasksPackageHash, out data)) { response.AddHeader("ETag", "SHA1:" + tasksPackageHash); if (size == -1) { size = data.Length; } response.ContentLength64 = size; Stream output = response.OutputStream; output.Write(data, (int)offset, (int)size); output.Close(); } } } catch (HttpListenerException ex) { Logger.LogMessage(ex.Message); } } }