/// <summary> /// Gets the next request from the request queue. /// </summary> /// <param name="block">Whether to block.</param> /// <returns>Next request on the queue.</returns> public CloudExecutionRequest GetRequest(bool block = false) { int pollInterval = 1000; // In milliseconds. CloudQueueMessage message = null; while ((message = this.requestQueue.GetMessage()) == null) { if (block) { Thread.Sleep(pollInterval); // Increase poll interval if we're below the cap. if (pollInterval < RequestQueuePollIntervalCap) { // Increase by 1/10th of a second each time. // This will take 14.5s to reach a 2s interval. pollInterval += 100; } continue; } else { return(null); } } ////Console.WriteLine(message.AsString); CloudExecutionRequest request = CloudExecutionRequest.FromMessage(message); return(request); }
/// <summary> /// Creates a CloudExecutionRequest from a CloudQueueMessage /// representation. /// </summary> /// <param name="message"> /// A CloudQueueMessage containing an XML document representing a /// CloudExecutionRequest. /// </param> /// <returns> /// A new request corresponding to the XML representation contained in /// the message. /// </returns> public static CloudExecutionRequest FromMessage(CloudQueueMessage message) { CloudExecutionRequest request = CloudExecutionRequest.FromXml(message.AsString); request.message = message; return(request); }
/// <summary> /// Initializes a new instance of the CloudExecutionQueue class. /// </summary> /// <remarks> /// This is the constructor for clients (does extra stuff). /// </remarks> /// <param name="reportQueueName">Name of the report queue this client is using.</param> public CloudExecutionQueue(string reportQueueName) : this() { if (string.IsNullOrEmpty(reportQueueName)) { // Use default shared report queue if none is specified. reportQueueName = CloudExecutionQueue.ReportQueueName; } else { // Schedule auto-deletion of our private report queue. // We do this by setting the initial invisibility of a delete queue request. // This will auto-clean up our private report queue if we exit unexpectedly. CloudExecutionRequest deleteQueueRequest = new CloudExecutionRequest( reportQueueName, reportQueueName, CloudExecutionRequest.Operation.DeleteQueue, null, null, null, null ); CloudQueueMessage deleteQueueMessage = new CloudQueueMessage(deleteQueueRequest.ToXml()); this.requestQueue.AddMessage(deleteQueueMessage, new TimeSpan(12, 0, 0), new TimeSpan(4, 0, 0)); } this.clientReportQueue = this.queueClient.GetQueueReference(reportQueueName); this.clientReportQueue.CreateIfNotExists(); this.reportWaiters = new Dictionary <string, CloudExecutionWaiter>(); this.reportWaitersLock = new Mutex(); this.haveReportWaiters = new ManualResetEvent(false); this.monitoringThread = new Thread(new ThreadStart(this.MonitorReportQueue)); this.monitoringThread.IsBackground = true; this.monitoringThread.Name = "Report Queue Monitor"; this.monitoringThread.Start(); }
/// <summary> /// Run the requested executable and produce a report of the results. /// </summary> /// <param name="executionRequest">The execution request.</param> /// <returns>A report of the results.</returns> private CloudExecutionReport RunAnExecutable(CloudExecutionRequest executionRequest) { // REVIEW: How/whether to use this. BuildObject diagnosticsBase = new BuildObject(Path.Combine("nuobj", "diagnostics", "process")); // Prep working directory with input files and output dirs. // TODO: The below will throw cache exceptions if something // isn't there (they should all be though). Need to catch // these and fail the execution request when this happens. WorkingDirectory workingDirectory = new WorkingDirectory(this.virtualIronRoot); foreach (BuildObjectValuePointer inputFile in executionRequest.InputFileMappings) { // REVIEW: How to determine cache container here. ItemCacheContainer container = ItemCacheContainer.Sources; if (this.multiplexedItemCache.GetItemSize(container, inputFile.ObjectHash) == -1) { container = ItemCacheContainer.Objects; } // TODO: Move path/directory manipulation code into // WorkingDirectory and/or ItemCache. string inputFilePath = workingDirectory.PathTo(inputFile.RelativePath); Directory.CreateDirectory(Path.GetDirectoryName(inputFilePath)); // REVIEW: Still neeeded? this.multiplexedItemCache.FetchItemToFile( container, inputFile.ObjectHash, inputFilePath); } foreach (BuildObject outputFile in executionRequest.OutputFiles) { workingDirectory.CreateDirectoryFor(outputFile); } // Run executable. ProcessInvoker pinv = new ProcessInvoker( workingDirectory, executionRequest.Executable, new string[] { executionRequest.Arguments }, diagnosticsBase, null, // This is captureStdout. TODO: Should cleanup how this is used in ProcessInvoker. null); // This is dbgText. REVIEW: How/whether to use this. // When ProcessInvoker's constructor returns, the process has // finished running. Console.WriteLine("Request {0} completed in {1} seconds.", executionRequest.Identifier, pinv.CpuTime); // Store output files in the (cloud) item cache, and create a // list of the mappings. List<BuildObjectValuePointer> outputFileMappings = new List<BuildObjectValuePointer>(); foreach (BuildObject outFile in executionRequest.OutputFiles) { if (File.Exists(workingDirectory.PathTo(outFile))) { string fileHash = Util.hashFilesystemPath(workingDirectory.PathTo(outFile)); Util.Assert(!string.IsNullOrEmpty(fileHash)); // Note we explicitly write to the cloud cache here. this.cloudCache.StoreItemFromFile(ItemCacheContainer.Objects, fileHash, workingDirectory.PathTo(outFile)); outputFileMappings.Add(new BuildObjectValuePointer(fileHash, outFile.getRelativePath())); } } // Collect the results into a report. CloudExecutionReport report = new CloudExecutionReport( executionRequest.Identifier, CloudExecutionReport.StatusCode.Completed, pinv.ExitCode, pinv.GetStdout(), pinv.GetStderr(), pinv.CpuTime, outputFileMappings); return report; }
/// <summary> /// Deletes a request from the queue. /// Used by reader to indicate request completion. /// </summary> /// <param name="request">Request to delete.</param> public void DeleteRequest(CloudExecutionRequest request) { CloudQueueMessage message = request.Message; this.requestQueue.DeleteMessage(message); }
/// <summary> /// Submits a request for cloud execution. /// </summary> /// <param name="request"> /// The CloudExecutionRequest detailing this request. /// </param> public void SubmitRequest(CloudExecutionRequest request) { CloudQueueMessage message = new CloudQueueMessage(request.ToXml()); this.requestQueue.AddMessage(message, this.queueEntryTtl); }
/// <summary> /// Initializes a new instance of the CloudExecutionQueue class. /// </summary> /// <remarks> /// This is the constructor for clients (does extra stuff). /// </remarks> /// <param name="reportQueueName">Name of the report queue this client is using.</param> public CloudExecutionQueue(string reportQueueName) : this() { if (string.IsNullOrEmpty(reportQueueName)) { // Use default shared report queue if none is specified. reportQueueName = CloudExecutionQueue.ReportQueueName; } else { // Schedule auto-deletion of our private report queue. // We do this by setting the initial invisibility of a delete queue request. // This will auto-clean up our private report queue if we exit unexpectedly. CloudExecutionRequest deleteQueueRequest = new CloudExecutionRequest( reportQueueName, reportQueueName, CloudExecutionRequest.Operation.DeleteQueue, null, null, null, null ); CloudQueueMessage deleteQueueMessage = new CloudQueueMessage(deleteQueueRequest.ToXml()); this.requestQueue.AddMessage(deleteQueueMessage, new TimeSpan(12, 0, 0), new TimeSpan(4, 0, 0)); } this.clientReportQueue = this.queueClient.GetQueueReference(reportQueueName); this.clientReportQueue.CreateIfNotExists(); this.reportWaiters = new Dictionary<string, CloudExecutionWaiter>(); this.reportWaitersLock = new Mutex(); this.haveReportWaiters = new ManualResetEvent(false); this.monitoringThread = new Thread(new ThreadStart(this.MonitorReportQueue)); this.monitoringThread.IsBackground = true; this.monitoringThread.Name = "Report Queue Monitor"; this.monitoringThread.Start(); }
/// <summary> /// Initializes a new instance of the CloudSubmitter class. /// </summary> /// <param name="requestIdentifier"> /// Unique identifier for this request. /// </param> /// <param name="workingDirectory"> /// Working directory the process is started in. /// </param> /// <param name="inputFiles"> /// List of input files expected by the process. /// </param> /// <param name="outputFiles"> /// List of potential output files generated by the process. /// </param> /// <param name="executable">Executable to run.</param> /// <param name="args"> /// Command line arguments to provide to the executable. /// </param> /// <param name="failureBase">Not sure what this is -- some debugging/diagnostic thing.</param> /// <param name="captureStdout">Where to (optionally) store the captured standard out.</param> /// <param name="dbgText">Debugging text for something or another.</param> public CloudSubmitter( string requestIdentifier, WorkingDirectory workingDirectory, IEnumerable <BuildObject> inputFiles, IEnumerable <BuildObject> outputFiles, string executable, string[] args, BuildObject failureBase, BuildObject captureStdout = null, string dbgText = null) { // Catch bad verb authors before they hurt themselves. Util.Assert(!executable.Contains(":")); // Hey, this looks like an absolute path! Use .getRelativePath(); it makes your output more stable. foreach (string arg in args) { // Pardon my distasteful heuristic to avoid flagging /flag:value args. Util.Assert(arg.Length < 2 || arg[1] != ':'); // Hey, this looks like an absolute path! Use .getRelativePath() to tolerate crossing machine boundaries. } // Stash away things we'll want to remember later. this.workingDirectory = workingDirectory; // Create list of input file mappings. // REVIEW: We're not running on the main thread at this point. Are below calls all thread-safe? List <BuildObjectValuePointer> inputFileMappings = new List <BuildObjectValuePointer>(); foreach (BuildObject file in inputFiles) { string fileHash = BuildEngine.theEngine.Repository.GetHash(file); Util.Assert(!string.IsNullOrEmpty(fileHash)); inputFileMappings.Add(new BuildObjectValuePointer(fileHash, file.getRelativePath())); // Ensure that the input files are in the cloud cache. // REVIEW: best way to determine this is a source file? ItemCacheContainer container; if (file is SourcePath) { container = ItemCacheContainer.Sources; } else { container = ItemCacheContainer.Objects; } ItemCacheMultiplexer multiplexedCache = BuildEngine.theEngine.ItemCache as ItemCacheMultiplexer; Util.Assert(multiplexedCache != null); multiplexedCache.SyncItemToCloud(container, fileHash); } // Prepare cloud execution request for submission. string arguments = string.Join(" ", args); CloudExecutionRequest request = new CloudExecutionRequest( BuildEngine.theEngine.CloudReportQueueName, requestIdentifier, CloudExecutionRequest.Operation.RunExecutable, executable, arguments, inputFileMappings, outputFiles); BuildEngine.theEngine.CloudExecutionQueue.SubmitRequest(request); // Wait for remote execution to finish. int requestsOutstanding; Console.WriteLine("Waiting on remote execution report for request '{0}'.", requestIdentifier); CloudExecutionReport executionReport = BuildEngine.theEngine.CloudExecutionQueue.GetReport(requestIdentifier, out requestsOutstanding); Console.WriteLine("Received remote execution report for request '{0}'. {1} others still outstanding.", requestIdentifier, requestsOutstanding); // Record what we got back. this.exitCode = executionReport.ExitCode; this.stdout = executionReport.StandardOutput; this.stderr = executionReport.StandardError; this.cpuTime = executionReport.CpuTime; // Copy output files from cloud cache back to local working dir. // REVIEW: This is just to set things up as expected for the // Scheduler's recordResult routine. Could re-architect this to // be more efficient for the remote execution case. foreach (BuildObjectValuePointer outputFileMapping in executionReport.OutputFileMappings) { BuildEngine.theEngine.CloudCache.FetchItemToFile( ItemCacheContainer.Objects, outputFileMapping.ObjectHash, workingDirectory.PathTo(outputFileMapping.RelativePath)); } }
/// <summary> /// Initializes a new instance of the CloudSubmitter class. /// </summary> /// <param name="requestIdentifier"> /// Unique identifier for this request. /// </param> /// <param name="workingDirectory"> /// Working directory the process is started in. /// </param> /// <param name="inputFiles"> /// List of input files expected by the process. /// </param> /// <param name="outputFiles"> /// List of potential output files generated by the process. /// </param> /// <param name="executable">Executable to run.</param> /// <param name="args"> /// Command line arguments to provide to the executable. /// </param> /// <param name="failureBase">Not sure what this is -- some debugging/diagnostic thing.</param> /// <param name="captureStdout">Where to (optionally) store the captured standard out.</param> /// <param name="dbgText">Debugging text for something or another.</param> public CloudSubmitter( string requestIdentifier, WorkingDirectory workingDirectory, IEnumerable<BuildObject> inputFiles, IEnumerable<BuildObject> outputFiles, string executable, string[] args, BuildObject failureBase, BuildObject captureStdout = null, string dbgText = null) { // Catch bad verb authors before they hurt themselves. Util.Assert(!executable.Contains(":")); // Hey, this looks like an absolute path! Use .getRelativePath(); it makes your output more stable. foreach (string arg in args) { // Pardon my distasteful heuristic to avoid flagging /flag:value args. Util.Assert(arg.Length < 2 || arg[1] != ':'); // Hey, this looks like an absolute path! Use .getRelativePath() to tolerate crossing machine boundaries. } // Stash away things we'll want to remember later. this.workingDirectory = workingDirectory; // Create list of input file mappings. // REVIEW: We're not running on the main thread at this point. Are below calls all thread-safe? List<BuildObjectValuePointer> inputFileMappings = new List<BuildObjectValuePointer>(); foreach (BuildObject file in inputFiles) { string fileHash = BuildEngine.theEngine.Repository.GetHash(file); Util.Assert(!string.IsNullOrEmpty(fileHash)); inputFileMappings.Add(new BuildObjectValuePointer(fileHash, file.getRelativePath())); // Ensure that the input files are in the cloud cache. // REVIEW: best way to determine this is a source file? ItemCacheContainer container; if (file is SourcePath) { container = ItemCacheContainer.Sources; } else { container = ItemCacheContainer.Objects; } ItemCacheMultiplexer multiplexedCache = BuildEngine.theEngine.ItemCache as ItemCacheMultiplexer; Util.Assert(multiplexedCache != null); multiplexedCache.SyncItemToCloud(container, fileHash); } // Prepare cloud execution request for submission. string arguments = string.Join(" ", args); CloudExecutionRequest request = new CloudExecutionRequest( BuildEngine.theEngine.CloudReportQueueName, requestIdentifier, CloudExecutionRequest.Operation.RunExecutable, executable, arguments, inputFileMappings, outputFiles); BuildEngine.theEngine.CloudExecutionQueue.SubmitRequest(request); // Wait for remote execution to finish. int requestsOutstanding; Console.WriteLine("Waiting on remote execution report for request '{0}'.", requestIdentifier); CloudExecutionReport executionReport = BuildEngine.theEngine.CloudExecutionQueue.GetReport(requestIdentifier, out requestsOutstanding); Console.WriteLine("Received remote execution report for request '{0}'. {1} others still outstanding.", requestIdentifier, requestsOutstanding); // Record what we got back. this.exitCode = executionReport.ExitCode; this.stdout = executionReport.StandardOutput; this.stderr = executionReport.StandardError; this.cpuTime = executionReport.CpuTime; // Copy output files from cloud cache back to local working dir. // REVIEW: This is just to set things up as expected for the // Scheduler's recordResult routine. Could re-architect this to // be more efficient for the remote execution case. foreach (BuildObjectValuePointer outputFileMapping in executionReport.OutputFileMappings) { BuildEngine.theEngine.CloudCache.FetchItemToFile( ItemCacheContainer.Objects, outputFileMapping.ObjectHash, workingDirectory.PathTo(outputFileMapping.RelativePath)); } }
/// <summary> /// Deletes a request from the queue. /// Used by reader to indicate request completion. /// </summary> /// <param name="request">Request to delete.</param> public void DeleteRequest(CloudExecutionRequest request) { CloudQueueMessage message = request.Message; this.requestQueue.DeleteMessage(message); }
/// <summary> /// Submits a request for cloud execution. /// </summary> /// <param name="request"> /// The CloudExecutionRequest detailing this request. /// </param> public void SubmitRequest(CloudExecutionRequest request) { CloudQueueMessage message = new CloudQueueMessage(request.ToXml()); this.requestQueue.AddMessage(message, this.queueEntryTtl); }