/// <summary>
/// Creates a CloudExecutionReport from a CloudQueueMessage
/// representation.
/// </summary>
/// <param name="message">
/// A CloudQueueMessage containing an XML document representing a
/// CloudExecutionReport.
/// </param>
/// <returns>
/// A new report corresponding to the XML representation contained in
/// the message.
/// </returns>
public static CloudExecutionReport FromMessage(CloudQueueMessage message)
{
CloudExecutionReport report = CloudExecutionReport.FromXml(message.AsString);
report.message = message;
return(report);
}
/// <summary>
/// Submits a report of cloud execution for processing by the submitter.
/// </summary>
/// <param name="report">Report to submit.</param>
/// <param name="reportQueueName">Name of the report queue to submit to.</param>
public void SubmitReport(CloudExecutionReport report, string reportQueueName)
{
// Check to see if we need to update the cached report queue.
if (reportQueueName != this.cachedServerReportQueueName)
{
this.cachedServerReportQueue = this.queueClient.GetQueueReference(reportQueueName);
this.cachedServerReportQueue.CreateIfNotExists();
this.cachedServerReportQueueName = reportQueueName;
}
CloudQueueMessage message = new CloudQueueMessage(report.ToXml());
this.cachedServerReportQueue.AddMessage(message, this.queueEntryTtl);
}
/// <summary>
/// Give an execution report to this waiter.
/// </summary>
/// <param name="executionReport">The execution report to give.</param>
/// <param name="numberOfOtherWaiters">Count of other report waiters.</param>
public void GiveReportToWaiter(CloudExecutionReport executionReport, int numberOfOtherWaiters)
{
this.executionReport = executionReport;
this.otherWaitersCount = numberOfOtherWaiters;
this.reportIsReady.Set();
}
/// <summary>
/// Runs the execution engine, which eternally processes execution
/// requests posted to the main queue.
/// </summary>
public void Run()
{
this.keepSwimming = true;
while (this.keepSwimming)
{
// Pull request off queue.
Console.WriteLine("Waiting for new work request.");
CloudExecutionRequest executionRequest = this.mainQueue.GetRequest(block: true);
if (!this.keepSwimming)
{
// Exit without deleting request from queue.
// Request will become visible again after timeout expires.
break;
}
// TODO: For now, we immediately delete the requests we take.
// Eventually we'll want to leave the request invisible until
// some timeout expires so that other engines can pick it up
// if we crash.
this.mainQueue.DeleteRequest(executionRequest);
Console.WriteLine("Received request with ID {0}.", executionRequest.Identifier);
// Make a progress report to soothe nervous users.
CloudExecutionReport statusUpdate = new CloudExecutionReport(
executionRequest.Identifier,
CloudExecutionReport.StatusCode.InProgress,
0,
null,
null,
0,
null);
// Submit progress report.
this.mainQueue.SubmitReport(statusUpdate, executionRequest.ReportQueue);
// Do the requested operation.
CloudExecutionReport report = null;
switch (executionRequest.RequestedOperation)
{
case CloudExecutionRequest.Operation.RunExecutable:
report = this.RunAnExecutable(executionRequest);
break;
case CloudExecutionRequest.Operation.CommitSuicide:
this.keepSwimming = false;
goto case CloudExecutionRequest.Operation.None;
case CloudExecutionRequest.Operation.DeleteQueue:
this.mainQueue.DeleteQueue(executionRequest.ReportQueue);
goto case CloudExecutionRequest.Operation.None;
case CloudExecutionRequest.Operation.None:
report = new CloudExecutionReport(
executionRequest.Identifier,
CloudExecutionReport.StatusCode.Completed);
break;
default:
// ToDo: This should probably be treated as an error.
goto case CloudExecutionRequest.Operation.None;
}
// Submit report.
this.mainQueue.SubmitReport(report, executionRequest.ReportQueue);
}
}
/// <summary>
/// Run the requested executable and produce a report of the results.
/// </summary>
/// <param name="executionRequest">The execution request.</param>
/// <returns>A report of the results.</returns>
private CloudExecutionReport RunAnExecutable(CloudExecutionRequest executionRequest)
{
// REVIEW: How/whether to use this.
BuildObject diagnosticsBase = new BuildObject(Path.Combine("nuobj", "diagnostics", "process"));
// Prep working directory with input files and output dirs.
// TODO: The below will throw cache exceptions if something
// isn't there (they should all be though). Need to catch
// these and fail the execution request when this happens.
WorkingDirectory workingDirectory = new WorkingDirectory(this.virtualIronRoot);
foreach (BuildObjectValuePointer inputFile in executionRequest.InputFileMappings)
{
// REVIEW: How to determine cache container here.
ItemCacheContainer container = ItemCacheContainer.Sources;
if (this.multiplexedItemCache.GetItemSize(container, inputFile.ObjectHash) == -1)
{
container = ItemCacheContainer.Objects;
}
// TODO: Move path/directory manipulation code into
// WorkingDirectory and/or ItemCache.
string inputFilePath = workingDirectory.PathTo(inputFile.RelativePath);
Directory.CreateDirectory(Path.GetDirectoryName(inputFilePath)); // REVIEW: Still neeeded?
this.multiplexedItemCache.FetchItemToFile(
container,
inputFile.ObjectHash,
inputFilePath);
}
foreach (BuildObject outputFile in executionRequest.OutputFiles)
{
workingDirectory.CreateDirectoryFor(outputFile);
}
// Run executable.
ProcessInvoker pinv = new ProcessInvoker(
workingDirectory,
executionRequest.Executable,
new string[] { executionRequest.Arguments },
diagnosticsBase,
null, // This is captureStdout. TODO: Should cleanup how this is used in ProcessInvoker.
null); // This is dbgText. REVIEW: How/whether to use this.
// When ProcessInvoker's constructor returns, the process has
// finished running.
Console.WriteLine("Request {0} completed in {1} seconds.", executionRequest.Identifier, pinv.CpuTime);
// Store output files in the (cloud) item cache, and create a
// list of the mappings.
List<BuildObjectValuePointer> outputFileMappings = new List<BuildObjectValuePointer>();
foreach (BuildObject outFile in executionRequest.OutputFiles)
{
if (File.Exists(workingDirectory.PathTo(outFile)))
{
string fileHash = Util.hashFilesystemPath(workingDirectory.PathTo(outFile));
Util.Assert(!string.IsNullOrEmpty(fileHash));
// Note we explicitly write to the cloud cache here.
this.cloudCache.StoreItemFromFile(ItemCacheContainer.Objects, fileHash, workingDirectory.PathTo(outFile));
outputFileMappings.Add(new BuildObjectValuePointer(fileHash, outFile.getRelativePath()));
}
}
// Collect the results into a report.
CloudExecutionReport report = new CloudExecutionReport(
executionRequest.Identifier,
CloudExecutionReport.StatusCode.Completed,
pinv.ExitCode,
pinv.GetStdout(),
pinv.GetStderr(),
pinv.CpuTime,
outputFileMappings);
return report;
}
/// <summary>
/// Submits a report of cloud execution for processing by the submitter.
/// </summary>
/// <param name="report">Report to submit.</param>
/// <param name="reportQueueName">Name of the report queue to submit to.</param>
public void SubmitReport(CloudExecutionReport report, string reportQueueName)
{
// Check to see if we need to update the cached report queue.
if (reportQueueName != this.cachedServerReportQueueName)
{
this.cachedServerReportQueue = this.queueClient.GetQueueReference(reportQueueName);
this.cachedServerReportQueue.CreateIfNotExists();
this.cachedServerReportQueueName = reportQueueName;
}
CloudQueueMessage message = new CloudQueueMessage(report.ToXml());
this.cachedServerReportQueue.AddMessage(message, this.queueEntryTtl);
}
/// <summary>
/// Initializes a new instance of the CloudSubmitter class.
/// </summary>
/// <param name="requestIdentifier">
/// Unique identifier for this request.
/// </param>
/// <param name="workingDirectory">
/// Working directory the process is started in.
/// </param>
/// <param name="inputFiles">
/// List of input files expected by the process.
/// </param>
/// <param name="outputFiles">
/// List of potential output files generated by the process.
/// </param>
/// <param name="executable">Executable to run.</param>
/// <param name="args">
/// Command line arguments to provide to the executable.
/// </param>
/// <param name="failureBase">Not sure what this is -- some debugging/diagnostic thing.</param>
/// <param name="captureStdout">Where to (optionally) store the captured standard out.</param>
/// <param name="dbgText">Debugging text for something or another.</param>
public CloudSubmitter(
string requestIdentifier,
WorkingDirectory workingDirectory,
IEnumerable <BuildObject> inputFiles,
IEnumerable <BuildObject> outputFiles,
string executable,
string[] args,
BuildObject failureBase,
BuildObject captureStdout = null,
string dbgText = null)
{
// Catch bad verb authors before they hurt themselves.
Util.Assert(!executable.Contains(":")); // Hey, this looks like an absolute path! Use .getRelativePath(); it makes your output more stable.
foreach (string arg in args)
{
// Pardon my distasteful heuristic to avoid flagging /flag:value args.
Util.Assert(arg.Length < 2 || arg[1] != ':'); // Hey, this looks like an absolute path! Use .getRelativePath() to tolerate crossing machine boundaries.
}
// Stash away things we'll want to remember later.
this.workingDirectory = workingDirectory;
// Create list of input file mappings.
// REVIEW: We're not running on the main thread at this point. Are below calls all thread-safe?
List <BuildObjectValuePointer> inputFileMappings = new List <BuildObjectValuePointer>();
foreach (BuildObject file in inputFiles)
{
string fileHash = BuildEngine.theEngine.Repository.GetHash(file);
Util.Assert(!string.IsNullOrEmpty(fileHash));
inputFileMappings.Add(new BuildObjectValuePointer(fileHash, file.getRelativePath()));
// Ensure that the input files are in the cloud cache.
// REVIEW: best way to determine this is a source file?
ItemCacheContainer container;
if (file is SourcePath)
{
container = ItemCacheContainer.Sources;
}
else
{
container = ItemCacheContainer.Objects;
}
ItemCacheMultiplexer multiplexedCache = BuildEngine.theEngine.ItemCache as ItemCacheMultiplexer;
Util.Assert(multiplexedCache != null);
multiplexedCache.SyncItemToCloud(container, fileHash);
}
// Prepare cloud execution request for submission.
string arguments = string.Join(" ", args);
CloudExecutionRequest request = new CloudExecutionRequest(
BuildEngine.theEngine.CloudReportQueueName,
requestIdentifier,
CloudExecutionRequest.Operation.RunExecutable,
executable,
arguments,
inputFileMappings,
outputFiles);
BuildEngine.theEngine.CloudExecutionQueue.SubmitRequest(request);
// Wait for remote execution to finish.
int requestsOutstanding;
Console.WriteLine("Waiting on remote execution report for request '{0}'.", requestIdentifier);
CloudExecutionReport executionReport = BuildEngine.theEngine.CloudExecutionQueue.GetReport(requestIdentifier, out requestsOutstanding);
Console.WriteLine("Received remote execution report for request '{0}'. {1} others still outstanding.", requestIdentifier, requestsOutstanding);
// Record what we got back.
this.exitCode = executionReport.ExitCode;
this.stdout = executionReport.StandardOutput;
this.stderr = executionReport.StandardError;
this.cpuTime = executionReport.CpuTime;
// Copy output files from cloud cache back to local working dir.
// REVIEW: This is just to set things up as expected for the
// Scheduler's recordResult routine. Could re-architect this to
// be more efficient for the remote execution case.
foreach (BuildObjectValuePointer outputFileMapping in executionReport.OutputFileMappings)
{
BuildEngine.theEngine.CloudCache.FetchItemToFile(
ItemCacheContainer.Objects,
outputFileMapping.ObjectHash,
workingDirectory.PathTo(outputFileMapping.RelativePath));
}
}
/// <summary>
/// Procedure performed by the report queue monitoring thread.
/// </summary>
private void MonitorReportQueue()
{
const int MaxBatchSize = 32; // Maximum allowed by API.
TimeSpan invisibleTime = new TimeSpan(0, 0, 2); // Two seconds.
TimeSpan pollInterval = new TimeSpan(0, 0, 1); // One second.
List <CloudQueueMessage> messages;
bool anyForUs;
while (true)
{
// Don't bother doing anything if we don't have any active waiters.
this.haveReportWaiters.WaitOne();
// Get a new batch of messages.
// No other queue readers will see these until the invisibleTime expires.
messages = new List <CloudQueueMessage>(this.clientReportQueue.GetMessages(MaxBatchSize, invisibleTime));
anyForUs = false;
foreach (CloudQueueMessage message in messages)
{
CloudExecutionReport report = CloudExecutionReport.FromMessage(message);
// Lookup the waiter for this report under lock.
CloudExecutionWaiter waiter;
int remainingWaitersCount = 0;
this.reportWaitersLock.WaitOne();
bool foundWaiter = this.reportWaiters.TryGetValue(report.Identifier, out waiter);
if (foundWaiter && report.Status == CloudExecutionReport.StatusCode.Completed)
{
this.reportWaiters.Remove(report.Identifier);
remainingWaitersCount = this.reportWaiters.Count;
if (remainingWaitersCount == 0)
{
// We don't have anyone else waiting for a report.
this.haveReportWaiters.Reset();
}
}
this.reportWaitersLock.ReleaseMutex();
if (foundWaiter)
{
// We delete the report from the queue right away,
// as nobody else should ever be interested in it.
this.clientReportQueue.DeleteMessage(message);
anyForUs = true;
switch (report.Status)
{
case CloudExecutionReport.StatusCode.Completed:
waiter.GiveReportToWaiter(report, remainingWaitersCount);
break;
case CloudExecutionReport.StatusCode.InProgress:
Console.WriteLine("Node '{0}' reports request '{1}' is in progress.", report.ProcessingNode, report.Identifier);
break;
default:
// REVIEW: Or just Assert here? This should never happen.
Console.WriteLine("Node '{0}' reports strange status '{1}' for request '{2}'.", report.ProcessingNode, report.Status, report.Identifier);
break;
}
}
}
if ((messages.Count != 0) && !anyForUs)
{
// Give other client(s) a chance to get their messages.
Thread.Sleep(invisibleTime + pollInterval + pollInterval);
}
else
{
Thread.Sleep(pollInterval);
}
}
}
/// <summary>
/// Give an execution report to this waiter.
/// </summary>
/// <param name="executionReport">The execution report to give.</param>
/// <param name="numberOfOtherWaiters">Count of other report waiters.</param>
public void GiveReportToWaiter(CloudExecutionReport executionReport, int numberOfOtherWaiters)
{
this.executionReport = executionReport;
this.otherWaitersCount = numberOfOtherWaiters;
this.reportIsReady.Set();
}