/// <summary>
/// Gets the next request from the request queue.
/// </summary>
/// <param name="block">Whether to block.</param>
/// <returns>Next request on the queue.</returns>
public CloudExecutionRequest GetRequest(bool block = false)
{
int pollInterval = 1000; // In milliseconds.
CloudQueueMessage message = null;
while ((message = this.requestQueue.GetMessage()) == null)
{
if (block)
{
Thread.Sleep(pollInterval);
// Increase poll interval if we're below the cap.
if (pollInterval < RequestQueuePollIntervalCap)
{
// Increase by 1/10th of a second each time.
// This will take 14.5s to reach a 2s interval.
pollInterval += 100;
}
continue;
}
else
{
return(null);
}
}
////Console.WriteLine(message.AsString);
CloudExecutionRequest request = CloudExecutionRequest.FromMessage(message);
return(request);
}
/// <summary>
/// Creates a CloudExecutionRequest from a CloudQueueMessage
/// representation.
/// </summary>
/// <param name="message">
/// A CloudQueueMessage containing an XML document representing a
/// CloudExecutionRequest.
/// </param>
/// <returns>
/// A new request corresponding to the XML representation contained in
/// the message.
/// </returns>
public static CloudExecutionRequest FromMessage(CloudQueueMessage message)
{
CloudExecutionRequest request = CloudExecutionRequest.FromXml(message.AsString);
request.message = message;
return(request);
}
/// <summary>
/// Initializes a new instance of the CloudExecutionQueue class.
/// </summary>
/// <remarks>
/// This is the constructor for clients (does extra stuff).
/// </remarks>
/// <param name="reportQueueName">Name of the report queue this client is using.</param>
public CloudExecutionQueue(string reportQueueName)
: this()
{
if (string.IsNullOrEmpty(reportQueueName))
{
// Use default shared report queue if none is specified.
reportQueueName = CloudExecutionQueue.ReportQueueName;
}
else
{
// Schedule auto-deletion of our private report queue.
// We do this by setting the initial invisibility of a delete queue request.
// This will auto-clean up our private report queue if we exit unexpectedly.
CloudExecutionRequest deleteQueueRequest = new CloudExecutionRequest(
reportQueueName,
reportQueueName,
CloudExecutionRequest.Operation.DeleteQueue,
null,
null,
null,
null
);
CloudQueueMessage deleteQueueMessage = new CloudQueueMessage(deleteQueueRequest.ToXml());
this.requestQueue.AddMessage(deleteQueueMessage, new TimeSpan(12, 0, 0), new TimeSpan(4, 0, 0));
}
this.clientReportQueue = this.queueClient.GetQueueReference(reportQueueName);
this.clientReportQueue.CreateIfNotExists();
this.reportWaiters = new Dictionary <string, CloudExecutionWaiter>();
this.reportWaitersLock = new Mutex();
this.haveReportWaiters = new ManualResetEvent(false);
this.monitoringThread = new Thread(new ThreadStart(this.MonitorReportQueue));
this.monitoringThread.IsBackground = true;
this.monitoringThread.Name = "Report Queue Monitor";
this.monitoringThread.Start();
}
/// <summary>
/// Run the requested executable and produce a report of the results.
/// </summary>
/// <param name="executionRequest">The execution request.</param>
/// <returns>A report of the results.</returns>
private CloudExecutionReport RunAnExecutable(CloudExecutionRequest executionRequest)
{
// REVIEW: How/whether to use this.
BuildObject diagnosticsBase = new BuildObject(Path.Combine("nuobj", "diagnostics", "process"));
// Prep working directory with input files and output dirs.
// TODO: The below will throw cache exceptions if something
// isn't there (they should all be though). Need to catch
// these and fail the execution request when this happens.
WorkingDirectory workingDirectory = new WorkingDirectory(this.virtualIronRoot);
foreach (BuildObjectValuePointer inputFile in executionRequest.InputFileMappings)
{
// REVIEW: How to determine cache container here.
ItemCacheContainer container = ItemCacheContainer.Sources;
if (this.multiplexedItemCache.GetItemSize(container, inputFile.ObjectHash) == -1)
{
container = ItemCacheContainer.Objects;
}
// TODO: Move path/directory manipulation code into
// WorkingDirectory and/or ItemCache.
string inputFilePath = workingDirectory.PathTo(inputFile.RelativePath);
Directory.CreateDirectory(Path.GetDirectoryName(inputFilePath)); // REVIEW: Still neeeded?
this.multiplexedItemCache.FetchItemToFile(
container,
inputFile.ObjectHash,
inputFilePath);
}
foreach (BuildObject outputFile in executionRequest.OutputFiles)
{
workingDirectory.CreateDirectoryFor(outputFile);
}
// Run executable.
ProcessInvoker pinv = new ProcessInvoker(
workingDirectory,
executionRequest.Executable,
new string[] { executionRequest.Arguments },
diagnosticsBase,
null, // This is captureStdout. TODO: Should cleanup how this is used in ProcessInvoker.
null); // This is dbgText. REVIEW: How/whether to use this.
// When ProcessInvoker's constructor returns, the process has
// finished running.
Console.WriteLine("Request {0} completed in {1} seconds.", executionRequest.Identifier, pinv.CpuTime);
// Store output files in the (cloud) item cache, and create a
// list of the mappings.
List<BuildObjectValuePointer> outputFileMappings = new List<BuildObjectValuePointer>();
foreach (BuildObject outFile in executionRequest.OutputFiles)
{
if (File.Exists(workingDirectory.PathTo(outFile)))
{
string fileHash = Util.hashFilesystemPath(workingDirectory.PathTo(outFile));
Util.Assert(!string.IsNullOrEmpty(fileHash));
// Note we explicitly write to the cloud cache here.
this.cloudCache.StoreItemFromFile(ItemCacheContainer.Objects, fileHash, workingDirectory.PathTo(outFile));
outputFileMappings.Add(new BuildObjectValuePointer(fileHash, outFile.getRelativePath()));
}
}
// Collect the results into a report.
CloudExecutionReport report = new CloudExecutionReport(
executionRequest.Identifier,
CloudExecutionReport.StatusCode.Completed,
pinv.ExitCode,
pinv.GetStdout(),
pinv.GetStderr(),
pinv.CpuTime,
outputFileMappings);
return report;
}
/// <summary>
/// Deletes a request from the queue.
/// Used by reader to indicate request completion.
/// </summary>
/// <param name="request">Request to delete.</param>
public void DeleteRequest(CloudExecutionRequest request)
{
CloudQueueMessage message = request.Message;
this.requestQueue.DeleteMessage(message);
}
/// <summary>
/// Submits a request for cloud execution.
/// </summary>
/// <param name="request">
/// The CloudExecutionRequest detailing this request.
/// </param>
public void SubmitRequest(CloudExecutionRequest request)
{
CloudQueueMessage message = new CloudQueueMessage(request.ToXml());
this.requestQueue.AddMessage(message, this.queueEntryTtl);
}
/// <summary>
/// Initializes a new instance of the CloudExecutionQueue class.
/// </summary>
/// <remarks>
/// This is the constructor for clients (does extra stuff).
/// </remarks>
/// <param name="reportQueueName">Name of the report queue this client is using.</param>
public CloudExecutionQueue(string reportQueueName)
: this()
{
if (string.IsNullOrEmpty(reportQueueName))
{
// Use default shared report queue if none is specified.
reportQueueName = CloudExecutionQueue.ReportQueueName;
}
else
{
// Schedule auto-deletion of our private report queue.
// We do this by setting the initial invisibility of a delete queue request.
// This will auto-clean up our private report queue if we exit unexpectedly.
CloudExecutionRequest deleteQueueRequest = new CloudExecutionRequest(
reportQueueName,
reportQueueName,
CloudExecutionRequest.Operation.DeleteQueue,
null,
null,
null,
null
);
CloudQueueMessage deleteQueueMessage = new CloudQueueMessage(deleteQueueRequest.ToXml());
this.requestQueue.AddMessage(deleteQueueMessage, new TimeSpan(12, 0, 0), new TimeSpan(4, 0, 0));
}
this.clientReportQueue = this.queueClient.GetQueueReference(reportQueueName);
this.clientReportQueue.CreateIfNotExists();
this.reportWaiters = new Dictionary<string, CloudExecutionWaiter>();
this.reportWaitersLock = new Mutex();
this.haveReportWaiters = new ManualResetEvent(false);
this.monitoringThread = new Thread(new ThreadStart(this.MonitorReportQueue));
this.monitoringThread.IsBackground = true;
this.monitoringThread.Name = "Report Queue Monitor";
this.monitoringThread.Start();
}
/// <summary>
/// Initializes a new instance of the CloudSubmitter class.
/// </summary>
/// <param name="requestIdentifier">
/// Unique identifier for this request.
/// </param>
/// <param name="workingDirectory">
/// Working directory the process is started in.
/// </param>
/// <param name="inputFiles">
/// List of input files expected by the process.
/// </param>
/// <param name="outputFiles">
/// List of potential output files generated by the process.
/// </param>
/// <param name="executable">Executable to run.</param>
/// <param name="args">
/// Command line arguments to provide to the executable.
/// </param>
/// <param name="failureBase">Not sure what this is -- some debugging/diagnostic thing.</param>
/// <param name="captureStdout">Where to (optionally) store the captured standard out.</param>
/// <param name="dbgText">Debugging text for something or another.</param>
public CloudSubmitter(
string requestIdentifier,
WorkingDirectory workingDirectory,
IEnumerable <BuildObject> inputFiles,
IEnumerable <BuildObject> outputFiles,
string executable,
string[] args,
BuildObject failureBase,
BuildObject captureStdout = null,
string dbgText = null)
{
// Catch bad verb authors before they hurt themselves.
Util.Assert(!executable.Contains(":")); // Hey, this looks like an absolute path! Use .getRelativePath(); it makes your output more stable.
foreach (string arg in args)
{
// Pardon my distasteful heuristic to avoid flagging /flag:value args.
Util.Assert(arg.Length < 2 || arg[1] != ':'); // Hey, this looks like an absolute path! Use .getRelativePath() to tolerate crossing machine boundaries.
}
// Stash away things we'll want to remember later.
this.workingDirectory = workingDirectory;
// Create list of input file mappings.
// REVIEW: We're not running on the main thread at this point. Are below calls all thread-safe?
List <BuildObjectValuePointer> inputFileMappings = new List <BuildObjectValuePointer>();
foreach (BuildObject file in inputFiles)
{
string fileHash = BuildEngine.theEngine.Repository.GetHash(file);
Util.Assert(!string.IsNullOrEmpty(fileHash));
inputFileMappings.Add(new BuildObjectValuePointer(fileHash, file.getRelativePath()));
// Ensure that the input files are in the cloud cache.
// REVIEW: best way to determine this is a source file?
ItemCacheContainer container;
if (file is SourcePath)
{
container = ItemCacheContainer.Sources;
}
else
{
container = ItemCacheContainer.Objects;
}
ItemCacheMultiplexer multiplexedCache = BuildEngine.theEngine.ItemCache as ItemCacheMultiplexer;
Util.Assert(multiplexedCache != null);
multiplexedCache.SyncItemToCloud(container, fileHash);
}
// Prepare cloud execution request for submission.
string arguments = string.Join(" ", args);
CloudExecutionRequest request = new CloudExecutionRequest(
BuildEngine.theEngine.CloudReportQueueName,
requestIdentifier,
CloudExecutionRequest.Operation.RunExecutable,
executable,
arguments,
inputFileMappings,
outputFiles);
BuildEngine.theEngine.CloudExecutionQueue.SubmitRequest(request);
// Wait for remote execution to finish.
int requestsOutstanding;
Console.WriteLine("Waiting on remote execution report for request '{0}'.", requestIdentifier);
CloudExecutionReport executionReport = BuildEngine.theEngine.CloudExecutionQueue.GetReport(requestIdentifier, out requestsOutstanding);
Console.WriteLine("Received remote execution report for request '{0}'. {1} others still outstanding.", requestIdentifier, requestsOutstanding);
// Record what we got back.
this.exitCode = executionReport.ExitCode;
this.stdout = executionReport.StandardOutput;
this.stderr = executionReport.StandardError;
this.cpuTime = executionReport.CpuTime;
// Copy output files from cloud cache back to local working dir.
// REVIEW: This is just to set things up as expected for the
// Scheduler's recordResult routine. Could re-architect this to
// be more efficient for the remote execution case.
foreach (BuildObjectValuePointer outputFileMapping in executionReport.OutputFileMappings)
{
BuildEngine.theEngine.CloudCache.FetchItemToFile(
ItemCacheContainer.Objects,
outputFileMapping.ObjectHash,
workingDirectory.PathTo(outputFileMapping.RelativePath));
}
}
/// <summary>
/// Initializes a new instance of the CloudSubmitter class.
/// </summary>
/// <param name="requestIdentifier">
/// Unique identifier for this request.
/// </param>
/// <param name="workingDirectory">
/// Working directory the process is started in.
/// </param>
/// <param name="inputFiles">
/// List of input files expected by the process.
/// </param>
/// <param name="outputFiles">
/// List of potential output files generated by the process.
/// </param>
/// <param name="executable">Executable to run.</param>
/// <param name="args">
/// Command line arguments to provide to the executable.
/// </param>
/// <param name="failureBase">Not sure what this is -- some debugging/diagnostic thing.</param>
/// <param name="captureStdout">Where to (optionally) store the captured standard out.</param>
/// <param name="dbgText">Debugging text for something or another.</param>
public CloudSubmitter(
string requestIdentifier,
WorkingDirectory workingDirectory,
IEnumerable<BuildObject> inputFiles,
IEnumerable<BuildObject> outputFiles,
string executable,
string[] args,
BuildObject failureBase,
BuildObject captureStdout = null,
string dbgText = null)
{
// Catch bad verb authors before they hurt themselves.
Util.Assert(!executable.Contains(":")); // Hey, this looks like an absolute path! Use .getRelativePath(); it makes your output more stable.
foreach (string arg in args)
{
// Pardon my distasteful heuristic to avoid flagging /flag:value args.
Util.Assert(arg.Length < 2 || arg[1] != ':'); // Hey, this looks like an absolute path! Use .getRelativePath() to tolerate crossing machine boundaries.
}
// Stash away things we'll want to remember later.
this.workingDirectory = workingDirectory;
// Create list of input file mappings.
// REVIEW: We're not running on the main thread at this point. Are below calls all thread-safe?
List<BuildObjectValuePointer> inputFileMappings = new List<BuildObjectValuePointer>();
foreach (BuildObject file in inputFiles)
{
string fileHash = BuildEngine.theEngine.Repository.GetHash(file);
Util.Assert(!string.IsNullOrEmpty(fileHash));
inputFileMappings.Add(new BuildObjectValuePointer(fileHash, file.getRelativePath()));
// Ensure that the input files are in the cloud cache.
// REVIEW: best way to determine this is a source file?
ItemCacheContainer container;
if (file is SourcePath)
{
container = ItemCacheContainer.Sources;
}
else
{
container = ItemCacheContainer.Objects;
}
ItemCacheMultiplexer multiplexedCache = BuildEngine.theEngine.ItemCache as ItemCacheMultiplexer;
Util.Assert(multiplexedCache != null);
multiplexedCache.SyncItemToCloud(container, fileHash);
}
// Prepare cloud execution request for submission.
string arguments = string.Join(" ", args);
CloudExecutionRequest request = new CloudExecutionRequest(
BuildEngine.theEngine.CloudReportQueueName,
requestIdentifier,
CloudExecutionRequest.Operation.RunExecutable,
executable,
arguments,
inputFileMappings,
outputFiles);
BuildEngine.theEngine.CloudExecutionQueue.SubmitRequest(request);
// Wait for remote execution to finish.
int requestsOutstanding;
Console.WriteLine("Waiting on remote execution report for request '{0}'.", requestIdentifier);
CloudExecutionReport executionReport = BuildEngine.theEngine.CloudExecutionQueue.GetReport(requestIdentifier, out requestsOutstanding);
Console.WriteLine("Received remote execution report for request '{0}'. {1} others still outstanding.", requestIdentifier, requestsOutstanding);
// Record what we got back.
this.exitCode = executionReport.ExitCode;
this.stdout = executionReport.StandardOutput;
this.stderr = executionReport.StandardError;
this.cpuTime = executionReport.CpuTime;
// Copy output files from cloud cache back to local working dir.
// REVIEW: This is just to set things up as expected for the
// Scheduler's recordResult routine. Could re-architect this to
// be more efficient for the remote execution case.
foreach (BuildObjectValuePointer outputFileMapping in executionReport.OutputFileMappings)
{
BuildEngine.theEngine.CloudCache.FetchItemToFile(
ItemCacheContainer.Objects,
outputFileMapping.ObjectHash,
workingDirectory.PathTo(outputFileMapping.RelativePath));
}
}
/// <summary>
/// Deletes a request from the queue.
/// Used by reader to indicate request completion.
/// </summary>
/// <param name="request">Request to delete.</param>
public void DeleteRequest(CloudExecutionRequest request)
{
CloudQueueMessage message = request.Message;
this.requestQueue.DeleteMessage(message);
}
/// <summary>
/// Submits a request for cloud execution.
/// </summary>
/// <param name="request">
/// The CloudExecutionRequest detailing this request.
/// </param>
public void SubmitRequest(CloudExecutionRequest request)
{
CloudQueueMessage message = new CloudQueueMessage(request.ToXml());
this.requestQueue.AddMessage(message, this.queueEntryTtl);
}