private static async Task <string> GetFileAsync(CloudTask boundTask, string fileName, bool dumpFile = true)
{
//Dump the standard out file of the task.
NodeFile file = await boundTask.GetNodeFileAsync(Batch.Constants.StandardOutFileName);
string fileContent = await file.ReadAsStringAsync();
if (dumpFile)
{
Console.WriteLine($"Task {boundTask.Id} {fileName}:");
Console.WriteLine("----------------------------------------");
Console.WriteLine(fileContent);
}
return(fileContent);
}
/// <summary>
/// Checks for a task's success or failure, and optionally dumps the output of the task. In the case that the task hit a scheduler or execution error,
/// dumps that information as well.
/// </summary>
/// <param name="boundTask">The task.</param>
/// <param name="dumpStandardOutOnTaskSuccess">True to log the standard output file of the task even if it succeeded. False to not log anything if the task succeeded.</param>
/// <returns>The string containing the standard out of the file, or null if stdout could not be gathered.</returns>
public static async Task <string> CheckForTaskSuccessAsync(CloudTask boundTask, bool dumpStandardOutOnTaskSuccess)
{
if (boundTask.State == TaskState.Completed)
{
string result = null;
//Check to see if the task has execution information metadata.
if (boundTask.ExecutionInformation != null)
{
//Dump the task scheduling error if there was one.
if (boundTask.ExecutionInformation.SchedulingError != null)
{
TaskSchedulingError schedulingError = boundTask.ExecutionInformation.SchedulingError;
Console.WriteLine("Task {0} hit scheduling error.", boundTask.Id);
Console.WriteLine("SchedulingError Code: {0}", schedulingError.Code);
Console.WriteLine("SchedulingError Message: {0}", schedulingError.Message);
Console.WriteLine("SchedulingError Category: {0}", schedulingError.Category);
Console.WriteLine("SchedulingError Details:");
foreach (NameValuePair detail in schedulingError.Details)
{
Console.WriteLine("{0} : {1}", detail.Name, detail.Value);
}
throw new TextSearchException(String.Format("Task {0} failed with a scheduling error", boundTask.Id));
}
//Read the content of the output files if the task exited.
if (boundTask.ExecutionInformation.ExitCode.HasValue)
{
Console.WriteLine("Task {0} exit code: {1}", boundTask.Id, boundTask.ExecutionInformation.ExitCode);
if (dumpStandardOutOnTaskSuccess && boundTask.ExecutionInformation.ExitCode.Value == 0 || boundTask.ExecutionInformation.ExitCode.Value != 0)
{
//Dump the standard out file of the task.
NodeFile taskStandardOut = await boundTask.GetNodeFileAsync(Batch.Constants.StandardOutFileName);
Console.WriteLine("Task {0} StdOut:", boundTask.Id);
Console.WriteLine("----------------------------------------");
string stdOutString = await taskStandardOut.ReadAsStringAsync();
result = stdOutString;
Console.WriteLine(stdOutString);
}
//Check for nonzero exit code and dump standard error if there was a nonzero exit code.
if (boundTask.ExecutionInformation.ExitCode.Value != 0)
{
NodeFile taskErrorFile = await boundTask.GetNodeFileAsync(Batch.Constants.StandardErrorFileName);
Console.WriteLine("Task {0} StdErr:", boundTask.Id);
Console.WriteLine("----------------------------------------");
string stdErrString = await taskErrorFile.ReadAsStringAsync();
Console.WriteLine(stdErrString);
throw new TextSearchException(String.Format("Task {0} failed with a nonzero exit code", boundTask.Id));
}
}
}
return(result);
}
else
{
throw new TextSearchException(String.Format("Task {0} is not completed yet. Current state: {1}", boundTask.Id, boundTask.State));
}
}
/// <summary>
/// Checks for a task's success or failure, and optionally dumps the output of the task. In the case that the task hit a scheduler or execution error,
/// dumps that information as well.
/// </summary>
/// <param name="boundTask">The task.</param>
/// <param name="dumpStandardOutOnTaskSuccess">True to log the standard output file of the task even if it succeeded. False to not log anything if the task succeeded.</param>
/// <returns>The string containing the standard out of the file, or null if stdout could not be gathered.</returns>
public static async Task<string> CheckForTaskSuccessAsync(CloudTask boundTask, bool dumpStandardOutOnTaskSuccess)
{
if (boundTask.State == TaskState.Completed)
{
string result = null;
//Check to see if the task has execution information metadata.
if (boundTask.ExecutionInformation != null)
{
//Dump the task scheduling error if there was one.
if (boundTask.ExecutionInformation.SchedulingError != null)
{
TaskSchedulingError schedulingError = boundTask.ExecutionInformation.SchedulingError;
Console.WriteLine("Task {0} hit scheduling error.", boundTask.Id);
Console.WriteLine("SchedulingError Code: {0}", schedulingError.Code);
Console.WriteLine("SchedulingError Message: {0}", schedulingError.Message);
Console.WriteLine("SchedulingError Category: {0}", schedulingError.Category);
Console.WriteLine("SchedulingError Details:");
foreach (NameValuePair detail in schedulingError.Details)
{
Console.WriteLine("{0} : {1}", detail.Name, detail.Value);
}
throw new TextSearchException(String.Format("Task {0} failed with a scheduling error", boundTask.Id));
}
//Read the content of the output files if the task exited.
if (boundTask.ExecutionInformation.ExitCode.HasValue)
{
Console.WriteLine("Task {0} exit code: {1}", boundTask.Id, boundTask.ExecutionInformation.ExitCode);
if (dumpStandardOutOnTaskSuccess && boundTask.ExecutionInformation.ExitCode.Value == 0 || boundTask.ExecutionInformation.ExitCode.Value != 0)
{
//Dump the standard out file of the task.
NodeFile taskStandardOut = await boundTask.GetNodeFileAsync(Batch.Constants.StandardOutFileName);
Console.WriteLine("Task {0} StdOut:", boundTask.Id);
Console.WriteLine("----------------------------------------");
string stdOutString = await taskStandardOut.ReadAsStringAsync();
result = stdOutString;
Console.WriteLine(stdOutString);
}
//Check for nonzero exit code and dump standard error if there was a nonzero exit code.
if (boundTask.ExecutionInformation.ExitCode.Value != 0)
{
NodeFile taskErrorFile = await boundTask.GetNodeFileAsync(Batch.Constants.StandardErrorFileName);
Console.WriteLine("Task {0} StdErr:", boundTask.Id);
Console.WriteLine("----------------------------------------");
string stdErrString = await taskErrorFile.ReadAsStringAsync();
Console.WriteLine(stdErrString);
throw new TextSearchException(String.Format("Task {0} failed with a nonzero exit code", boundTask.Id));
}
}
}
return result;
}
else
{
throw new TextSearchException(String.Format("Task {0} is not completed yet. Current state: {1}", boundTask.Id, boundTask.State));
}
}
public async static Task <IActionResult> Run([HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] RunICPRequest req, TraceWriter log)
{
try
{
var startTime = DateTimeOffset.Now;
log.Info($"Dunction started: {startTime}");
var timer = new Stopwatch();
timer.Start();
if (String.IsNullOrEmpty(BatchAccountName) ||
String.IsNullOrEmpty(BatchAccountKey) ||
String.IsNullOrEmpty(BatchAccountUrl) ||
String.IsNullOrEmpty(StorageAccountName) ||
String.IsNullOrEmpty(StorageAccountKey))
{
throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified.");
}
// Create batch job and task names
var nameUnifier = startTime.ToUniversalTime().ToString("yyyyMMdd_HHmmss_ffff");
var jobName = JobNamePrefix + (CreateNewJobOnEveryRun ? $"__{nameUnifier}" : String.Empty);
var taskName = TaskNamePrefix + $"__{nameUnifier}";
// Create Blob client
var storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
// Create input container
var container = blobClient.GetContainerReference(InputContainerName);
await container.CreateIfNotExistsAsync();
// Upload the data files to Azure Storage. This is the data that will be processed by each of the tasks that are
// executed on the compute nodes within the pool.
var inputFiles = new List <ResourceFile>();
var blobFileNamePrefix = $"{jobName}/{taskName}/";
inputFiles.Add(await UploadStringToContainer(blobClient, InputContainerName, blobFileNamePrefix + ModelPointsFileName, ModelPointsFileName, req.ModelPoints));
inputFiles.Add(await UploadStringToContainer(blobClient, InputContainerName, blobFileNamePrefix + ScanPointsFileName, ScanPointsFileName, req.ScanPoints));
// Get a Batch client using account creds
var batchCredentials = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (var batchClient = BatchClient.Open(batchCredentials))
{
// Create a Batch pool, VM configuration, Windows Server image
log.Info("Creating pool [{PoolName}]...");
var imageReference = new ImageReference(
publisher: NodeOsPublisher,
offer: NodeOsOffer,
sku: NodeOsSku,
version: NodeOsVersion);
var virtualMachineConfiguration = new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: NodeAgentSkuId);
try
{
var pool = batchClient.PoolOperations.CreatePool(
poolId: PoolName,
virtualMachineSize: PoolVmSize,
virtualMachineConfiguration: virtualMachineConfiguration);
var startTaskCmd = "vc_redist.x64.exe /quiet /install";
var redistFile = new ResourceFile(VCRedistributableUri, "vc_redist.x64.exe");
pool.StartTask = new StartTask(startTaskCmd)
{
ResourceFiles = new List <ResourceFile> {
redistFile
},
UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin))
};
if (!String.IsNullOrEmpty(PoolAutoscaleFormula))
{
pool.AutoScaleEnabled = true;
pool.AutoScaleFormula = PoolAutoscaleFormula;
pool.AutoScaleEvaluationInterval = TimeSpan.FromMinutes(5);
}
else
{
pool.TargetDedicatedComputeNodes = PoolDedicatedNodeCount;
pool.TargetLowPriorityComputeNodes = PoolLowPriorityNodeCount;
}
pool.MaxTasksPerComputeNode = PoolMaxTasksPerNodeCount;
pool.Commit();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
log.Info($"The pool {PoolName} already existed when we tried to create it");
}
else
{
throw; // Any other exception is unexpected
}
}
// Create a Batch job
log.Info("Creating job [" + jobName + "]...");
try
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = jobName;
job.PoolInformation = new PoolInformation {
PoolId = PoolName
};
job.Commit();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
log.Info($"The job {JobNamePrefix} already existed when we tried to create it");
}
else
{
throw; // Any other exception is unexpected
}
}
// Adding task
var taskCommandLine = String.Format($"cmd /c %AZ_BATCH_APP_PACKAGE_ICP%\\ICP_PCL.exe %AZ_BATCH_TASK_WORKING_DIR% {ScanPointsFileName} {ModelPointsFileName} 1 1");
var task = new CloudTask(taskName, taskCommandLine)
{
ResourceFiles = inputFiles,
ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference {
ApplicationId = ApplicationPackageId
}
}
};
await batchClient.JobOperations.AddTaskAsync(jobName, task);
// Monitor task
var stateMonitor = batchClient.Utilities.CreateTaskStateMonitor();
task = batchClient.JobOperations.GetTask(jobName, taskName);
stateMonitor.WaitAll(new List <CloudTask> {
task
}, TaskState.Completed, timeout: TimeSpan.FromMinutes(5));
log.Info("Task completed");
try
{
var result = new RunICPResponse
{
Content = (await task.GetNodeFileAsync("wd\\" + ApplicationOutputFileName)).ReadAsString()
};
log.Info("Execution finished successfully");
return(new OkObjectResult(result));
}
catch (Exception e)
{
var stdout = (await task.GetNodeFileAsync("stdout.txt")).ReadAsString();
var stderr = (await task.GetNodeFileAsync("stderr.txt")).ReadAsString();
var stdall = $"stdout:\n{stdout}\n\nstderr:{stderr}";
log.Error($"File not found, app logs:\n\n\n\n{stdall}");
return(new NotFoundObjectResult(e));
}
}
}
catch (Exception e)
{
log.Error("Unknown exception:", e);
throw;
}
}