public static string GetFailureInfoDetails(TaskFailureInformation failureInfo)
{
StringBuilder builder = new StringBuilder();
builder.AppendLine($"Category: {failureInfo.Category}");
builder.AppendLine($"Code: {failureInfo.Code}");
builder.AppendLine($"Message: {failureInfo.Message}");
builder.AppendLine("Details:");
foreach (var detail in failureInfo.Details)
{
builder.AppendLine($" {detail.Name}: {detail.Value}");
}
return(builder.ToString());
}
private void OutputFailureInfo(TaskFailureInformation failureInfo)
{
this.testOutputHelper.WriteLine("JP Failure Info:");
this.testOutputHelper.WriteLine(" category: " + failureInfo.Category);
this.testOutputHelper.WriteLine(" code: " + failureInfo.Code);
this.testOutputHelper.WriteLine(" details:" + (null == failureInfo.Details ? " <null>" : string.Empty));
if (null != failureInfo.Details)
{
foreach (NameValuePair curDetail in failureInfo.Details)
{
this.testOutputHelper.WriteLine(" name: " + curDetail.Name + ", value: " + curDetail.Value);
}
}
this.testOutputHelper.WriteLine(" message: " + failureInfo.Message);
}
/// <summary>
/// Checks for a task's success or failure, and optionally dumps the output of the task. In the case that the task hit a scheduler or execution error,
/// dumps that information as well.
/// </summary>
/// <param name="boundTask">The task.</param>
/// <param name="dumpStandardOutOnTaskSuccess">True to log the standard output file of the task even if it succeeded. False to not log anything if the task succeeded.</param>
/// <returns>The string containing the standard out of the file, or null if stdout could not be gathered.</returns>
public static async Task <string> CheckForTaskSuccessAsync(CloudTask boundTask, bool dumpStandardOutOnTaskSuccess)
{
if (boundTask.State == TaskState.Completed)
{
//Dump the task failure info if there was one.
if (boundTask.ExecutionInformation.FailureInformation != null)
{
TaskFailureInformation failureInformation = boundTask.ExecutionInformation.FailureInformation;
Console.WriteLine($"Task {boundTask.Id} had a failure.");
Console.WriteLine($"Failure Code: {failureInformation.Code}");
Console.WriteLine($"Failure Message: {failureInformation.Message}");
Console.WriteLine($"Failure Category: {failureInformation.Category}");
Console.WriteLine("Failure Details:");
foreach (NameValuePair detail in failureInformation.Details)
{
Console.WriteLine("{0} : {1}", detail.Name, detail.Value);
}
if (boundTask.ExecutionInformation.ExitCode.HasValue)
{
Console.WriteLine($"Task {boundTask.Id} exit code: {boundTask.ExecutionInformation.ExitCode}");
if (boundTask.ExecutionInformation.ExitCode.Value != 0)
{
await GetFileAsync(boundTask, Batch.Constants.StandardOutFileName);
await GetFileAsync(boundTask, Batch.Constants.StandardErrorFileName);
}
}
throw new TextSearchException($"Task {boundTask.Id} failed");
}
else
{
string result = await GetFileAsync(boundTask, Batch.Constants.StandardOutFileName, dumpStandardOutOnTaskSuccess);
return(result);
}
}
else
{
throw new TextSearchException($"Task {boundTask.Id} is not completed yet. Current state: {boundTask.State}");
}
}
/// <summary>
/// Gets the combined state of Azure Batch job, task and pool that corresponds to the given TES task
/// </summary>
/// <param name="tesTaskId">The unique TES task ID</param>
/// <returns>Job state information</returns>
public async Task <AzureBatchJobAndTaskState> GetBatchJobAndTaskStateAsync(string tesTaskId)
{
var nodeAllocationFailed = false;
TaskState?taskState = null;
int? taskExitCode = null;
TaskFailureInformation taskFailureInformation = null;
var jobFilter = new ODATADetailLevel
{
FilterClause = $"startswith(id,'{tesTaskId}{BatchJobAttemptSeparator}')",
SelectClause = "*"
};
var jobInfos = (await batchClient.JobOperations.ListJobs(jobFilter).ToListAsync())
.Select(j => new { Job = j, AttemptNumber = int.Parse(j.Id.Split(BatchJobAttemptSeparator)[1]) });
if (!jobInfos.Any())
{
return(new AzureBatchJobAndTaskState {
JobState = null
});
}
if (jobInfos.Count(j => j.Job.State == JobState.Active) > 1)
{
return(new AzureBatchJobAndTaskState {
MoreThanOneActiveJobFound = true
});
}
var lastJobInfo = jobInfos.OrderBy(j => j.AttemptNumber).Last();
var job = lastJobInfo.Job;
var attemptNumber = lastJobInfo.AttemptNumber;
if (job.State == JobState.Active)
{
try
{
nodeAllocationFailed = job.ExecutionInformation?.PoolId != null &&
(await batchClient.PoolOperations.GetPoolAsync(job.ExecutionInformation.PoolId)).ResizeErrors?.Count > 0;
}
catch (Exception ex)
{
// assume that node allocation failed
nodeAllocationFailed = true;
logger.LogError(ex, $"Failed to determine if the node allocation failed for TesTask {tesTaskId} with PoolId {job.ExecutionInformation?.PoolId}.");
}
}
var jobPreparationTaskExecutionInformation = (await batchClient.JobOperations.ListJobPreparationAndReleaseTaskStatus(job.Id).ToListAsync()).FirstOrDefault()?.JobPreparationTaskExecutionInformation;
var jobPreparationTaskExitCode = jobPreparationTaskExecutionInformation?.ExitCode;
var jobPreparationTaskState = jobPreparationTaskExecutionInformation?.State;
try
{
var batchTask = await batchClient.JobOperations.GetTaskAsync(job.Id, tesTaskId);
taskState = batchTask.State;
taskExitCode = batchTask.ExecutionInformation?.ExitCode;
taskFailureInformation = batchTask.ExecutionInformation.FailureInformation;
}
catch (Exception ex)
{
logger.LogError(ex, $"Failed to get task for TesTask {tesTaskId}");
}
return(new AzureBatchJobAndTaskState
{
JobState = job.State,
JobPreparationTaskState = jobPreparationTaskState,
JobPreparationTaskExitCode = jobPreparationTaskExitCode,
TaskState = taskState,
MoreThanOneActiveJobFound = false,
NodeAllocationFailed = nodeAllocationFailed,
TaskExitCode = taskExitCode,
TaskFailureInformation = taskFailureInformation,
AttemptNumber = attemptNumber
});
}
