internal TaskInformation(string taskUrl, string jobId, string taskId, int?subtaskId, TaskState taskState, TaskExecutionInformation executionInfo)
{
TaskUrl = taskUrl;
JobId = jobId;
TaskId = taskId;
SubtaskId = subtaskId;
TaskState = taskState;
ExecutionInfo = executionInfo;
}
/// <summary>
/// Gets the combined state of Azure Batch job, task and pool that corresponds to the given TES task
/// </summary>
/// <param name="tesTaskId">The unique TES task ID</param>
/// <returns>Job state information</returns>
public async Task <AzureBatchJobAndTaskState> GetBatchJobAndTaskStateAsync(string tesTaskId)
{
try
{
var nodeAllocationFailed = false;
string nodeErrorCode = null;
IEnumerable <string> nodeErrorDetails = null;
var activeJobWithMissingAutoPool = false;
ComputeNodeState? nodeState = null;
TaskState? taskState = null;
TaskExecutionInformation taskExecutionInformation = null;
var jobFilter = new ODATADetailLevel
{
FilterClause = $"startswith(id,'{tesTaskId}{BatchJobAttemptSeparator}')",
SelectClause = "*"
};
var jobInfos = (await batchClient.JobOperations.ListJobs(jobFilter).ToListAsync())
.Select(j => new { Job = j, AttemptNumber = int.Parse(j.Id.Split(BatchJobAttemptSeparator)[1]) });
if (!jobInfos.Any())
{
return(new AzureBatchJobAndTaskState {
JobState = null
});
}
if (jobInfos.Count(j => j.Job.State == JobState.Active) > 1)
{
return(new AzureBatchJobAndTaskState {
MoreThanOneActiveJobFound = true
});
}
var lastJobInfo = jobInfos.OrderBy(j => j.AttemptNumber).Last();
var job = lastJobInfo.Job;
var attemptNumber = lastJobInfo.AttemptNumber;
if (job.State == JobState.Active && job.ExecutionInformation?.PoolId != null)
{
var poolFilter = new ODATADetailLevel
{
FilterClause = $"id eq '{job.ExecutionInformation.PoolId}'",
SelectClause = "*"
};
var pool = (await batchClient.PoolOperations.ListPools(poolFilter).ToListAsync()).FirstOrDefault();
if (pool != null)
{
nodeAllocationFailed = pool.ResizeErrors?.Count > 0;
var node = (await pool.ListComputeNodes().ToListAsync()).FirstOrDefault();
if (node != null)
{
nodeState = node.State;
var nodeError = node.Errors?.FirstOrDefault();
nodeErrorCode = nodeError?.Code;
nodeErrorDetails = nodeError?.ErrorDetails?.Select(e => e.Value);
}
}
else
{
if (job.CreationTime.HasValue && DateTime.UtcNow.Subtract(job.CreationTime.Value) > TimeSpan.FromMinutes(30))
{
activeJobWithMissingAutoPool = true;
}
}
}
try
{
var batchTask = await batchClient.JobOperations.GetTaskAsync(job.Id, tesTaskId);
taskState = batchTask.State;
taskExecutionInformation = batchTask.ExecutionInformation;
}
catch (Exception ex)
{
logger.LogError(ex, $"Failed to get task for TesTask {tesTaskId}");
}
return(new AzureBatchJobAndTaskState
{
MoreThanOneActiveJobFound = false,
ActiveJobWithMissingAutoPool = activeJobWithMissingAutoPool,
AttemptNumber = attemptNumber,
NodeAllocationFailed = nodeAllocationFailed,
NodeErrorCode = nodeErrorCode,
NodeErrorDetails = nodeErrorDetails,
NodeState = nodeState,
JobState = job.State,
JobStartTime = job.ExecutionInformation?.StartTime,
JobEndTime = job.ExecutionInformation?.EndTime,
JobSchedulingError = job.ExecutionInformation?.SchedulingError,
TaskState = taskState,
TaskExecutionResult = taskExecutionInformation?.Result,
TaskStartTime = taskExecutionInformation?.StartTime,
TaskEndTime = taskExecutionInformation?.EndTime,
TaskExitCode = taskExecutionInformation?.ExitCode,
TaskFailureInformation = taskExecutionInformation?.FailureInformation,
TaskContainerState = taskExecutionInformation?.ContainerInformation?.State,
TaskContainerError = taskExecutionInformation?.ContainerInformation?.Error
});
}
catch (Exception ex)
{
logger.LogError(ex, $"GetBatchJobAndTaskStateAsync failed for TesTask {tesTaskId}");
throw;
}
}
