private async Task CheckForWorkItemFailureAsync(string workItemName, string jobName)
{
await Task.Yield();
try
{
WorkItemDetails details = await HelixApi.RetryAsync(
() => HelixApi.WorkItem.DetailsAsync(workItemName, jobName),
LogExceptionRetry);
string message = $"Work item {workItemName} in job {jobName} has {details.State} with exit code {details.ExitCode}";
if (IsFailed(details))
{
Log.LogError(message);
}
else
{
Log.LogMessage(MessageImportance.Low, message);
}
}
catch (Exception ex)
{
Log.LogError($"Unable to get work item status for '{workItemName}', assuming failure. Exception: {ex}");
}
}
private async Task WaitForHelixJobAsync(string jobName)
{
await Task.Yield();
Log.LogMessage($"Waiting for completion of job {jobName}");
while (true)
{
var workItems = await HelixApi.RetryAsync(
() => HelixApi.WorkItem.ListAsync(jobName),
LogExceptionRetry);
var waitingCount = workItems.Count(wi => wi.State == "Waiting");
var runningCount = workItems.Count(wi => wi.State == "Running");
var finishedCount = workItems.Count(wi => wi.State == "Finished");
if (waitingCount == 0 && runningCount == 0 && finishedCount > 0)
{
// determines whether any of the work items failed (fireballed)
await Task.WhenAll(workItems.Select(wi => CheckForWorkItemFailureAsync(wi.Name, jobName)));
Log.LogMessage(MessageImportance.High, $"Job {jobName} is completed with {finishedCount} finished work items.");
return;
}
Log.LogMessage($"Job {jobName} is not yet completed with Waiting: {waitingCount}, Running: {runningCount}, Finished: {finishedCount}");
await Task.Delay(10000);
}
}
private async Task WaitForHelixJobAsync(string jobName, CancellationToken cancellationToken)
{
await Task.Yield();
cancellationToken.ThrowIfCancellationRequested();
Log.LogMessage(MessageImportance.High, $"Waiting for completion of job {jobName}");
for (;; await Task.Delay(10000, cancellationToken)) // delay every time this loop repeats
{
cancellationToken.ThrowIfCancellationRequested();
var pf = await HelixApi.RetryAsync(
() => HelixApi.Job.PassFailAsync(jobName, cancellationToken),
LogExceptionRetry,
cancellationToken);
if (pf.Working == 0 && pf.Total != 0)
{
Log.LogMessage(MessageImportance.High, $"Job {jobName} is completed with {pf.Total} finished work items.");
return;
}
Log.LogMessage($"Job {jobName} is not yet completed with Pending: {pf.Working}, Finished: {pf.Total - pf.Working}");
cancellationToken.ThrowIfCancellationRequested();
}
}
private async Task <IEnumerable <ITaskItem> > GetFailedWorkItemsAsync(ITaskItem job, CancellationToken cancellationToken)
{
var jobName = job.GetMetadata("Identity");
Log.LogMessage($"Getting status of job {jobName}");
var status = await HelixApi.RetryAsync(
() => HelixApi.Job.PassFailAsync(jobName, cancellationToken),
LogExceptionRetry,
cancellationToken);
if (status.Working > 0)
{
Log.LogError(
FailureCategory.Build,
$"This task can only be used on completed jobs. There are {status.Working} of {status.Total} unfinished work items.");
return(Array.Empty <ITaskItem>());
}
return(await Task.WhenAll(status.Failed.Select(async wi =>
{
// copy all job metadata into the new item
var metadata = job.CloneCustomMetadata();
metadata["JobName"] = jobName;
metadata["WorkItemName"] = wi;
var consoleUri = HelixApi.BaseUri.AbsoluteUri.TrimEnd('/') + $"/api/2019-06-17/jobs/{jobName}/workitems/{Uri.EscapeDataString(wi)}/console";
metadata["ConsoleOutputUri"] = consoleUri;
try
{
var files = await HelixApi.RetryAsync(
() => HelixApi.WorkItem.ListFilesAsync(wi, jobName, cancellationToken),
LogExceptionRetry,
cancellationToken);
if (!string.IsNullOrEmpty(AccessToken))
{
// Add AccessToken to all file links because the api requires auth if we submitted the job with auth
files = files
.Select(file => new UploadedFile(file.Name, file.Link + "?access_token=" + AccessToken))
.ToImmutableList();
}
metadata["UploadedFiles"] = JsonConvert.SerializeObject(files);
}
catch (Exception ex)
{
Log.LogWarningFromException(ex);
}
return new TaskItem($"{jobName}/{wi}", metadata);
})));
}
private async Task CheckHelixJobAsync(HttpClient client, string jobName, string testRunId, CancellationToken cancellationToken)
{
await Task.Yield();
cancellationToken.ThrowIfCancellationRequested();
Log.LogMessage($"Checking status of job {jobName}");
var status = await HelixApi.RetryAsync(
() => HelixApi.Job.PassFailAsync(jobName, cancellationToken),
LogExceptionRetry,
cancellationToken);
if (status.Working > 0)
{
Log.LogError(
$"This task can only be used on completed jobs. There are {status.Working} of {status.Total} unfinished work items.");
return;
}
if (FailOnWorkItemFailure)
{
foreach (string failedWorkItem in status.Failed)
{
var consoleUri = HelixApi.BaseUri.AbsoluteUri.TrimEnd('/') + $"/api/2019-06-17/jobs/{jobName}/workitems/{Uri.EscapeDataString(failedWorkItem)}/console";
Log.LogError($"Work item {failedWorkItem} in job {jobName} has failed, logs available here: {consoleUri}.");
var testResultId = await CreateFakeTestResultAsync(client, testRunId, failedWorkItem);
string fileStreamString = await GetUploadedFilesAsync(jobName, failedWorkItem, cancellationToken);
if (fileStreamString != null)
{
await AttachResultFileToTestResultAsync(client, testRunId, testResultId, fileStreamString);
}
}
}
if (FailOnMissionControlTestFailure)
{
for (; ; await Task.Delay(10000, cancellationToken)) // delay every time this loop repeats
{
if (await MissionControlTestProcessingDoneAsync(jobName, cancellationToken))
{
break;
}
Log.LogMessage($"Job {jobName} is still processing xunit results.");
cancellationToken.ThrowIfCancellationRequested();
}
}
}
private async Task CheckHelixJobAsync(string jobName)
{
await Task.Yield();
Log.LogMessage($"Checking status of job {jobName}");
var workItems = await HelixApi.RetryAsync(
() => HelixApi.WorkItem.ListAsync(jobName),
LogExceptionRetry);
var waitingCount = workItems.Count(wi => wi.State == "Waiting");
var runningCount = workItems.Count(wi => wi.State == "Running");
if (waitingCount != 0 || runningCount != 0)
{
Log.LogError(
$"This task can only be used on completed jobs. There are {waitingCount} waiting and {runningCount} running work items.");
return;
}
if (FailOnWorkItemFailure)
{
// determines whether any of the work items failed (fireballed)
// Doing these all in parallel overloads the Helix server
foreach (var wi in workItems)
{
await CheckForWorkItemFailureAsync(wi.Name, jobName);
}
}
if (FailOnMissionControlTestFailure)
{
for (; ; await Task.Delay(10000)) // delay every time this loop repeats
{
if (await MissionControlTestProcessingDoneAsync(jobName))
{
break;
}
Log.LogMessage($"Job {jobName} is still processing xunit results.");
}
}
}
private async Task CheckForWorkItemFailureAsync(string workItemName, string jobName)
{
await Task.Yield();
try
{
var details = await HelixApi.RetryAsync(
() => HelixApi.WorkItem.DetailsAsync(jobName, workItemName),
LogExceptionRetry);
if (details.State == "Failed")
{
Log.LogError(
$"Work item {workItemName} on job {jobName} has failed with exit code {details.ExitCode}.");
}
}
catch (Exception ex)
{
Log.LogError($"Unable to get work item status for '{workItemName}', assuming failure. Exception: {ex}");
}
}
private async Task <string> GetUploadedFilesAsync(string jobName, string workItemFriendlyName, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
Log.LogMessage($"Looking up files for work item {workItemFriendlyName} in job {jobName}");
try
{
var uploadedFiles = await HelixApi.RetryAsync(
() => HelixApi.WorkItem.ListFilesAsync(workItemFriendlyName, jobName, cancellationToken),
LogExceptionRetry,
cancellationToken);
if (uploadedFiles.Count > 0)
{
using (var ms = new MemoryStream())
{
TextWriter tw = new StreamWriter(ms);
tw.WriteLine("<ul>");
foreach (var uploadedFile in uploadedFiles)
{
tw.WriteLine($"<li><a href='{uploadedFile.Link}' target='_blank'>{uploadedFile.Name}</a></li>");
}
tw.WriteLine("</ul>");
tw.Flush();
ms.Position = 0;
return(Convert.ToBase64String(ms.ToArray()));
}
}
return(null);
}
catch (Exception ex)
{
Log.LogErrorFromException(ex);
return(null);
}
}
private async Task WaitForHelixJobAsync(string jobName)
{
await Task.Yield();
Log.LogMessage($"Waiting for completion of job {jobName}");
for (;; await Task.Delay(10000)) // delay every time this loop repeats
{
var workItems = await HelixApi.RetryAsync(
() => HelixApi.WorkItem.ListAsync(jobName),
LogExceptionRetry);
var waitingCount = workItems.Count(wi => wi.State == "Waiting");
var runningCount = workItems.Count(wi => wi.State == "Running");
var finishedCount = workItems.Count(wi => wi.State == "Finished");
if (waitingCount == 0 && runningCount == 0 && finishedCount > 0)
{
if (FailOnWorkItemFailure)
{
// determines whether any of the work items failed (fireballed)
await Task.WhenAll(workItems.Select(wi => CheckForWorkItemFailureAsync(wi.Name, jobName)));
}
if (FailOnMissionControlTestFailure)
{
if (!(await MissionControlTestProcessingDoneAsync(jobName)))
{
Log.LogMessage($"Job {jobName} is still processing xunit results.");
continue;
}
}
Log.LogMessage(MessageImportance.High, $"Job {jobName} is completed with {finishedCount} finished work items.");
return;
}
Log.LogMessage($"Job {jobName} is not yet completed with Waiting: {waitingCount}, Running: {runningCount}, Finished: {finishedCount}");
}
}
private async Task WaitForHelixJobAsync(string jobName)
{
await Task.Yield();
Log.LogMessage(MessageImportance.High, $"Waiting for completion of job {jobName}");
for (;; await Task.Delay(10000)) // delay every time this loop repeats
{
var workItems = await HelixApi.RetryAsync(
() => HelixApi.WorkItem.ListAsync(jobName),
LogExceptionRetry);
var waitingCount = workItems.Count(wi => wi.State == "Waiting");
var runningCount = workItems.Count(wi => wi.State == "Running");
var finishedCount = workItems.Count(wi => wi.State == "Finished");
if (waitingCount == 0 && runningCount == 0 && finishedCount > 0)
{
Log.LogMessage(MessageImportance.High, $"Job {jobName} is completed with {finishedCount} finished work items.");
return;
}
Log.LogMessage($"Job {jobName} is not yet completed with Waiting: {waitingCount}, Running: {runningCount}, Finished: {finishedCount}");
}
}
private async Task CheckHelixJobAsync(string jobName)
{
await Task.Yield();
Log.LogMessage($"Checking status of job {jobName}");
var status = await HelixApi.RetryAsync(
() => HelixApi.Job.PassFailAsync(jobName),
LogExceptionRetry);
if (status.Working > 0)
{
Log.LogError(
$"This task can only be used on completed jobs. There are {status.Working} of {status.Total} unfinished work items.");
return;
}
if (FailOnWorkItemFailure)
{
foreach (string failedWorkItem in status.Failed)
{
Log.LogError($"Work item {failedWorkItem} in job {jobName} has failed.");
}
}
if (FailOnMissionControlTestFailure)
{
for (; ; await Task.Delay(10000)) // delay every time this loop repeats
{
if (await MissionControlTestProcessingDoneAsync(jobName))
{
break;
}
Log.LogMessage($"Job {jobName} is still processing xunit results.");
}
}
}