private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null)
{
// Clean TEMP.
_tempDirectoryManager?.CleanupTempDirectory(jobContext);
jobContext.Section(StringUtil.Loc("StepFinishing", message.JobName));
TaskResult result = jobContext.Complete(taskResult);
if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
{
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
await ShutdownQueue();
Trace.Info("Raising job completed event.");
IEnumerable <Variable> outputVariables = jobContext.Variables.GetOutputVariables();
//var webApiVariables = outputVariables.ToJobCompletedEventOutputVariables();
//var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, webApiVariables);
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
public async Task ReportJobCompleted(string message, ITaskExecutionHandlerResult taskExecutionHandlerResult, CancellationToken cancellationToken)
{
var completedEvent = new JobCompletedEvent(this.taskMessage.JobId, taskExecutionHandlerResult.Result);
var taskClient = GetTaskClient(this.taskMessage.PlanUri, this.taskMessage.AuthToken);
await taskClient.RaisePlanEventAsync(this.taskMessage.ProjectId, this.taskMessage.HubName, this.taskMessage.PlanId, completedEvent, cancellationToken).ConfigureAwait(false);
// Find all existing timeline records and close them
await this.CompleteTimelineRecords(this.taskMessage.ProjectId, this.taskMessage.PlanId, this.taskMessage.HubName, this.taskMessage.TimelineId, taskExecutionHandlerResult.Result, cancellationToken, taskClient);
}
public override Task <Void> JobCompletedHandler(JobCompletedEvent request, ServerCallContext context)
{
if (IsDone)
{
return(Task.FromResult(Void));
}
Log.Log(Level.Info, "Job id {0} completed", request.JobId);
UpdateStatusAndNotify(LauncherStatus.CompletedStatus);
_completedJobHandler.OnNext(new CompletedJob(request.JobId));
return(Task.FromResult(Void));
}
private void MonitorJobStatus(Action job, string id, CancellationToken token)
{
var monitor = PeriodicTaskFactory.Start(job,
intervalInMilliseconds: 2000,
duration: 60000 * 60, // wait for max. 1 hour until abort
cancelToken: token);
monitor.ContinueWith(_ =>
{
JobCompletedEvent?.Invoke(id);
});
}
private void OnJobCompleted(bool completed)
{
JobCompletedEvent handler = this.JobCompleted;
if (handler != null)
{
handler(this,
new JobCompletedEventArgs
{
Completed = completed
});
}
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null)
{
var result = jobContext.Complete(taskResult);
if (message.Plan.Version < Constants.OmitFinishAgentRequestRunPlanVersion)
{
Trace.Verbose($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
var outputVariables = jobContext.Variables.GetOutputVariables();
//var webApiVariables = outputVariables.ToJobCompletedEventOutputVariables();
//var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, webApiVariables);
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow all catched exceptions during retry.
throw new AggregateException(exceptions);
}
public async Task ReportJobCompleted(DateTimeOffset offsetTime, string message, bool isPassed, CancellationToken cancellationToken)
{
var vstsPlanUrl = this.vstsContext.VstsPlanUri;
var vstsUrl = this.vstsContext.VstsUri;
var authToken = this.vstsContext.AuthToken;
var planId = this.vstsContext.PlanId;
var projectId = this.vstsContext.ProjectId;
var jobId = this.vstsContext.JobId;
var hubName = this.vstsContext.VstsHub.ToString();
var timelineId = this.vstsContext.TimelineId;
var eventTime = offsetTime.UtcDateTime;
var buildHttpClientWrapper = this.CreateBuildClient(vstsUrl, authToken);
var releaseHttpClientWrapper = this.CreateReleaseClient(vstsPlanUrl, authToken);
var isSessionValid = await IsSessionValid(this.vstsContext, buildHttpClientWrapper, releaseHttpClientWrapper, cancellationToken).ConfigureAwait(false);
if (!isSessionValid)
{
await this.baseInstrumentation.HandleInfoEvent("SessionAlreadyCancelled", "Skipping ReportJobStarted for cancelled or deleted build", this.eventProperties, cancellationToken).ConfigureAwait(false);
return;
}
var taskHttpClientWrapper = this.CreateTaskHttpClient(vstsPlanUrl, authToken, this.baseInstrumentation, this.vstsContext.SkipRaisePlanEvents);
var vstsBrokerInstrumentation = new VstsBrokerInstrumentation(this.baseInstrumentation, taskHttpClientWrapper, hubName, projectId, planId, this.vstsContext.TaskLogId, this.eventProperties);
var completedEvent = new JobCompletedEvent(jobId, isPassed ? TaskResult.Succeeded : TaskResult.Failed);
await taskHttpClientWrapper.RaisePlanEventAsync(projectId, hubName, planId, completedEvent, cancellationToken).ConfigureAwait(false);
if (isPassed)
{
await vstsBrokerInstrumentation.HandleInfoEvent("JobCompleted", message, this.eventProperties, cancellationToken, eventTime);
}
else
{
await vstsBrokerInstrumentation.HandleErrorEvent("JobFailed", message, this.eventProperties, cancellationToken, eventTime);
}
// Find all existing timeline records and close them
await this.CompleteTimelineRecords(projectId, planId, hubName, timelineId, isPassed?TaskResult.Succeeded : TaskResult.Failed, cancellationToken, taskHttpClientWrapper);
}
private async Task TryFailOrchestrationPlan(T vstsMessage, CancellationToken cancellationToken)
{
if (vstsMessage == null)
{
return;
}
try
{
var projectId = vstsMessage.ProjectId;
var planId = vstsMessage.PlanId;
var vstsPlanUrl = vstsMessage.VstsPlanUri;
var authToken = vstsMessage.AuthToken;
var jobId = vstsMessage.JobId;
var hubName = vstsMessage.VstsHub.ToString();
var taskHttpClient = GetTaskClient(vstsPlanUrl, authToken, vstsMessage.SkipRaisePlanEvents);
var completedEvent = new JobCompletedEvent(jobId, TaskResult.Abandoned);
await taskHttpClient.RaisePlanEventAsync(projectId, hubName, planId, completedEvent, cancellationToken).ConfigureAwait(false);
}
catch (Exception)
{
// yes this really is a horrible best effort that ignores all ex's.
}
}
public async Task TryAbandonJob(CancellationToken cancellationToken)
{
if (taskMessage == null)
{
return;
}
try
{
var projectId = taskMessage.ProjectId;
var planId = taskMessage.PlanId;
var vstsPlanUrl = taskMessage.PlanUri;
var authToken = taskMessage.AuthToken;
var jobId = taskMessage.JobId;
var hubName = taskMessage.HubName;
var taskHttpClient = GetTaskClient(vstsPlanUrl, authToken);
var completedEvent = new JobCompletedEvent(jobId, TaskResult.Abandoned);
await taskHttpClient.RaisePlanEventAsync(projectId, hubName, planId, completedEvent, cancellationToken).ConfigureAwait(false);
}
catch (Exception)
{
// yes this really is a horrible best effort that ignores all ex's.
}
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null)
{
// Clean TEMP.
_tempDirectoryManager?.CleanupTempDirectory(jobContext);
jobContext.Section(StringUtil.Loc("StepFinishing", message.JobName));
TaskResult result = jobContext.Complete(taskResult);
try
{
await ShutdownQueue(throwOnFailure : true);
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}");
Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs.");
Trace.Error(ex);
result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
}
if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
{
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
Trace.Info("Raising job completed event.");
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult?taskResult = null)
{
jobContext.Debug($"Finishing: {message.JobDisplayName}");
TaskResult result = jobContext.Complete(taskResult);
try
{
await ShutdownQueue(throwOnFailure : true);
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}");
Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs.");
Trace.Error(ex);
result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
}
// Clean TEMP after finish process jobserverqueue, since there might be a pending fileupload still use the TEMP dir.
_tempDirectoryManager?.CleanupTempDirectory();
if (!jobContext.Global.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
{
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
// Load any upgrade telemetry
LoadFromTelemetryFile(jobContext.JobTelemetry);
// Make sure we don't submit secrets as telemetry
MaskTelemetrySecrets(jobContext.JobTelemetry);
Trace.Info("Raising job completed event.");
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, jobContext.JobOutputs, jobContext.ActionsEnvironment, jobContext.ActionsStepsTelemetry, jobContext.JobTelemetry);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanTerminatedException ex)
{
Trace.Error($"TaskOrchestrationPlanTerminatedException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
