private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null) { // Clean TEMP. _tempDirectoryManager?.CleanupTempDirectory(jobContext); jobContext.Section(StringUtil.Loc("StepFinishing", message.JobName)); TaskResult result = jobContext.Complete(taskResult); if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent)) { Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}"); return(result); } await ShutdownQueue(); Trace.Info("Raising job completed event."); IEnumerable <Variable> outputVariables = jobContext.Variables.GetOutputVariables(); //var webApiVariables = outputVariables.ToJobCompletedEventOutputVariables(); //var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, webApiVariables); var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result); var completeJobRetryLimit = 5; var exceptions = new List <Exception>(); while (completeJobRetryLimit-- > 0) { try { await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken)); return(result); } catch (TaskOrchestrationPlanNotFoundException ex) { Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}"); return(TaskResult.Failed); } catch (TaskOrchestrationPlanSecurityException ex) { Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}"); return(TaskResult.Failed); } catch (Exception ex) { Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}."); Trace.Error(ex); exceptions.Add(ex); } // delay 5 seconds before next retry. await Task.Delay(TimeSpan.FromSeconds(5)); } // rethrow exceptions from all attempts. throw new AggregateException(exceptions); }
public async Task ReportJobCompleted(string message, ITaskExecutionHandlerResult taskExecutionHandlerResult, CancellationToken cancellationToken) { var completedEvent = new JobCompletedEvent(this.taskMessage.JobId, taskExecutionHandlerResult.Result); var taskClient = GetTaskClient(this.taskMessage.PlanUri, this.taskMessage.AuthToken); await taskClient.RaisePlanEventAsync(this.taskMessage.ProjectId, this.taskMessage.HubName, this.taskMessage.PlanId, completedEvent, cancellationToken).ConfigureAwait(false); // Find all existing timeline records and close them await this.CompleteTimelineRecords(this.taskMessage.ProjectId, this.taskMessage.PlanId, this.taskMessage.HubName, this.taskMessage.TimelineId, taskExecutionHandlerResult.Result, cancellationToken, taskClient); }
public override Task <Void> JobCompletedHandler(JobCompletedEvent request, ServerCallContext context) { if (IsDone) { return(Task.FromResult(Void)); } Log.Log(Level.Info, "Job id {0} completed", request.JobId); UpdateStatusAndNotify(LauncherStatus.CompletedStatus); _completedJobHandler.OnNext(new CompletedJob(request.JobId)); return(Task.FromResult(Void)); }
private void MonitorJobStatus(Action job, string id, CancellationToken token) { var monitor = PeriodicTaskFactory.Start(job, intervalInMilliseconds: 2000, duration: 60000 * 60, // wait for max. 1 hour until abort cancelToken: token); monitor.ContinueWith(_ => { JobCompletedEvent?.Invoke(id); }); }
private void OnJobCompleted(bool completed) { JobCompletedEvent handler = this.JobCompleted; if (handler != null) { handler(this, new JobCompletedEventArgs { Completed = completed }); } }
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null) { var result = jobContext.Complete(taskResult); if (message.Plan.Version < Constants.OmitFinishAgentRequestRunPlanVersion) { Trace.Verbose($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}"); return(result); } var outputVariables = jobContext.Variables.GetOutputVariables(); //var webApiVariables = outputVariables.ToJobCompletedEventOutputVariables(); //var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, webApiVariables); var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result); var completeJobRetryLimit = 5; var exceptions = new List <Exception>(); while (completeJobRetryLimit-- > 0) { try { await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken)); return(result); } catch (TaskOrchestrationPlanNotFoundException ex) { Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}"); return(TaskResult.Failed); } catch (TaskOrchestrationPlanSecurityException ex) { Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}"); return(TaskResult.Failed); } catch (Exception ex) { Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}."); Trace.Error(ex); exceptions.Add(ex); } // delay 5 seconds before next retry. await Task.Delay(TimeSpan.FromSeconds(5)); } // rethrow all catched exceptions during retry. throw new AggregateException(exceptions); }
public async Task ReportJobCompleted(DateTimeOffset offsetTime, string message, bool isPassed, CancellationToken cancellationToken) { var vstsPlanUrl = this.vstsContext.VstsPlanUri; var vstsUrl = this.vstsContext.VstsUri; var authToken = this.vstsContext.AuthToken; var planId = this.vstsContext.PlanId; var projectId = this.vstsContext.ProjectId; var jobId = this.vstsContext.JobId; var hubName = this.vstsContext.VstsHub.ToString(); var timelineId = this.vstsContext.TimelineId; var eventTime = offsetTime.UtcDateTime; var buildHttpClientWrapper = this.CreateBuildClient(vstsUrl, authToken); var releaseHttpClientWrapper = this.CreateReleaseClient(vstsPlanUrl, authToken); var isSessionValid = await IsSessionValid(this.vstsContext, buildHttpClientWrapper, releaseHttpClientWrapper, cancellationToken).ConfigureAwait(false); if (!isSessionValid) { await this.baseInstrumentation.HandleInfoEvent("SessionAlreadyCancelled", "Skipping ReportJobStarted for cancelled or deleted build", this.eventProperties, cancellationToken).ConfigureAwait(false); return; } var taskHttpClientWrapper = this.CreateTaskHttpClient(vstsPlanUrl, authToken, this.baseInstrumentation, this.vstsContext.SkipRaisePlanEvents); var vstsBrokerInstrumentation = new VstsBrokerInstrumentation(this.baseInstrumentation, taskHttpClientWrapper, hubName, projectId, planId, this.vstsContext.TaskLogId, this.eventProperties); var completedEvent = new JobCompletedEvent(jobId, isPassed ? TaskResult.Succeeded : TaskResult.Failed); await taskHttpClientWrapper.RaisePlanEventAsync(projectId, hubName, planId, completedEvent, cancellationToken).ConfigureAwait(false); if (isPassed) { await vstsBrokerInstrumentation.HandleInfoEvent("JobCompleted", message, this.eventProperties, cancellationToken, eventTime); } else { await vstsBrokerInstrumentation.HandleErrorEvent("JobFailed", message, this.eventProperties, cancellationToken, eventTime); } // Find all existing timeline records and close them await this.CompleteTimelineRecords(projectId, planId, hubName, timelineId, isPassed?TaskResult.Succeeded : TaskResult.Failed, cancellationToken, taskHttpClientWrapper); }
private async Task TryFailOrchestrationPlan(T vstsMessage, CancellationToken cancellationToken) { if (vstsMessage == null) { return; } try { var projectId = vstsMessage.ProjectId; var planId = vstsMessage.PlanId; var vstsPlanUrl = vstsMessage.VstsPlanUri; var authToken = vstsMessage.AuthToken; var jobId = vstsMessage.JobId; var hubName = vstsMessage.VstsHub.ToString(); var taskHttpClient = GetTaskClient(vstsPlanUrl, authToken, vstsMessage.SkipRaisePlanEvents); var completedEvent = new JobCompletedEvent(jobId, TaskResult.Abandoned); await taskHttpClient.RaisePlanEventAsync(projectId, hubName, planId, completedEvent, cancellationToken).ConfigureAwait(false); } catch (Exception) { // yes this really is a horrible best effort that ignores all ex's. } }
public async Task TryAbandonJob(CancellationToken cancellationToken) { if (taskMessage == null) { return; } try { var projectId = taskMessage.ProjectId; var planId = taskMessage.PlanId; var vstsPlanUrl = taskMessage.PlanUri; var authToken = taskMessage.AuthToken; var jobId = taskMessage.JobId; var hubName = taskMessage.HubName; var taskHttpClient = GetTaskClient(vstsPlanUrl, authToken); var completedEvent = new JobCompletedEvent(jobId, TaskResult.Abandoned); await taskHttpClient.RaisePlanEventAsync(projectId, hubName, planId, completedEvent, cancellationToken).ConfigureAwait(false); } catch (Exception) { // yes this really is a horrible best effort that ignores all ex's. } }
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null) { // Clean TEMP. _tempDirectoryManager?.CleanupTempDirectory(jobContext); jobContext.Section(StringUtil.Loc("StepFinishing", message.JobName)); TaskResult result = jobContext.Complete(taskResult); try { await ShutdownQueue(throwOnFailure : true); } catch (Exception ex) { Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}"); Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs."); Trace.Error(ex); result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed); } if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent)) { Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}"); return(result); } Trace.Info("Raising job completed event."); var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result); var completeJobRetryLimit = 5; var exceptions = new List <Exception>(); while (completeJobRetryLimit-- > 0) { try { await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken)); return(result); } catch (TaskOrchestrationPlanNotFoundException ex) { Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}."); Trace.Error(ex); return(TaskResult.Failed); } catch (TaskOrchestrationPlanSecurityException ex) { Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}."); Trace.Error(ex); return(TaskResult.Failed); } catch (Exception ex) { Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}."); Trace.Error(ex); exceptions.Add(ex); } // delay 5 seconds before next retry. await Task.Delay(TimeSpan.FromSeconds(5)); } // rethrow exceptions from all attempts. throw new AggregateException(exceptions); }
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult?taskResult = null) { jobContext.Debug($"Finishing: {message.JobDisplayName}"); TaskResult result = jobContext.Complete(taskResult); try { await ShutdownQueue(throwOnFailure : true); } catch (Exception ex) { Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}"); Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs."); Trace.Error(ex); result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed); } // Clean TEMP after finish process jobserverqueue, since there might be a pending fileupload still use the TEMP dir. _tempDirectoryManager?.CleanupTempDirectory(); if (!jobContext.Global.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent)) { Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}"); return(result); } // Load any upgrade telemetry LoadFromTelemetryFile(jobContext.JobTelemetry); // Make sure we don't submit secrets as telemetry MaskTelemetrySecrets(jobContext.JobTelemetry); Trace.Info("Raising job completed event."); var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, jobContext.JobOutputs, jobContext.ActionsEnvironment, jobContext.ActionsStepsTelemetry, jobContext.JobTelemetry); var completeJobRetryLimit = 5; var exceptions = new List <Exception>(); while (completeJobRetryLimit-- > 0) { try { await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken)); return(result); } catch (TaskOrchestrationPlanNotFoundException ex) { Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}."); Trace.Error(ex); return(TaskResult.Failed); } catch (TaskOrchestrationPlanSecurityException ex) { Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}."); Trace.Error(ex); return(TaskResult.Failed); } catch (TaskOrchestrationPlanTerminatedException ex) { Trace.Error($"TaskOrchestrationPlanTerminatedException received, while attempting to raise JobCompletedEvent for job {message.JobId}."); Trace.Error(ex); return(TaskResult.Failed); } catch (Exception ex) { Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}."); Trace.Error(ex); exceptions.Add(ex); } // delay 5 seconds before next retry. await Task.Delay(TimeSpan.FromSeconds(5)); } // rethrow exceptions from all attempts. throw new AggregateException(exceptions); }