Exemplo n.º 1
0
        private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null)
        {
            // Clean TEMP.
            _tempDirectoryManager?.CleanupTempDirectory(jobContext);

            jobContext.Section(StringUtil.Loc("StepFinishing", message.JobName));
            TaskResult result = jobContext.Complete(taskResult);

            if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
            {
                Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
                return(result);
            }

            await ShutdownQueue();

            Trace.Info("Raising job completed event.");
            IEnumerable <Variable> outputVariables = jobContext.Variables.GetOutputVariables();
            //var webApiVariables = outputVariables.ToJobCompletedEventOutputVariables();
            //var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, webApiVariables);
            var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);

            var completeJobRetryLimit = 5;
            var exceptions            = new List <Exception>();

            while (completeJobRetryLimit-- > 0)
            {
                try
                {
                    await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));

                    return(result);
                }
                catch (TaskOrchestrationPlanNotFoundException ex)
                {
                    Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
                    return(TaskResult.Failed);
                }
                catch (TaskOrchestrationPlanSecurityException ex)
                {
                    Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
                    return(TaskResult.Failed);
                }
                catch (Exception ex)
                {
                    Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
                    Trace.Error(ex);
                    exceptions.Add(ex);
                }

                // delay 5 seconds before next retry.
                await Task.Delay(TimeSpan.FromSeconds(5));
            }

            // rethrow exceptions from all attempts.
            throw new AggregateException(exceptions);
        }
Exemplo n.º 2
0
        public async Task ReportJobCompleted(string message, ITaskExecutionHandlerResult taskExecutionHandlerResult, CancellationToken cancellationToken)
        {
            var completedEvent = new JobCompletedEvent(this.taskMessage.JobId, taskExecutionHandlerResult.Result);
            var taskClient     = GetTaskClient(this.taskMessage.PlanUri, this.taskMessage.AuthToken);
            await taskClient.RaisePlanEventAsync(this.taskMessage.ProjectId, this.taskMessage.HubName, this.taskMessage.PlanId, completedEvent, cancellationToken).ConfigureAwait(false);

            // Find all existing timeline records and close them
            await this.CompleteTimelineRecords(this.taskMessage.ProjectId, this.taskMessage.PlanId, this.taskMessage.HubName, this.taskMessage.TimelineId, taskExecutionHandlerResult.Result, cancellationToken, taskClient);
        }
Exemplo n.º 3
0
 public override Task <Void> JobCompletedHandler(JobCompletedEvent request, ServerCallContext context)
 {
     if (IsDone)
     {
         return(Task.FromResult(Void));
     }
     Log.Log(Level.Info, "Job id {0} completed", request.JobId);
     UpdateStatusAndNotify(LauncherStatus.CompletedStatus);
     _completedJobHandler.OnNext(new CompletedJob(request.JobId));
     return(Task.FromResult(Void));
 }
Exemplo n.º 4
0
        private void MonitorJobStatus(Action job, string id, CancellationToken token)
        {
            var monitor = PeriodicTaskFactory.Start(job,
                                                    intervalInMilliseconds: 2000,
                                                    duration: 60000 * 60, // wait for max. 1 hour until abort
                                                    cancelToken: token);

            monitor.ContinueWith(_ =>
            {
                JobCompletedEvent?.Invoke(id);
            });
        }
Exemplo n.º 5
0
        private void OnJobCompleted(bool completed)
        {
            JobCompletedEvent handler = this.JobCompleted;

            if (handler != null)
            {
                handler(this,
                        new JobCompletedEventArgs
                {
                    Completed = completed
                });
            }
        }
Exemplo n.º 6
0
        private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null)
        {
            var result = jobContext.Complete(taskResult);

            if (message.Plan.Version < Constants.OmitFinishAgentRequestRunPlanVersion)
            {
                Trace.Verbose($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");

                return(result);
            }

            var outputVariables = jobContext.Variables.GetOutputVariables();
            //var webApiVariables = outputVariables.ToJobCompletedEventOutputVariables();
            //var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, webApiVariables);
            var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);

            var completeJobRetryLimit = 5;
            var exceptions            = new List <Exception>();

            while (completeJobRetryLimit-- > 0)
            {
                try
                {
                    await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));

                    return(result);
                }
                catch (TaskOrchestrationPlanNotFoundException ex)
                {
                    Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
                    return(TaskResult.Failed);
                }
                catch (TaskOrchestrationPlanSecurityException ex)
                {
                    Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
                    return(TaskResult.Failed);
                }
                catch (Exception ex)
                {
                    Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
                    Trace.Error(ex);
                    exceptions.Add(ex);
                }

                // delay 5 seconds before next retry.
                await Task.Delay(TimeSpan.FromSeconds(5));
            }

            // rethrow all catched exceptions during retry.
            throw new AggregateException(exceptions);
        }
Exemplo n.º 7
0
        public async Task ReportJobCompleted(DateTimeOffset offsetTime, string message, bool isPassed, CancellationToken cancellationToken)
        {
            var vstsPlanUrl = this.vstsContext.VstsPlanUri;
            var vstsUrl     = this.vstsContext.VstsUri;
            var authToken   = this.vstsContext.AuthToken;
            var planId      = this.vstsContext.PlanId;
            var projectId   = this.vstsContext.ProjectId;
            var jobId       = this.vstsContext.JobId;
            var hubName     = this.vstsContext.VstsHub.ToString();
            var timelineId  = this.vstsContext.TimelineId;
            var eventTime   = offsetTime.UtcDateTime;

            var buildHttpClientWrapper   = this.CreateBuildClient(vstsUrl, authToken);
            var releaseHttpClientWrapper = this.CreateReleaseClient(vstsPlanUrl, authToken);
            var isSessionValid           = await IsSessionValid(this.vstsContext, buildHttpClientWrapper, releaseHttpClientWrapper, cancellationToken).ConfigureAwait(false);

            if (!isSessionValid)
            {
                await this.baseInstrumentation.HandleInfoEvent("SessionAlreadyCancelled", "Skipping ReportJobStarted for cancelled or deleted build", this.eventProperties, cancellationToken).ConfigureAwait(false);

                return;
            }

            var taskHttpClientWrapper     = this.CreateTaskHttpClient(vstsPlanUrl, authToken, this.baseInstrumentation, this.vstsContext.SkipRaisePlanEvents);
            var vstsBrokerInstrumentation = new VstsBrokerInstrumentation(this.baseInstrumentation, taskHttpClientWrapper, hubName, projectId, planId, this.vstsContext.TaskLogId, this.eventProperties);

            var completedEvent = new JobCompletedEvent(jobId, isPassed ? TaskResult.Succeeded : TaskResult.Failed);
            await taskHttpClientWrapper.RaisePlanEventAsync(projectId, hubName, planId, completedEvent, cancellationToken).ConfigureAwait(false);

            if (isPassed)
            {
                await vstsBrokerInstrumentation.HandleInfoEvent("JobCompleted", message, this.eventProperties, cancellationToken, eventTime);
            }
            else
            {
                await vstsBrokerInstrumentation.HandleErrorEvent("JobFailed", message, this.eventProperties, cancellationToken, eventTime);
            }

            // Find all existing timeline records and close them
            await this.CompleteTimelineRecords(projectId, planId, hubName, timelineId, isPassed?TaskResult.Succeeded : TaskResult.Failed, cancellationToken, taskHttpClientWrapper);
        }
        private async Task TryFailOrchestrationPlan(T vstsMessage, CancellationToken cancellationToken)
        {
            if (vstsMessage == null)
            {
                return;
            }

            try
            {
                var projectId      = vstsMessage.ProjectId;
                var planId         = vstsMessage.PlanId;
                var vstsPlanUrl    = vstsMessage.VstsPlanUri;
                var authToken      = vstsMessage.AuthToken;
                var jobId          = vstsMessage.JobId;
                var hubName        = vstsMessage.VstsHub.ToString();
                var taskHttpClient = GetTaskClient(vstsPlanUrl, authToken, vstsMessage.SkipRaisePlanEvents);
                var completedEvent = new JobCompletedEvent(jobId, TaskResult.Abandoned);
                await taskHttpClient.RaisePlanEventAsync(projectId, hubName, planId, completedEvent, cancellationToken).ConfigureAwait(false);
            }
            catch (Exception)
            {
                // yes this really is a horrible best effort that ignores all ex's.
            }
        }
Exemplo n.º 9
0
        public async Task TryAbandonJob(CancellationToken cancellationToken)
        {
            if (taskMessage == null)
            {
                return;
            }

            try
            {
                var projectId      = taskMessage.ProjectId;
                var planId         = taskMessage.PlanId;
                var vstsPlanUrl    = taskMessage.PlanUri;
                var authToken      = taskMessage.AuthToken;
                var jobId          = taskMessage.JobId;
                var hubName        = taskMessage.HubName;
                var taskHttpClient = GetTaskClient(vstsPlanUrl, authToken);
                var completedEvent = new JobCompletedEvent(jobId, TaskResult.Abandoned);
                await taskHttpClient.RaisePlanEventAsync(projectId, hubName, planId, completedEvent, cancellationToken).ConfigureAwait(false);
            }
            catch (Exception)
            {
                // yes this really is a horrible best effort that ignores all ex's.
            }
        }
Exemplo n.º 10
0
        private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null)
        {
            // Clean TEMP.
            _tempDirectoryManager?.CleanupTempDirectory(jobContext);

            jobContext.Section(StringUtil.Loc("StepFinishing", message.JobName));
            TaskResult result = jobContext.Complete(taskResult);

            try
            {
                await ShutdownQueue(throwOnFailure : true);
            }
            catch (Exception ex)
            {
                Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}");
                Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs.");
                Trace.Error(ex);
                result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
            }

            if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
            {
                Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
                return(result);
            }

            Trace.Info("Raising job completed event.");
            var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);

            var completeJobRetryLimit = 5;
            var exceptions            = new List <Exception>();

            while (completeJobRetryLimit-- > 0)
            {
                try
                {
                    await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));

                    return(result);
                }
                catch (TaskOrchestrationPlanNotFoundException ex)
                {
                    Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
                    Trace.Error(ex);
                    return(TaskResult.Failed);
                }
                catch (TaskOrchestrationPlanSecurityException ex)
                {
                    Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
                    Trace.Error(ex);
                    return(TaskResult.Failed);
                }
                catch (Exception ex)
                {
                    Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
                    Trace.Error(ex);
                    exceptions.Add(ex);
                }

                // delay 5 seconds before next retry.
                await Task.Delay(TimeSpan.FromSeconds(5));
            }

            // rethrow exceptions from all attempts.
            throw new AggregateException(exceptions);
        }
Exemplo n.º 11
0
        private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult?taskResult = null)
        {
            jobContext.Debug($"Finishing: {message.JobDisplayName}");
            TaskResult result = jobContext.Complete(taskResult);

            try
            {
                await ShutdownQueue(throwOnFailure : true);
            }
            catch (Exception ex)
            {
                Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}");
                Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs.");
                Trace.Error(ex);
                result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
            }

            // Clean TEMP after finish process jobserverqueue, since there might be a pending fileupload still use the TEMP dir.
            _tempDirectoryManager?.CleanupTempDirectory();

            if (!jobContext.Global.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
            {
                Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
                return(result);
            }

            // Load any upgrade telemetry
            LoadFromTelemetryFile(jobContext.JobTelemetry);

            // Make sure we don't submit secrets as telemetry
            MaskTelemetrySecrets(jobContext.JobTelemetry);

            Trace.Info("Raising job completed event.");
            var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, jobContext.JobOutputs, jobContext.ActionsEnvironment, jobContext.ActionsStepsTelemetry, jobContext.JobTelemetry);


            var completeJobRetryLimit = 5;
            var exceptions            = new List <Exception>();

            while (completeJobRetryLimit-- > 0)
            {
                try
                {
                    await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));

                    return(result);
                }
                catch (TaskOrchestrationPlanNotFoundException ex)
                {
                    Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
                    Trace.Error(ex);
                    return(TaskResult.Failed);
                }
                catch (TaskOrchestrationPlanSecurityException ex)
                {
                    Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
                    Trace.Error(ex);
                    return(TaskResult.Failed);
                }
                catch (TaskOrchestrationPlanTerminatedException ex)
                {
                    Trace.Error($"TaskOrchestrationPlanTerminatedException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
                    Trace.Error(ex);
                    return(TaskResult.Failed);
                }
                catch (Exception ex)
                {
                    Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
                    Trace.Error(ex);
                    exceptions.Add(ex);
                }

                // delay 5 seconds before next retry.
                await Task.Delay(TimeSpan.FromSeconds(5));
            }

            // rethrow exceptions from all attempts.
            throw new AggregateException(exceptions);
        }