protected override JobRunStatus RetrieveCurrentStatus()
{
if (_jobServer == null)
{
Logger.Debug("Creating rest api client for scheduler");
_jobServer = _jobServerProvider();
}
var taskId = _originalStatus.Id;
JobRunStatus currentStatus;
try
{
Logger.Debug($"Fetching current status for task {taskId} with previous index of {_previousMessageIndex}");
currentStatus = _jobServer.GetJobRunStatus(taskId, _previousMessageIndex).Result;
_previousMessageIndex = currentStatus.CurrentMessageIndex;
Logger.Debug($"Task {taskId} has status of {currentStatus}");
Logger.Debug($"Showing all {currentStatus.Messages.Length} messages");
foreach (var statusMessage in currentStatus.Messages)
{
Logger.Debug($"Message: {statusMessage}");
_messagePresenter.ShowMessage(statusMessage);
}
}
catch (Exception ex)
{
Logger.Error(ex,
$"While waiting for the task to finish, an exception was thrown: {ex.Message} Stack Trace: {ex.StackTrace}");
currentStatus = _originalStatus.Copy();
currentStatus.Result = Result.Failure(
"Lost connection to the server, and so couldn't finish processing this task");
currentStatus.State = JobRunState.Finished;
currentStatus.FinishTime = DateTime.Now;
}
return(currentStatus);
}
public F_FoodInfoBusiness(IRepository repository, IOperator op, IJobServer _job, ILogger <F_FoodInfoBusiness> _loger)
: base(repository)
{
oOperator = op;
Job = _job;
loger = _loger;
}
protected override JobRunStatus RetrieveCurrentStatus()
{
if (_jobServer == null)
{
Logger.Debug("Creating rest api client for scheduler");
_jobServer = _jobServerProvider();
}
var taskId = _originalStatus.Id;
JobRunStatus currentStatus;
try
{
Log.Debug($"Fetching current status for task {taskId}");
currentStatus = _jobServer.GetJobRunStatus(taskId).Result;
Log.Debug($"Task {taskId} has status of {currentStatus}");
_jobRunStatusPresenter.PresentAnyChangesTo(currentStatus);
}
catch (Exception ex)
{
Logger.Error(ex,
$"While waiting for the task to finish, an exception was thrown: {ex.Message} Stack Trace: {ex.StackTrace}");
currentStatus = _originalStatus.Copy();
currentStatus.Result = Result.Failure(
"Lost connection to the server, and so couldn't finish processing this task");
currentStatus.State = JobRunState.Finished;
currentStatus.FinishTime = DateTime.Now;
}
return(currentStatus);
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null)
{
// Clean TEMP.
_tempDirectoryManager?.CleanupTempDirectory(jobContext);
jobContext.Section(StringUtil.Loc("StepFinishing", message.JobName));
TaskResult result = jobContext.Complete(taskResult);
if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
{
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
await ShutdownQueue();
Trace.Info("Raising job completed event.");
IEnumerable <Variable> outputVariables = jobContext.Variables.GetOutputVariables();
//var webApiVariables = outputVariables.ToJobCompletedEventOutputVariables();
//var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, webApiVariables);
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
public Form1()
{
InitializeComponent();
// Get a reference to the remote object.
m_IJobServer = GetIJobServer();
EventProxy eventProxy = new EventProxy();
eventProxy.MyJobEventHandler += this.MyJobEventHandler;
// Subscribe to the JobEvent.
m_IJobServer.JobEvent +=
new JobEventHandler(eventProxy.OnCalculateStateChanged);
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null)
{
var result = jobContext.Complete(taskResult);
if (message.Plan.Version < Constants.OmitFinishAgentRequestRunPlanVersion)
{
Trace.Verbose($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
var outputVariables = jobContext.Variables.GetOutputVariables();
//var webApiVariables = outputVariables.ToJobCompletedEventOutputVariables();
//var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, webApiVariables);
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow all catched exceptions during retry.
throw new AggregateException(exceptions);
}
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_jobServer = hostContext.GetService <IJobServer>();
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null)
{
// Clean TEMP.
_tempDirectoryManager?.CleanupTempDirectory(jobContext);
jobContext.Section(StringUtil.Loc("StepFinishing", message.JobName));
TaskResult result = jobContext.Complete(taskResult);
try
{
await ShutdownQueue(throwOnFailure : true);
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}");
Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs.");
Trace.Error(ex);
result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
}
if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
{
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
Trace.Info("Raising job completed event.");
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult?taskResult = null)
{
jobContext.Debug($"Finishing: {message.JobDisplayName}");
TaskResult result = jobContext.Complete(taskResult);
try
{
await ShutdownQueue(throwOnFailure : true);
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}");
Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs.");
Trace.Error(ex);
result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
}
// Clean TEMP after finish process jobserverqueue, since there might be a pending fileupload still use the TEMP dir.
_tempDirectoryManager?.CleanupTempDirectory();
if (!jobContext.Global.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
{
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
// Load any upgrade telemetry
LoadFromTelemetryFile(jobContext.JobTelemetry);
// Make sure we don't submit secrets as telemetry
MaskTelemetrySecrets(jobContext.JobTelemetry);
Trace.Info("Raising job completed event.");
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, jobContext.JobOutputs, jobContext.ActionsEnvironment, jobContext.ActionsStepsTelemetry, jobContext.JobTelemetry);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanTerminatedException ex)
{
Trace.Error($"TaskOrchestrationPlanTerminatedException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_jobServer = hostContext.GetService<IJobServer>();
}
