/// <summary>
/// Executes job's child jobs synchronously.
/// </summary>
/// <param name="tasksExecuted">
/// Receives the number of tasks executed by this job's child jobs.
/// </param>
/// <returns>
/// The result of the execution of this job's child jobs.
/// </returns>
private OrchestratedExecutionResult ExecuteChildJobsSynchronous(out int tasksExecuted)
{
OrchestratedExecutionResult result = OrchestratedExecutionResult.Success;
tasksExecuted = 0;
if (Job.ChildJobs != null)
{
Log.Verbose("ScheduledJob ID {0}: Executing synchronous job type {1} child jobs.", JobDetails.JobId,
Job.GetType().Name);
foreach (IOrchestratedJob job in Job.ChildJobs)
{
JobOrchestrator jobOrchestrator = new JobOrchestrator(job, JobDetails, Log);
OrchestratedExecutionResult jobResult = jobOrchestrator.Execute(out tasksExecuted);
if (jobResult == OrchestratedExecutionResult.TerminalError)
{
result = jobResult;
break;
}
else if (jobResult == OrchestratedExecutionResult.NonTerminalError)
{
result = jobResult;
}
}
}
return(result);
}
/// <summary>
/// Runs the orchestrated job described in the specified ScheduleJobDetails object.
/// </summary>
/// <param name="jobDetails">
/// The details of the orchestrated job to run.
/// </param>
/// <remarks>
/// A job is limited to MaxJobRetries runs, even for successful runs, before being placed back in the queue to ensure
/// that a job in an endless loop does not take down the entire worker role (pre-emptive multi-tasking.) But, if the job
/// times out, another worker instance may attempt to run the job while it's still being run in another instance. Going
/// forward, some mechanism to halt execution at time out should be added.
/// </remarks>
public async Task RunJobAsync(ScheduledJobDetails jobDetails)
{
OrchestratedExecutionResult executionResult;
int maxRetries = CommerceConfig.MaxJobRetries;
int retryLatency = CommerceConfig.InitialJobRetryLatency;
int tryCount = 0;
int tasksPerformed = 0;
IOrchestratedJob job = JobFactory(jobDetails, Scheduler, Log);
JobOrchestrator jobOrchestrator = null;
if (job != null)
{
jobOrchestrator = new JobOrchestrator(job, jobDetails, Log);
}
if (jobOrchestrator != null)
{
do
{
try
{
Task <OrchestratedExecutionResult> executionResultTask =
Task.Factory.StartNew(() => jobOrchestrator.Execute(out tasksPerformed));
executionResult = await executionResultTask;
}
catch (InvalidOperationException ex)
{
ResultCode resultCode;
if (Enum.TryParse <ResultCode>(ex.Message, out resultCode) == true)
{
executionResult = OrchestratedExecutionResult.NonTerminalError;
}
else
{
throw;
}
}
Log.Verbose("{0} orchestrated job completed {1} steps with result {2}.", jobDetails.JobType,
tasksPerformed, executionResult);
if (executionResult == OrchestratedExecutionResult.NonTerminalError && tryCount <= maxRetries)
{
Log.Verbose("Waiting {0} milliseconds before retrying job execution.", retryLatency);
Thread.Sleep(retryLatency);
retryLatency *= 2;
}
tryCount++;
}while (executionResult != OrchestratedExecutionResult.TerminalError &&
tasksPerformed > 0 &&
tryCount <= maxRetries);
// tear down the job here.
executionResult = jobOrchestrator.Cleanup(executionResult);
}
else
{
executionResult = OrchestratedExecutionResult.TerminalError;
}
StringBuilder stringBuilder = new StringBuilder("{0} orchestrated job completed with result {1}.");
if (executionResult == OrchestratedExecutionResult.NonTerminalError)
{
stringBuilder.Append(" Job will be sent to the back of the queue for reprocessing.");
}
Log.Information(stringBuilder.ToString(), jobDetails.JobType, executionResult);
// Update Scheduler with result of running the job.
switch (executionResult)
{
case OrchestratedExecutionResult.Success:
await Scheduler.CompleteJobIterationAsync(jobDetails).ConfigureAwait(false);
break;
case OrchestratedExecutionResult.TerminalError:
jobDetails.JobState = ScheduledJobState.Canceled;
jobDetails.Payload = null;
await Scheduler.UpdateJobAsync(jobDetails).ConfigureAwait(false);
break;
case OrchestratedExecutionResult.NonTerminalError:
await Scheduler.ExponentiallyBackoffAsync(jobDetails, Log).ConfigureAwait(false);
break;
}
}
/// <summary>
/// Executes job tasks and child jobs asynchronously.
/// </summary>
/// <param name="tasksExecuted">
/// Receives the number of tasks executed by this job and all child jobs.
/// </param>
/// <returns>
/// The result of the execution of this job and all child jobs.
/// </returns>
private OrchestratedExecutionResult ExecuteAsynchronous(out int tasksExecuted)
{
OrchestratedExecutionResult result = OrchestratedExecutionResult.Success;
tasksExecuted = 0;
Log.Verbose("ScheduledJob ID {0}: Executing job type {1} tasks and child jobs asynchronously.", JobDetails.JobId,
Job.GetType().Name);
// Determine how many item will execute while running this job.
int itemCount = 0;
if (Job.Tasks != null)
{
itemCount = Job.Tasks.Count;
}
if (Job.ChildJobs != null)
{
itemCount += Job.ChildJobs.Count;
}
// If there are any items to execute, do so asynchronously.
if (itemCount > 0)
{
// Spin up threads to execute the tasks and child jobs.
List <Task> executors = new List <Task>(itemCount);
ConcurrentDictionary <Guid, Tuple <int, OrchestratedExecutionResult> > itemTaskResults =
new ConcurrentDictionary <Guid, Tuple <int, OrchestratedExecutionResult> >(itemCount, itemCount);
if (Job.Tasks != null)
{
IEnumerable <OrchestratedExecutionResult> results = ParallelTaskThrottler.Instance.Run(Job.Tasks);
foreach (OrchestratedExecutionResult orchestratedExecutionResult in results)
{
itemTaskResults[Guid.NewGuid()] = new Tuple <int, OrchestratedExecutionResult>(1, orchestratedExecutionResult);
}
}
if (Job.ChildJobs != null)
{
foreach (IOrchestratedJob job in Job.ChildJobs)
{
JobOrchestrator jobOrchestrator = new JobOrchestrator(job, JobDetails, Log);
executors.Add(Task.Factory.StartNew(() =>
{
int childTasksExecuted;
OrchestratedExecutionResult childResult =
jobOrchestrator.Execute(out childTasksExecuted);
itemTaskResults[Guid.NewGuid()] =
new Tuple <int, OrchestratedExecutionResult>(childTasksExecuted, childResult);
}));
}
}
// Wait until all threads have completed their work.
Task.WaitAll(executors.ToArray());
// Tally up the completed tasks and get overall result.
foreach (Guid key in itemTaskResults.Keys)
{
Tuple <int, OrchestratedExecutionResult> itemTaskResult = itemTaskResults[key];
tasksExecuted += itemTaskResult.Item1;
if (itemTaskResult.Item2 == OrchestratedExecutionResult.TerminalError)
{
result = OrchestratedExecutionResult.TerminalError;
}
else if (itemTaskResult.Item2 == OrchestratedExecutionResult.NonTerminalError &&
result != OrchestratedExecutionResult.TerminalError)
{
result = OrchestratedExecutionResult.NonTerminalError;
}
}
}
return(result);
}
