/// <summary> /// Executes job's child jobs synchronously. /// </summary> /// <param name="tasksExecuted"> /// Receives the number of tasks executed by this job's child jobs. /// </param> /// <returns> /// The result of the execution of this job's child jobs. /// </returns> private OrchestratedExecutionResult ExecuteChildJobsSynchronous(out int tasksExecuted) { OrchestratedExecutionResult result = OrchestratedExecutionResult.Success; tasksExecuted = 0; if (Job.ChildJobs != null) { Log.Verbose("ScheduledJob ID {0}: Executing synchronous job type {1} child jobs.", JobDetails.JobId, Job.GetType().Name); foreach (IOrchestratedJob job in Job.ChildJobs) { JobOrchestrator jobOrchestrator = new JobOrchestrator(job, JobDetails, Log); OrchestratedExecutionResult jobResult = jobOrchestrator.Execute(out tasksExecuted); if (jobResult == OrchestratedExecutionResult.TerminalError) { result = jobResult; break; } else if (jobResult == OrchestratedExecutionResult.NonTerminalError) { result = jobResult; } } } return(result); }
/// <summary> /// Runs the orchestrated job described in the specified ScheduleJobDetails object. /// </summary> /// <param name="jobDetails"> /// The details of the orchestrated job to run. /// </param> /// <remarks> /// A job is limited to MaxJobRetries runs, even for successful runs, before being placed back in the queue to ensure /// that a job in an endless loop does not take down the entire worker role (pre-emptive multi-tasking.) But, if the job /// times out, another worker instance may attempt to run the job while it's still being run in another instance. Going /// forward, some mechanism to halt execution at time out should be added. /// </remarks> public async Task RunJobAsync(ScheduledJobDetails jobDetails) { OrchestratedExecutionResult executionResult; int maxRetries = CommerceConfig.MaxJobRetries; int retryLatency = CommerceConfig.InitialJobRetryLatency; int tryCount = 0; int tasksPerformed = 0; IOrchestratedJob job = JobFactory(jobDetails, Scheduler, Log); JobOrchestrator jobOrchestrator = null; if (job != null) { jobOrchestrator = new JobOrchestrator(job, jobDetails, Log); } if (jobOrchestrator != null) { do { try { Task <OrchestratedExecutionResult> executionResultTask = Task.Factory.StartNew(() => jobOrchestrator.Execute(out tasksPerformed)); executionResult = await executionResultTask; } catch (InvalidOperationException ex) { ResultCode resultCode; if (Enum.TryParse <ResultCode>(ex.Message, out resultCode) == true) { executionResult = OrchestratedExecutionResult.NonTerminalError; } else { throw; } } Log.Verbose("{0} orchestrated job completed {1} steps with result {2}.", jobDetails.JobType, tasksPerformed, executionResult); if (executionResult == OrchestratedExecutionResult.NonTerminalError && tryCount <= maxRetries) { Log.Verbose("Waiting {0} milliseconds before retrying job execution.", retryLatency); Thread.Sleep(retryLatency); retryLatency *= 2; } tryCount++; }while (executionResult != OrchestratedExecutionResult.TerminalError && tasksPerformed > 0 && tryCount <= maxRetries); // tear down the job here. executionResult = jobOrchestrator.Cleanup(executionResult); } else { executionResult = OrchestratedExecutionResult.TerminalError; } StringBuilder stringBuilder = new StringBuilder("{0} orchestrated job completed with result {1}."); if (executionResult == OrchestratedExecutionResult.NonTerminalError) { stringBuilder.Append(" Job will be sent to the back of the queue for reprocessing."); } Log.Information(stringBuilder.ToString(), jobDetails.JobType, executionResult); // Update Scheduler with result of running the job. switch (executionResult) { case OrchestratedExecutionResult.Success: await Scheduler.CompleteJobIterationAsync(jobDetails).ConfigureAwait(false); break; case OrchestratedExecutionResult.TerminalError: jobDetails.JobState = ScheduledJobState.Canceled; jobDetails.Payload = null; await Scheduler.UpdateJobAsync(jobDetails).ConfigureAwait(false); break; case OrchestratedExecutionResult.NonTerminalError: await Scheduler.ExponentiallyBackoffAsync(jobDetails, Log).ConfigureAwait(false); break; } }
/// <summary> /// Executes job tasks and child jobs asynchronously. /// </summary> /// <param name="tasksExecuted"> /// Receives the number of tasks executed by this job and all child jobs. /// </param> /// <returns> /// The result of the execution of this job and all child jobs. /// </returns> private OrchestratedExecutionResult ExecuteAsynchronous(out int tasksExecuted) { OrchestratedExecutionResult result = OrchestratedExecutionResult.Success; tasksExecuted = 0; Log.Verbose("ScheduledJob ID {0}: Executing job type {1} tasks and child jobs asynchronously.", JobDetails.JobId, Job.GetType().Name); // Determine how many item will execute while running this job. int itemCount = 0; if (Job.Tasks != null) { itemCount = Job.Tasks.Count; } if (Job.ChildJobs != null) { itemCount += Job.ChildJobs.Count; } // If there are any items to execute, do so asynchronously. if (itemCount > 0) { // Spin up threads to execute the tasks and child jobs. List <Task> executors = new List <Task>(itemCount); ConcurrentDictionary <Guid, Tuple <int, OrchestratedExecutionResult> > itemTaskResults = new ConcurrentDictionary <Guid, Tuple <int, OrchestratedExecutionResult> >(itemCount, itemCount); if (Job.Tasks != null) { IEnumerable <OrchestratedExecutionResult> results = ParallelTaskThrottler.Instance.Run(Job.Tasks); foreach (OrchestratedExecutionResult orchestratedExecutionResult in results) { itemTaskResults[Guid.NewGuid()] = new Tuple <int, OrchestratedExecutionResult>(1, orchestratedExecutionResult); } } if (Job.ChildJobs != null) { foreach (IOrchestratedJob job in Job.ChildJobs) { JobOrchestrator jobOrchestrator = new JobOrchestrator(job, JobDetails, Log); executors.Add(Task.Factory.StartNew(() => { int childTasksExecuted; OrchestratedExecutionResult childResult = jobOrchestrator.Execute(out childTasksExecuted); itemTaskResults[Guid.NewGuid()] = new Tuple <int, OrchestratedExecutionResult>(childTasksExecuted, childResult); })); } } // Wait until all threads have completed their work. Task.WaitAll(executors.ToArray()); // Tally up the completed tasks and get overall result. foreach (Guid key in itemTaskResults.Keys) { Tuple <int, OrchestratedExecutionResult> itemTaskResult = itemTaskResults[key]; tasksExecuted += itemTaskResult.Item1; if (itemTaskResult.Item2 == OrchestratedExecutionResult.TerminalError) { result = OrchestratedExecutionResult.TerminalError; } else if (itemTaskResult.Item2 == OrchestratedExecutionResult.NonTerminalError && result != OrchestratedExecutionResult.TerminalError) { result = OrchestratedExecutionResult.NonTerminalError; } } } return(result); }