private async void ProcessSingleJob(
TJobKey jobId,
IJobMetadata jobMetadata,
TSchedulerKey schedluerId,
CancellationToken cancellationToken)
{
var sw = Stopwatch.StartNew();
try
{
this.logger.LogTrace("Trying to execute job {0} at scheduler {1}", jobId, schedluerId);
await Task.Yield();
var res = await this.singleJobProcessor.ProcessSingleJob(jobId, jobMetadata, schedluerId, cancellationToken).ConfigureAwait(false);
sw.Stop();
this.logger.LogInformation("Job {0} executed at scheduler {1} in {2}. Result: {3}", jobId, schedluerId, sw.Elapsed, res);
}
catch (OperationCanceledException ex) when(cancellationToken.IsCancellationRequested)
{
sw.Stop();
this.logger.LogWarning(ex, "Job {0} has been canceled due scheduler {1} cancellation after {2}", jobId, schedluerId, sw.Elapsed);
}
catch (Exception ex)
{
sw.Stop();
this.logger.LogError(ex, "Job {0} failed at scheduler {1} in {2}: {3}", jobId, schedluerId, sw.Elapsed, ex.Message);
}
}
private async Task ProcessTimeoutedJob(TJobKey jobId, IJobMetadata jobMetadata, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
jobMetadata.SetNextExecutionTime();
}
catch (Exception ex)
{
this.logger.LogError(ex, "Failed to calculate execution time of job {0}: {1}. Job will be rescheduled immediately", jobId, ex.Message);
}
try
{
await this.jobStore.FinalizeJob(jobId, jobMetadata, JobExecutionResult.Timeouted).ConfigureAwait(false);
this.logger.LogInformation("TImeouted job {0} rescheduled", jobId);
}
catch (ConcurrencyException ex)
{
this.logger.LogInformation(ex, "Job {0} was recovered by someone else: {1}", jobId, ex.Message);
}
catch (Exception ex)
{
this.logger.LogError(ex, "Processing timeouted job {0} failed: {1}", jobId, ex.Message);
}
}
private IServiceProvider CreateServiceProvider(IJobMetadata jobMetadata, ServiceLifetime serviceLifetime)
{
var serviceCollection = new ServiceCollection();
serviceCollection.Insert(0, new ServiceDescriptor(jobMetadata.JobClass, jobMetadata.JobClass, serviceLifetime));
return(serviceCollection.BuildServiceProvider());
}
public Task Invoke(IJobMetadata jobMetadata, CancellationToken cancellationToken)
{
if (this.objectDisposed)
{
throw new ObjectDisposedException(this.GetType().Name);
}
return(Task.CompletedTask);
}
private async Task ExecuteJobInternal(IJobMetadata jobMetadata, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
using (this.jobFactory.BeginScope())
{
var job = this.jobFactory.CreateJobInstance(jobMetadata);
this.logger.LogTrace("Job created");
var sw = Stopwatch.StartNew();
await job.Invoke(jobMetadata, cancellationToken).ConfigureAwait(false);
sw.Stop();
this.logger.LogTrace("Job executed successfully in {0}", sw.Elapsed);
}
}
public async Task ShouldReturnFalseWhenJobIsCapturedBySomeonElse(
Guid jobId,
IJobMetadata jobMetadata,
string schedluerId,
CancellationToken cancellationToken)
{
// Arrange
this.jobStore.Setup(s => s.SetJobOwner(jobId, schedluerId)).ThrowsAsync(new ConcurrencyException());
// Act
var result = await this.sut.ProcessSingleJob(jobId, jobMetadata, schedluerId, cancellationToken);
// Assert
Assert.Equal(JobExecutionResult.NotStarted, result);
this.syncHelper.Verify(s => s.Release(), Times.Once);
}
public async Task ShouldReturnFalseOnAllThreadsBusy(
Guid jobId,
IJobMetadata jobMetadata,
string schedluerId,
CancellationToken cancellationToken)
{
// Arrange
this.syncHelper.Setup(s => s.WaitOne(cancellationToken)).ReturnsAsync(false);
// Act
var result = await this.sut.ProcessSingleJob(jobId, jobMetadata, schedluerId, cancellationToken);
// Assert
Assert.Equal(JobExecutionResult.NotStarted, result);
this.syncHelper.Verify(s => s.Release(), Times.Never);
}
private async Task SetJobExecutionResult(TJobKey jobId, IJobMetadata jobMetadata, JobExecutionResult jobExecutionResult)
{
try
{
await this.jobStore.FinalizeJob(jobId, jobMetadata, jobExecutionResult).ConfigureAwait(false);
this.logger.LogTrace("Job {0} state cleared", jobId);
}
catch (ConcurrencyException ex)
{
this.logger.LogWarning(ex, "Somebody has already updated job {0}: {1}", jobId, ex.Message);
}
catch (Exception ex)
{
this.logger.LogError(ex, "Clearing job {0} state failed unexpectedly: {1}", jobId, ex.Message);
}
}
public async Task ShouldRethrowExceptionOnCaptureJobFailue(
Exception ex,
Guid jobId,
IJobMetadata jobMetadata,
string schedluerId,
CancellationToken cancellationToken)
{
// Arrange
this.jobStore.Setup(s => s.SetJobOwner(jobId, schedluerId)).ThrowsAsync(ex);
// Act
var actualEx = await Assert.ThrowsAsync(ex.GetType(), () => this.sut.ProcessSingleJob(jobId, jobMetadata, schedluerId, cancellationToken));
// Assert
Assert.Same(ex, actualEx);
this.syncHelper.Verify(s => s.Release(), Times.Once);
}
private async Task <JobExecutionResult> ExecuteJobWithTimeout(IJobMetadata jobMetadata, TimeSpan timeout, CancellationToken cancellationToken)
{
using (var timeoutCts = new CancellationTokenSource(timeout))
{
using (var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(timeoutCts.Token, cancellationToken))
{
try
{
await this.ExecuteJobInternal(jobMetadata, linkedCts.Token).ConfigureAwait(false);
return(JobExecutionResult.Succeeded);
}
catch (OperationCanceledException ex) when(timeoutCts.IsCancellationRequested)
{
this.logger.LogWarning(ex, "Job execution cancelled due timeout {0}: [1}. Job will be rescheduled as usual", timeout, ex.Message);
return(JobExecutionResult.Timeouted);
}
}
}
}
public IJob CreateJobInstance(IJobMetadata jobMetadata)
{
if (jobMetadata == null)
{
throw new ArgumentNullException(nameof(jobMetadata));
}
if (jobMetadata.JobClass == null)
{
throw new ArgumentException(nameof(jobMetadata.JobClass) + " property is null", nameof(jobMetadata));
}
if (!typeof(IJob).IsAssignableFrom(jobMetadata.JobClass))
{
throw new ArgumentException($"{jobMetadata.JobClass} doesn't implement {nameof(IJob)}", nameof(jobMetadata));
}
var provider = this.scopes.Value?.Count > 0 ? this.scopes.Value.Peek().Scope.ServiceProvider : this.serviceProvider;
return((IJob)provider.GetRequiredService(jobMetadata.JobClass));
}
private async Task <JobExecutionResult> ExecuteJob(IJobMetadata jobMetadata, CancellationToken cancellationToken)
{
JobExecutionResult result;
try
{
if (jobMetadata.Timeout > TimeSpan.Zero && jobMetadata.Timeout != Timeout.InfiniteTimeSpan)
{
result = await this.ExecuteJobWithTimeout(jobMetadata, jobMetadata.Timeout.Value, cancellationToken).ConfigureAwait(false);
}
else
{
await this.ExecuteJobInternal(jobMetadata, cancellationToken).ConfigureAwait(false);
result = JobExecutionResult.Succeeded;
}
}
catch (OperationCanceledException ex) when(cancellationToken.IsCancellationRequested)
{
this.logger.LogWarning(ex, "Job execution was interrupted by external cancellation. Job will be rescheduled immediately");
return(JobExecutionResult.Cancelled);
}
catch (Exception ex)
{
this.logger.LogError(ex, "Job execution failed. Job will be rescheduled as usual. Failure Message: {0}", ex.Message);
result = JobExecutionResult.Failed;
}
try
{
jobMetadata.SetNextExecutionTime();
this.logger.LogTrace("Next execution time updated to {0}", jobMetadata.NextExecution);
}
catch (Exception ex)
{
this.logger.LogError(ex, "Calculation of next execution time failed: {0}. Job will be rescheduled immediately", ex.Message);
}
return(result);
}
public async Task <JobExecutionResult> ProcessSingleJob(
TJobKey jobId,
IJobMetadata jobMetadata,
TSchedulerKey schedluerId,
CancellationToken cancellationToken)
{
if (!await this.TryCaptureExecutionThread(cancellationToken).ConfigureAwait(false))
{
return(JobExecutionResult.NotStarted);
}
try
{
if (!await this.TrySetJobOwner(jobId, schedluerId).ConfigureAwait(false))
{
return(JobExecutionResult.NotStarted);
}
JobExecutionResult result = JobExecutionResult.Failed;
try
{
this.logger.LogInformation("Starting job {0} execution at scheduler {1}", jobId, schedluerId);
result = await this.ExecuteJob(jobMetadata, cancellationToken).ConfigureAwait(false);
this.logger.LogInformation("Job {0} execution completed: {1}", jobId, result);
return(result);
}
finally
{
await this.SetJobExecutionResult(jobId, jobMetadata, result).ConfigureAwait(false);
}
}
finally
{
this.syncHelper.Release();
}
}
