private void ProcessJob(
string jobId,
IStorageConnection connection,
IJobPerformanceProcess process,
CancellationToken shutdownToken)
{
var stateMachine = _context.StateMachineFactory.Create(connection);
var processingState = new ProcessingState(_context.ServerId, _context.WorkerNumber);
if (!stateMachine.TryToChangeState(
jobId,
processingState,
new[] { EnqueuedState.StateName, ProcessingState.StateName }))
{
return;
}
// Checkpoint #3. Job is in the Processing state. However, there are
// no guarantees that it was performed. We need to re-queue it even
// it was performed to guarantee that it was performed AT LEAST once.
// It will be re-queued after the JobTimeout was expired.
IState state;
try
{
var jobData = connection.GetJobData(jobId);
jobData.EnsureLoaded();
var cancellationToken = new ServerJobCancellationToken(
jobId, connection, _context, shutdownToken);
var performContext = new PerformContext(
_context, connection, jobId, jobData.Job, jobData.CreatedAt, cancellationToken);
var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
var duration = Stopwatch.StartNew();
process.Run(performContext, jobData.Job);
duration.Stop();
state = new SucceededState((long) latency, duration.ElapsedMilliseconds);
}
catch (OperationCanceledException)
{
throw;
}
catch (JobPerformanceException ex)
{
state = new FailedState(ex.InnerException)
{
Reason = ex.Message
};
}
catch (Exception ex)
{
state = new FailedState(ex)
{
Reason = "Internal Hangfire Server exception occurred. Please, report it to Hangfire developers."
};
}
// Ignore return value, because we should not do
// anything when current state is not Processing.
stateMachine.TryToChangeState(jobId, state, new[] { ProcessingState.StateName });
}
private IState PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId)
{
try
{
var jobData = connection.GetJobData(jobId);
if (jobData == null)
{
// Job expired just after moving to a processing state. This is an
// unreal scenario, but shit happens. Returning null instead of throwing
// an exception and rescuing from en-queueing a poisoned jobId back
// to a queue.
return null;
}
jobData.EnsureLoaded();
var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt);
var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, _workerId, context.CancellationToken);
var performContext = new PerformContext(connection, backgroundJob, jobToken);
var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
var duration = Stopwatch.StartNew();
var result = _performer.Perform(performContext);
duration.Stop();
return new SucceededState(result, (long) latency, duration.ElapsedMilliseconds);
}
catch (OperationCanceledException)
{
throw;
}
catch (JobPerformanceException ex)
{
return new FailedState(ex.InnerException)
{
Reason = ex.Message
};
}
catch (Exception ex)
{
return new FailedState(ex)
{
Reason = "An exception occurred during processing of a background job."
};
}
}
public void Execute(CancellationToken cancellationToken)
{
using (var connection = _storage.GetConnection())
using (var fetchedJob = connection.FetchNextJob(_context.Queues, cancellationToken))
{
try
{
var stateMachine = _stateMachineFactory.Create(connection);
using (var timeoutCts = new CancellationTokenSource(JobInitializationWaitTimeout))
using (var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(
cancellationToken,
timeoutCts.Token))
{
var processingState = new ProcessingState(_context.ServerId, _context.WorkerNumber);
if (!stateMachine.ChangeState(
fetchedJob.JobId,
processingState,
new[] { EnqueuedState.StateName, ProcessingState.StateName },
linkedCts.Token))
{
// We should re-queue a job identifier only when graceful shutdown
// initiated.
cancellationToken.ThrowIfCancellationRequested();
// We should forget a job in a wrong state, or when timeout exceeded.
fetchedJob.RemoveFromQueue();
return;
}
}
// Checkpoint #3. Job is in the Processing state. However, there are
// no guarantees that it was performed. We need to re-queue it even
// it was performed to guarantee that it was performed AT LEAST once.
// It will be re-queued after the JobTimeout was expired.
var jobCancellationToken = new ServerJobCancellationToken(
fetchedJob.JobId, connection, _context, cancellationToken);
var state = PerformJob(fetchedJob.JobId, connection, jobCancellationToken);
if (state != null)
{
// Ignore return value, because we should not do anything when current state is not Processing.
stateMachine.ChangeState(fetchedJob.JobId, state, new[] { ProcessingState.StateName });
}
// Checkpoint #4. The job was performed, and it is in the one
// of the explicit states (Succeeded, Scheduled and so on).
// It should not be re-queued, but we still need to remove its
// processing information.
fetchedJob.RemoveFromQueue();
// Success point. No things must be done after previous command
// was succeeded.
}
catch (JobAbortedException)
{
fetchedJob.RemoveFromQueue();
}
catch (Exception ex)
{
Logger.DebugException("An exception occurred while processing a job. It will be re-queued.", ex);
fetchedJob.Requeue();
throw;
}
}
}
