private IState TryChangeState(
BackgroundProcessContext context,
IStorageConnection connection,
IFetchedJob fetchedJob,
IState state,
string[] expectedStates,
CancellationToken initializeToken,
CancellationToken abortToken)
{
Exception exception = null;
abortToken.ThrowIfCancellationRequested();
for (var retryAttempt = 0; retryAttempt < _maxStateChangeAttempts; retryAttempt++)
{
try
{
return(_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
state,
expectedStates,
disableFilters: false,
initializeToken,
_profiler)));
}
catch (Exception ex)
{
_logger.DebugException(
$"State change attempt {retryAttempt + 1} of {_maxStateChangeAttempts} failed due to an error, see inner exception for details",
ex);
exception = ex;
}
abortToken.Wait(TimeSpan.FromSeconds(retryAttempt));
abortToken.ThrowIfCancellationRequested();
}
_logger.ErrorException(
$"{_maxStateChangeAttempts} state change attempt(s) failed due to an exception, moving job to the FailedState",
exception);
return(_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
new FailedState(exception)
{
Reason = $"Failed to change state to a '{state.Name}' one due to an exception after {_maxStateChangeAttempts} retry attempts"
},
expectedStates,
disableFilters: true,
initializeToken,
_profiler)));
}
private IState TryChangeState(
BackgroundProcessContext context,
IStorageConnection connection,
IFetchedJob fetchedJob,
IState state,
string[] expectedStates,
CancellationToken cancellationToken)
{
Exception exception = null;
for (var retryAttempt = 0; retryAttempt < MaxStateChangeAttempts; retryAttempt++)
{
try
{
return(_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
state,
expectedStates,
cancellationToken,
_profiler)));
}
catch (Exception ex)
{
_logger.DebugException(
String.Format("State change attempt {0} of {1} failed due to an error, see inner exception for details", retryAttempt + 1, MaxStateChangeAttempts),
ex);
exception = ex;
}
context.CancellationToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(retryAttempt));
context.CancellationToken.ThrowIfCancellationRequested();
}
return(_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
new FailedState(exception)
{
Reason = $"Failed to change state to a '{state.Name}' one due to an exception after {MaxStateChangeAttempts} retry attempts"
},
expectedStates,
cancellationToken,
_profiler)));
}
public bool ChangeState(string jobId, IState state, string expectedState)
{
if (jobId == null)
{
throw new ArgumentNullException(nameof(jobId));
}
if (state == null)
{
throw new ArgumentNullException(nameof(state));
}
try
{
using (var connection = _storage.GetConnection())
{
var appliedState = _stateChanger.ChangeState(new StateChangeContext(
_storage,
connection,
jobId,
state,
expectedState != null ? new[] { expectedState } : null));
return(appliedState != null && appliedState.Name.Equals(state.Name, StringComparison.OrdinalIgnoreCase));
}
}
catch (Exception ex)
{
throw new BackgroundJobClientException("State change of a background job failed. See inner exception for details", ex);
}
}
public void OnStateApplied(ApplyStateContext context, IWriteOnlyTransaction transaction)
{
if (context.NewState is AtomRunningState state)
{
var subatomIds = context.Connection.GetAllItemsFromSet(Atom.GenerateSubAtomKeys(state.AtomId));
foreach (var subatomId in subatomIds)
{
var subatomStateData = context.Connection.GetStateData(subatomId);
var subatomInitialState = subatomStateData.Data.GetByKey(nameof(SubAtomCreatedState.NextState));
if (subatomInitialState == null)
{
throw new InvalidOperationException("Next state is NULL.");
}
var nextState = JsonUtils.Deserialize <IState>(subatomInitialState);
_stateChanger.ChangeState(
new StateChangeContext(
context.Storage,
context.Connection,
subatomId,
nextState));
}
}
}
private void EnqueueBackgroundJob(BackgroundProcessContext context, IStorageConnection connection, string jobId)
{
var appliedState = _stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
jobId,
new EnqueuedState {
Reason = $"Triggered by {ToString()}"
},
new [] { ScheduledState.StateName },
CancellationToken.None,
_profiler));
if (appliedState == null)
{
// When a background job with the given id does not exist, we should
// remove its id from a schedule manually. This may happen when someone
// modifies a storage bypassing Hangfire API.
using (var transaction = connection.CreateWriteTransaction())
{
transaction.RemoveFromSet("schedule", jobId);
transaction.Commit();
}
}
}
private void DeleteAsAtom(JobStorageConnection jsc, string atomId)
{
var subatomIds = jsc.GetAllItemsFromSet(Atom.GenerateSubAtomKeys(atomId));
foreach (var subatomId in subatomIds)
{
var context = new StateChangeContext(JobStorage.Current, jsc, subatomId, new DeletedState());
_stateChanger.ChangeState(context);
}
}
private bool EnqueueNextScheduledJob(BackgroundProcessContext context)
{
return(UseConnectionDistributedLock(context.Storage, connection =>
{
var timestamp = JobHelper.ToTimestamp(DateTime.UtcNow);
// TODO: it is very slow. Add batching.
var jobId = connection.GetFirstByLowestScoreFromSet("schedule", 0, timestamp);
if (jobId == null)
{
// No more scheduled jobs pending.
return false;
}
var appliedState = _stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
jobId,
new EnqueuedState {
Reason = $"Triggered by {ToString()}"
},
new [] { ScheduledState.StateName },
CancellationToken.None,
_profiler));
if (appliedState == null)
{
// When a background job with the given id does not exist, we should
// remove its id from a schedule manually. This may happen when someone
// modifies a storage bypassing Hangfire API.
using (var transaction = connection.CreateWriteTransaction())
{
transaction.RemoveFromSet("schedule", jobId);
transaction.Commit();
}
}
return true;
}));
}
private bool EnqueueNextScheduledJob(BackgroundProcessContext context)
{
using (var connection = context.Storage.GetConnection())
using (connection.AcquireDistributedLock("locks:schedulepoller", DefaultLockTimeout))
{
var timestamp = JobHelper.ToTimestamp(DateTime.UtcNow);
// TODO: it is very slow. Add batching.
var jobId = connection.GetFirstByLowestScoreFromSet("schedule", 0, timestamp);
if (jobId == null)
{
// No more scheduled jobs pending.
return(false);
}
var appliedState = _stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
jobId,
new EnqueuedState {
Reason = String.Format("Triggered by {0}", ToString())
},
ScheduledState.StateName));
if (appliedState == null)
{
// When a background job with the given id does not exist, we should
// remove its id from a schedule manually. This may happen when someone
// modifies a storage bypassing Hangfire API.
using (var transaction = connection.CreateWriteTransaction())
{
transaction.RemoveFromSet("schedule", jobId);
transaction.Commit();
}
}
return(true);
}
}
private void EnqueueBackgroundJob(BackgroundProcessContext context, IStorageConnection connection, string jobId)
{
Exception exception = null;
for (var retryAttempt = 0; retryAttempt < MaxStateChangeAttempts; retryAttempt++)
{
try
{
var appliedState = _stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
jobId,
new EnqueuedState {
Reason = $"Triggered by {ToString()}"
},
new [] { ScheduledState.StateName },
disableFilters: false,
context.StoppingToken,
_profiler));
if (appliedState == null && connection.GetJobData(jobId) == null)
{
// When a background job with the given id does not exist, we should
// remove its id from a schedule manually. This may happen when someone
// modifies a storage bypassing Hangfire API.
using (var transaction = connection.CreateWriteTransaction())
{
transaction.RemoveFromSet("schedule", jobId);
transaction.Commit();
}
}
return;
}
catch (Exception ex)
{
_logger.DebugException(
$"State change attempt {retryAttempt + 1} of {MaxStateChangeAttempts} failed due to an error, see inner exception for details",
ex);
exception = ex;
}
context.StoppingToken.Wait(TimeSpan.FromSeconds(retryAttempt));
context.StoppingToken.ThrowIfCancellationRequested();
}
_logger.ErrorException(
$"{MaxStateChangeAttempts} state change attempt(s) failed due to an exception, moving job to the FailedState",
exception);
// When exception occurs, it's essential to remove a background job identifier from the schedule,
// because otherwise delayed job scheduler will fetch such a failing job identifier again and again
// and will be unable to make any progress. Any successful state change will cause that identifier
// to be removed from the schedule.
_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
jobId,
new FailedState(exception)
{
Reason = $"Failed to change state to the '{EnqueuedState.StateName}' one due to an exception after {MaxStateChangeAttempts} retry attempts"
},
new[] { ScheduledState.StateName },
disableFilters: true,
context.StoppingToken,
_profiler));
}
public IState ChangeState(StateChangeContext context)
{
Console.WriteLine($"ChangeState {context.BackgroundJobId} to {context.NewState}");
return(_inner.ChangeState(context));
}
/// <inheritdoc />
public void Execute(BackgroundProcessContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
using (var connection = context.Storage.GetConnection())
using (var fetchedJob = connection.FetchNextJob(_queues, context.CancellationToken))
{
context.CancellationToken.ThrowIfCancellationRequested();
try
{
using (var timeoutCts = new CancellationTokenSource(JobInitializationWaitTimeout))
using (var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(
context.CancellationToken,
timeoutCts.Token))
{
var processingState = new ProcessingState(context.ServerId, _workerId);
var appliedState = _stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
processingState,
new[] { EnqueuedState.StateName, ProcessingState.StateName },
linkedCts.Token));
// Cancel job processing if the job could not be loaded, was not in the initial state expected
// or if a job filter changed the state to something other than processing state
if (appliedState == null || !appliedState.Name.Equals(ProcessingState.StateName, StringComparison.OrdinalIgnoreCase))
{
// We should re-queue a job identifier only when graceful shutdown
// initiated.
context.CancellationToken.ThrowIfCancellationRequested();
// We should forget a job in a wrong state, or when timeout exceeded.
fetchedJob.RemoveFromQueue();
return;
}
}
// Checkpoint #3. Job is in the Processing state. However, there are
// no guarantees that it was performed. We need to re-queue it even
// it was performed to guarantee that it was performed AT LEAST once.
// It will be re-queued after the JobTimeout was expired.
var state = PerformJob(context, connection, fetchedJob.JobId);
if (state != null)
{
// Ignore return value, because we should not do anything when current state is not Processing.
_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
state,
ProcessingState.StateName));
// TODO: Log error, when applied state is FailedState
}
// Checkpoint #4. The job was performed, and it is in the one
// of the explicit states (Succeeded, Scheduled and so on).
// It should not be re-queued, but we still need to remove its
// processing information.
fetchedJob.RemoveFromQueue();
// Success point. No things must be done after previous command
// was succeeded.
}
catch (Exception ex)
{
if (context.IsShutdownRequested)
{
Logger.Info(String.Format(
"Shutdown request requested while processing background job '{0}'. It will be re-queued.",
fetchedJob.JobId));
}
else
{
Logger.DebugException("An exception occurred while processing a job. It will be re-queued.", ex);
}
Requeue(fetchedJob);
throw;
}
}
}
public IState ChangeState(StateChangeContext context)
{
return(_inner.ChangeState(context));
}
