public IState ChangeState(StateChangeContext context)
{
// To ensure that job state will be changed only from one of the
// specified states, we need to ensure that other users/workers
// are not able to change the state of the job during the
// execution of this method. To guarantee this behavior, we are
// using distributed application locks and rely on fact, that
// any state transitions will be made only within a such lock.
using (context.Connection.AcquireDistributedJobLock(context.BackgroundJobId, JobLockTimeout))
{
var jobData = GetJobData(context);
if (jobData == null)
{
// The job does not exist. This may happen, because not
// all storage backends support foreign keys.
return null;
}
if (context.ExpectedStates != null && !context.ExpectedStates.Contains(jobData.State, StringComparer.OrdinalIgnoreCase))
{
return null;
}
var appliedState = context.NewState;
try
{
jobData.EnsureLoaded();
}
catch (JobLoadException ex)
{
// If the job type could not be loaded, we are unable to
// load corresponding filters, unable to process the job
// and sometimes unable to change its state (the enqueued
// state depends on the type of a job).
if (!appliedState.IgnoreJobLoadException)
{
appliedState = new FailedState(ex.InnerException)
{
Reason = String.Format(
"Can not change the state to '{0}': target method was not found.",
appliedState.Name)
};
}
}
var backgroundJob = new BackgroundJob(context.BackgroundJobId, jobData.Job, jobData.CreatedAt);
appliedState = ChangeState(context, backgroundJob, appliedState, jobData.State);
return appliedState;
}
}
public IState ChangeState(StateChangeContext context)
{
// To ensure that job state will be changed only from one of the
// specified states, we need to ensure that other users/workers
// are not able to change the state of the job during the
// execution of this method. To guarantee this behavior, we are
// using distributed application locks and rely on fact, that
// any state transitions will be made only within a such lock.
using (context.Connection.AcquireDistributedJobLock(context.BackgroundJobId, JobLockTimeout))
{
var jobData = GetJobData(context);
if (jobData == null)
{
// The job does not exist. This may happen, because not
// all storage backends support foreign keys.
return(null);
}
if (context.ExpectedStates != null && !context.ExpectedStates.Contains(jobData.State, StringComparer.OrdinalIgnoreCase))
{
return(null);
}
var appliedState = context.NewState;
try
{
jobData.EnsureLoaded();
}
catch (JobLoadException ex)
{
// If the job type could not be loaded, we are unable to
// load corresponding filters, unable to process the job
// and sometimes unable to change its state (the enqueued
// state depends on the type of a job).
if (!appliedState.IgnoreJobLoadException)
{
appliedState = new FailedState(ex.InnerException)
{
Reason = String.Format(
"Can not change the state to '{0}': target method was not found.",
appliedState.Name)
};
}
}
var backgroundJob = new BackgroundJob(context.BackgroundJobId, jobData.Job, jobData.CreatedAt);
appliedState = ChangeState(context, backgroundJob, appliedState, jobData.State);
return(appliedState);
}
}
private IState ChangeState(
StateChangeContext context, BackgroundJob backgroundJob, IState toState, string oldStateName)
{
Exception exception = null;
for (var i = 0; i < MaxStateChangeAttempts; i++)
{
try
{
using (var transaction = context.Connection.CreateWriteTransaction())
{
var applyContext = new ApplyStateContext(
context.Storage,
context.Connection,
transaction,
backgroundJob,
toState,
oldStateName);
var appliedState = _stateMachine.ApplyState(applyContext);
transaction.Commit();
return(appliedState);
}
}
catch (Exception ex)
{
exception = ex;
}
}
var failedState = new FailedState(exception)
{
Reason = $"Failed to change state to a '{toState.Name}' one due to an exception after {MaxStateChangeAttempts} retry attempts"
};
using (var transaction = context.Connection.CreateWriteTransaction())
{
_coreStateMachine.ApplyState(new ApplyStateContext(
context.Storage,
context.Connection,
transaction,
backgroundJob,
failedState,
oldStateName));
transaction.Commit();
}
return(failedState);
}
private IState ChangeState(
StateChangeContext context, BackgroundJob backgroundJob, IState toState, string oldStateName)
{
using (var transaction = context.Connection.CreateWriteTransaction())
{
var applyContext = new ApplyStateContext(
context.Storage,
context.Connection,
transaction,
backgroundJob,
toState,
oldStateName);
var appliedState = _stateMachine.ApplyState(applyContext);
transaction.Commit();
return(appliedState);
}
}
private static JobData GetJobData(StateChangeContext context)
{
var firstAttempt = true;
while (true)
{
var jobData = context.Connection.GetJobData(context.BackgroundJobId);
if (jobData != null && !String.IsNullOrEmpty(jobData.State))
{
return(jobData);
}
if (context.CancellationToken.IsCancellationRequested)
{
return(null);
}
Thread.Sleep(firstAttempt ? 0 : 100);
firstAttempt = false;
}
}
public IState ChangeState(StateChangeContext context)
{
// To ensure that job state will be changed only from one of the
// specified states, we need to ensure that other users/workers
// are not able to change the state of the job during the
// execution of this method. To guarantee this behavior, we are
// using distributed application locks and rely on fact, that
// any state transitions will be made only within a such lock.
using (context.Connection.AcquireDistributedJobLock(context.BackgroundJobId, JobLockTimeout))
{
var jobData = GetJobData(context);
if (jobData == null)
{
return(null);
}
if (context.ExpectedStates != null && !context.ExpectedStates.Contains(jobData.State, StringComparer.OrdinalIgnoreCase))
{
return(null);
}
var stateToApply = context.NewState;
try
{
jobData.EnsureLoaded();
}
catch (JobLoadException ex)
{
// This happens when Hangfire couldn't find the target method,
// serialized within a background job. There are many reasons
// for this case, including refactored code, or a missing
// assembly reference due to a mistake or erroneous deployment.
//
// The problem is that in this case we can't get any filters,
// applied at a method or a class level, and we can't proceed
// with the state change without breaking a consistent behavior:
// in some cases our filters will be applied, and in other ones
// will not.
// TODO 1.X/2.0:
// There's a problem with filters related to handling the states
// which ignore this exception, i.e. fitlers for the FailedState
// and the DeletedState, such as AutomaticRetryAttrubute filter.
//
// We should document that such a filters may not be fired, when
// we can't find a target method, and these filters should be
// applied only at the global level to get consistent results.
//
// In 2.0 we should have a special state for all the errors, when
// Hangfire doesn't know what to do, without any possibility to
// add method or class-level filters for such a state to provide
// the same behavior no matter what.
if (!stateToApply.IgnoreJobLoadException)
{
stateToApply = new FailedState(ex.InnerException)
{
Reason = $"Can not change the state to '{stateToApply.Name}': target method was not found."
};
}
}
using (var transaction = context.Connection.CreateWriteTransaction())
{
var applyContext = new ApplyStateContext(
context.Storage,
context.Connection,
transaction,
new BackgroundJob(context.BackgroundJobId, jobData.Job, jobData.CreatedAt),
stateToApply,
jobData.State);
var appliedState = _stateMachine.ApplyState(applyContext);
transaction.Commit();
return(appliedState);
}
}
}
private static JobData GetJobData(StateChangeContext context)
{
// This code was introduced as a fix for an issue, which appeared when an
// external queue implementation was used together with a non-linearizable
// storage. The problem was likely related to the SQL Azure + Azure ServiceBus
// (or RabbitMQ or so) bundle, because the READCOMMITTED_SNAPSHOT_ON setting
// is enabled by default there.
//
// Since external queueing doesn't share the linearization point with the
// storage, it is possible that a worker will pick up a background job before
// its transaction was committed. Non-linearizable read will simply return
// the NULL value instead of waiting for a transaction to be committed. With
// this code, we will make several retry attempts to handle this case to wait
// on the client side.
//
// On the other hand, we need to give up after some retry attempt, because
// we should also handle the case, when our queue and job storage became
// unsynchronized with each other due to failures, manual intervention or so.
// Otherwise we will wait forever in this cases, since And there's no way to
// make a distinction between a non-linearizable read and the storages, non-
// synchronized with each other.
//
// In recent versions, Hangfire.SqlServer uses query hints to make all the
// reads linearizable no matter what, but there may be other storages that
// still require this workaround.
// TODO 2.0:
// Eliminate the need of this timeout by placing an explicit requirement to
// storage implementations to either have a single linearization point for all
// the operations inside a transaction; or make all the reads linearizable and
// execute queueing operations after all the other ones in a transaction.
var firstAttempt = true;
while (true)
{
var jobData = context.Connection.GetJobData(context.BackgroundJobId);
// Empty state means our job wasn't moved to any state after its creation.
// Such a jobs may be created by internal logic, and those jobs have very
// special meaning, thus we shouldn't allow state changer to alter them,
// using this class (which can be used by users), leaving this logic to
// low level API only, i.e. state machine.
// TODO 1.X:
// However, we shouldn't wait for the initial state change, because in some
// cases (like in batches) it may take days. We should throw an exception
// instead, clearly indicating that such a state change is prohibited. There
// may be some issues on GitHub, related to the hanging dashboard requests
// in this case.
if (!String.IsNullOrEmpty(jobData?.State))
{
return(jobData);
}
// State change can also be requested from user's request processing logic.
// There is always a chance it will be issued against a non-existing or an
// already expired background job, and a minute wait (or whatever timeout is
// used) is completely unnecessary in this case.
//
// Since waiting is only required when a worker picks up a job, and
// cancellation tokens are used only by the Worker class, we can avoid the
// unnecessary waiting logic when no cancellation token is passed.
if (context.CancellationToken.IsCancellationRequested ||
context.CancellationToken == CancellationToken.None)
{
return(null);
}
context.CancellationToken.WaitHandle.WaitOne(firstAttempt ? 0 : 100);
firstAttempt = false;
}
}
private static JobData GetJobData(StateChangeContext context)
{
var firstAttempt = true;
while (true)
{
var jobData = context.Connection.GetJobData(context.BackgroundJobId);
if (jobData != null && !String.IsNullOrEmpty(jobData.State))
{
return jobData;
}
if (context.CancellationToken.IsCancellationRequested)
{
return null;
}
Thread.Sleep(firstAttempt ? 0 : 100);
firstAttempt = false;
}
}
private IState ChangeState(
StateChangeContext context, BackgroundJob backgroundJob, IState toState, string oldStateName)
{
using (var transaction = context.Connection.CreateWriteTransaction())
{
var applyContext = new ApplyStateContext(
context.Storage,
context.Connection,
transaction,
backgroundJob,
toState,
oldStateName);
var appliedState = _stateMachine.ApplyState(applyContext);
transaction.Commit();
return appliedState;
}
}
public IState ChangeState(StateChangeContext context)
{
// To ensure that job state will be changed only from one of the
// specified states, we need to ensure that other users/workers
// are not able to change the state of the job during the
// execution of this method. To guarantee this behavior, we are
// using distributed application locks and rely on fact, that
// any state transitions will be made only within a such lock.
using (context.Connection.AcquireDistributedJobLock(context.BackgroundJobId, JobLockTimeout))
{
var jobData = GetJobData(context);
if (jobData == null)
{
return(null);
}
if (context.ExpectedStates != null && !context.ExpectedStates.Contains(jobData.State, StringComparer.OrdinalIgnoreCase))
{
return(null);
}
var stateToApply = context.NewState;
try
{
jobData.EnsureLoaded();
}
catch (JobLoadException ex)
{
// This happens when Hangfire couldn't find the target method,
// serialized within a background job. There are many reasons
// for this case, including refactored code, or a missing
// assembly reference due to a mistake or erroneous deployment.
//
// The problem is that in this case we can't get any filters,
// applied at a method or a class level, and we can't proceed
// with the state change without breaking a consistent behavior:
// in some cases our filters will be applied, and in other ones
// will not.
if (!stateToApply.IgnoreJobLoadException)
{
stateToApply = new FailedState(ex.InnerException)
{
Reason = $"Can not change the state to '{stateToApply.Name}': target method was not found."
};
}
}
using (var transaction = context.Connection.CreateWriteTransaction())
{
var applyContext = new ApplyStateContext(
context.Storage,
context.Connection,
transaction,
new BackgroundJob(context.BackgroundJobId, jobData.Job, jobData.CreatedAt),
stateToApply,
jobData.State,
context.Profiler);
// State changing process can fail due to an exception in state filters themselves,
// and DisableFilters property will cause state machine to perform a state transition
// without calling any filters. This is required when all the other state change
// attempts failed and we need to remove such a job from the processing pipeline.
// In this case all the filters are ignored, which may lead to confusion, so it's
// highly recommended to use the DisableFilters property only when changing state
// to the FailedState.
var stateMachine = context.DisableFilters ? _innerStateMachine : _stateMachine;
var appliedState = stateMachine.ApplyState(applyContext);
transaction.Commit();
return(appliedState);
}
}
}
