public bool ChangeState(StateContext context, IState toState, string oldStateName)
{
try
{
var filterInfo = GetFilters(context.Job);
var electStateContext = new ElectStateContext(context, toState, oldStateName);
var electedState = ElectState(electStateContext, filterInfo.ElectStateFilters);
var applyStateContext = new ApplyStateContext(context, electedState, oldStateName);
ApplyState(applyStateContext, filterInfo.ApplyStateFilters);
// State transition was succeeded.
return true;
}
catch (Exception ex)
{
var failedState = new FailedState(ex)
{
Reason = "An exception occurred during the transition of job's state"
};
var applyStateContext = new ApplyStateContext(context, failedState, oldStateName);
// We should not use any state changed filters, because
// some of the could cause an exception.
ApplyState(applyStateContext, Enumerable.Empty<IApplyStateFilter>());
// State transition was failed due to exception.
return false;
}
}
/// <summary>
/// Schedules the job to run again later. See <see cref="SecondsToDelay"/>.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="retryAttempt">The count of retry attempts made so far.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState)
{
context.SetJobParameter("RetryCount", retryAttempt);
var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt));
const int maxMessageLength = 50;
var exceptionMessage = failedState.Exception.Message;
// If attempt number is less than max attempts, we should
// schedule the job to run again later.
context.CandidateState = new ScheduledState(delay)
{
Reason = String.Format(
"Retry attempt {0} of {1}: {2}",
retryAttempt,
Attempts,
exceptionMessage.Length > maxMessageLength
? exceptionMessage.Substring(0, maxMessageLength - 1) + "…"
: exceptionMessage)
};
if (LogEvents)
{
Logger.WarnException(
String.Format(
"Failed to process the job '{0}': an exception occurred. Retry attempt {1} of {2} will be performed in {3}.",
context.JobId,
retryAttempt,
Attempts,
delay),
failedState.Exception);
}
}
public RetryAttributeFacts()
{
_failedState = new FailedState(new InvalidOperationException());
_connection = new Mock<IStorageConnection>();
_context = new ElectStateContextMock();
_context.ApplyContext.BackgroundJob.Id = JobId;
_context.ApplyContext.Connection = _connection;
_context.ApplyContext.NewStateObject = _failedState;
}
public RetryAttributeFacts()
{
_failedState = new FailedState(new InvalidOperationException());
_connection = new Mock<IStorageConnection>();
_context = new ElectStateContextMock();
_context.StateContextValue.JobIdValue = JobId;
_context.StateContextValue.ConnectionValue = _connection;
_context.CandidateStateValue = _failedState;
}
public void SerializeData_ReturnsCorrectData()
{
var state = new FailedState(new Exception("Message"));
var serializedData = state.SerializeData();
Assert.Equal(JobHelper.SerializeDateTime(state.FailedAt), serializedData["FailedAt"]);
Assert.Equal("System.Exception", serializedData["ExceptionType"]);
Assert.Equal("Message", serializedData["ExceptionMessage"]);
Assert.Equal(state.Exception.ToString(), serializedData["ExceptionDetails"]);
}
/// <summary>
/// Schedules the job to run again later. See <see cref="SecondsToDelay"/>.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="retryAttempt">The count of retry attempts made so far.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState)
{
var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt));
context.SetJobParameter("RetryCount", retryAttempt);
// If attempt number is less than max attempts, we should
// schedule the job to run again later.
context.CandidateState = new ScheduledState(delay)
{
Reason = String.Format("Retry attempt {0} of {1}", retryAttempt, Attempts)
};
if (LogEvents)
{
Logger.WarnFormat(
"Failed to process the job '{0}': an exception occurred. Retry attempt {1} of {2} will be performed in {3}.",
failedState.Exception,
context.JobId,
retryAttempt,
Attempts,
delay);
}
}
private void ProcessJob(
string jobId,
IStorageConnection connection,
IJobPerformanceProcess process,
CancellationToken shutdownToken)
{
var stateMachine = _context.StateMachineFactory.Create(connection);
var processingState = new ProcessingState(_context.ServerId, _context.WorkerNumber);
if (!stateMachine.TryToChangeState(
jobId,
processingState,
new[] { EnqueuedState.StateName, ProcessingState.StateName }))
{
return;
}
// Checkpoint #3. Job is in the Processing state. However, there are
// no guarantees that it was performed. We need to re-queue it even
// it was performed to guarantee that it was performed AT LEAST once.
// It will be re-queued after the JobTimeout was expired.
IState state;
try
{
var jobData = connection.GetJobData(jobId);
jobData.EnsureLoaded();
var cancellationToken = new ServerJobCancellationToken(
jobId, connection, _context, shutdownToken);
var performContext = new PerformContext(
_context, connection, jobId, jobData.Job, jobData.CreatedAt, cancellationToken);
var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
var duration = Stopwatch.StartNew();
process.Run(performContext, jobData.Job);
duration.Stop();
state = new SucceededState((long) latency, duration.ElapsedMilliseconds);
}
catch (OperationCanceledException)
{
throw;
}
catch (JobPerformanceException ex)
{
state = new FailedState(ex.InnerException)
{
Reason = ex.Message
};
}
catch (Exception ex)
{
state = new FailedState(ex)
{
Reason = "Internal Hangfire Server exception occurred. Please, report it to Hangfire developers."
};
}
// Ignore return value, because we should not do
// anything when current state is not Processing.
stateMachine.TryToChangeState(jobId, state, new[] { ProcessingState.StateName });
}
/// <summary>
/// Transition the candidate state to the deleted state.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void TransitionToDeleted(ElectStateContext context, FailedState failedState)
{
context.CandidateState = new DeletedState
{
Reason = Attempts > 0
? "Exceeded the maximum number of retry attempts."
: "Retries were disabled for this job."
};
if (LogEvents)
{
Logger.WarnException(
$"Failed to process the job '{context.BackgroundJob.Id}': an exception occured. Job was automatically deleted because the retry attempt count exceeded {Attempts}.",
failedState.Exception);
}
}
/// <summary>
/// Schedules the job to run again later. See <see cref="SecondsToDelay"/>.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="retryAttempt">The count of retry attempts made so far.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState)
{
context.SetJobParameter("RetryCount", retryAttempt);
var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt));
const int maxMessageLength = 50;
var exceptionMessage = failedState.Exception.Message.Length > maxMessageLength
? failedState.Exception.Message.Substring(0, maxMessageLength - 1) + "…"
: failedState.Exception.Message;
// If attempt number is less than max attempts, we should
// schedule the job to run again later.
context.CandidateState = new ScheduledState(delay)
{
Reason = $"Retry attempt {retryAttempt} of {Attempts}: {exceptionMessage}"
};
if (LogEvents)
{
Logger.WarnException(
$"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred. Retry attempt {retryAttempt} of {Attempts} will be performed in {delay}.",
failedState.Exception);
}
}
public IState ChangeState(StateChangeContext context)
{
// To ensure that job state will be changed only from one of the
// specified states, we need to ensure that other users/workers
// are not able to change the state of the job during the
// execution of this method. To guarantee this behavior, we are
// using distributed application locks and rely on fact, that
// any state transitions will be made only within a such lock.
using (context.Connection.AcquireDistributedJobLock(context.BackgroundJobId, JobLockTimeout))
{
var jobData = GetJobData(context);
if (jobData == null)
{
// The job does not exist. This may happen, because not
// all storage backends support foreign keys.
return null;
}
if (context.ExpectedStates != null && !context.ExpectedStates.Contains(jobData.State, StringComparer.OrdinalIgnoreCase))
{
return null;
}
var appliedState = context.NewState;
try
{
jobData.EnsureLoaded();
}
catch (JobLoadException ex)
{
// If the job type could not be loaded, we are unable to
// load corresponding filters, unable to process the job
// and sometimes unable to change its state (the enqueued
// state depends on the type of a job).
if (!appliedState.IgnoreJobLoadException)
{
appliedState = new FailedState(ex.InnerException)
{
Reason = String.Format(
"Can not change the state to '{0}': target method was not found.",
appliedState.Name)
};
}
}
var backgroundJob = new BackgroundJob(context.BackgroundJobId, jobData.Job, jobData.CreatedAt);
appliedState = ChangeState(context, backgroundJob, appliedState, jobData.State);
return appliedState;
}
}
public bool TryToChangeState(
string jobId, IState toState, string[] fromStates)
{
if (jobId == null) throw new ArgumentNullException("jobId");
if (toState == null) throw new ArgumentNullException("toState");
if (fromStates != null && fromStates.Length == 0)
{
throw new ArgumentException("From states array should be null or non-empty.", "fromStates");
}
// To ensure that job state will be changed only from one of the
// specified states, we need to ensure that other users/workers
// are not able to change the state of the job during the
// execution of this method. To guarantee this behavior, we are
// using distributed application locks and rely on fact, that
// any state transitions will be made only within a such lock.
using (_connection.AcquireDistributedLock(
String.Format("job:{0}:state-lock", jobId),
JobLockTimeout))
{
var jobData = _connection.GetJobData(jobId);
if (jobData == null)
{
// The job does not exist. This may happen, because not
// all storage backends support foreign keys.
return false;
}
if (fromStates != null && !fromStates.Contains(jobData.State, StringComparer.OrdinalIgnoreCase))
{
return false;
}
bool loadSucceeded = true;
try
{
jobData.EnsureLoaded();
}
catch (JobLoadException ex)
{
// If the job type could not be loaded, we are unable to
// load corresponding filters, unable to process the job
// and sometimes unable to change its state (the enqueued
// state depends on the type of a job).
if (!toState.IgnoreJobLoadException)
{
toState = new FailedState(ex.InnerException)
{
Reason = String.Format(
"Can not change the state of a job to '{0}': target method was not found.",
toState.Name)
};
loadSucceeded = false;
}
}
var context = new StateContext(jobId, jobData.Job, jobData.CreatedAt, _connection);
var stateChanged = _stateChangeProcess.ChangeState(context, toState, jobData.State);
return loadSucceeded && stateChanged;
}
}
public IState ChangeState(StateChangeContext context)
{
// To ensure that job state will be changed only from one of the
// specified states, we need to ensure that other users/workers
// are not able to change the state of the job during the
// execution of this method. To guarantee this behavior, we are
// using distributed application locks and rely on fact, that
// any state transitions will be made only within a such lock.
using (context.Connection.AcquireDistributedJobLock(context.BackgroundJobId, JobLockTimeout))
{
var jobData = GetJobData(context);
if (jobData == null)
{
return(null);
}
if (context.ExpectedStates != null && !context.ExpectedStates.Contains(jobData.State, StringComparer.OrdinalIgnoreCase))
{
return(null);
}
var stateToApply = context.NewState;
try
{
jobData.EnsureLoaded();
}
catch (JobLoadException ex)
{
// This happens when Hangfire couldn't find the target method,
// serialized within a background job. There are many reasons
// for this case, including refactored code, or a missing
// assembly reference due to a mistake or erroneous deployment.
//
// The problem is that in this case we can't get any filters,
// applied at a method or a class level, and we can't proceed
// with the state change without breaking a consistent behavior:
// in some cases our filters will be applied, and in other ones
// will not.
// TODO 1.X/2.0:
// There's a problem with filters related to handling the states
// which ignore this exception, i.e. fitlers for the FailedState
// and the DeletedState, such as AutomaticRetryAttrubute filter.
//
// We should document that such a filters may not be fired, when
// we can't find a target method, and these filters should be
// applied only at the global level to get consistent results.
//
// In 2.0 we should have a special state for all the errors, when
// Hangfire doesn't know what to do, without any possibility to
// add method or class-level filters for such a state to provide
// the same behavior no matter what.
if (!stateToApply.IgnoreJobLoadException)
{
stateToApply = new FailedState(ex.InnerException)
{
Reason = $"Can not change the state to '{stateToApply.Name}': target method was not found."
};
}
}
using (var transaction = context.Connection.CreateWriteTransaction())
{
var applyContext = new ApplyStateContext(
context.Storage,
context.Connection,
transaction,
new BackgroundJob(context.BackgroundJobId, jobData.Job, jobData.CreatedAt),
stateToApply,
jobData.State,
context.Profiler);
var appliedState = _stateMachine.ApplyState(applyContext);
transaction.Commit();
return(appliedState);
}
}
}
public bool TryToChangeState(
string jobId, IState toState, string[] fromStates)
{
if (jobId == null)
{
throw new ArgumentNullException("jobId");
}
if (toState == null)
{
throw new ArgumentNullException("toState");
}
if (fromStates != null && fromStates.Length == 0)
{
throw new ArgumentException("From states array should be null or non-empty.", "fromStates");
}
// To ensure that job state will be changed only from one of the
// specified states, we need to ensure that other users/workers
// are not able to change the state of the job during the
// execution of this method. To guarantee this behavior, we are
// using distributed application locks and rely on fact, that
// any state transitions will be made only within a such lock.
using (_connection.AcquireDistributedLock(
String.Format("job:{0}:state-lock", jobId),
JobLockTimeout))
{
var jobData = _connection.GetJobData(jobId);
if (jobData == null)
{
// The job does not exist. This may happen, because not
// all storage backends support foreign keys.
return(false);
}
if (fromStates != null && !fromStates.Contains(jobData.State, StringComparer.OrdinalIgnoreCase))
{
return(false);
}
bool loadSucceeded = true;
try
{
jobData.EnsureLoaded();
}
catch (JobLoadException ex)
{
// If the job type could not be loaded, we are unable to
// load corresponding filters, unable to process the job
// and sometimes unable to change its state (the enqueued
// state depends on the type of a job).
if (!toState.IgnoreJobLoadException)
{
toState = new FailedState(ex.InnerException)
{
Reason = String.Format(
"Can not change the state of a job to '{0}': target method was not found.",
toState.Name)
};
loadSucceeded = false;
}
}
var context = new StateContext(jobId, jobData.Job, jobData.CreatedAt, _connection);
var stateChanged = _stateChangeProcess.ChangeState(context, toState, jobData.State);
return(loadSucceeded && stateChanged);
}
}
public IState ChangeState(StateChangeContext context)
{
// To ensure that job state will be changed only from one of the
// specified states, we need to ensure that other users/workers
// are not able to change the state of the job during the
// execution of this method. To guarantee this behavior, we are
// using distributed application locks and rely on fact, that
// any state transitions will be made only within a such lock.
using (context.Connection.AcquireDistributedJobLock(context.BackgroundJobId, JobLockTimeout))
{
var jobData = GetJobData(context);
if (jobData == null)
{
return(null);
}
if (context.ExpectedStates != null && !context.ExpectedStates.Contains(jobData.State, StringComparer.OrdinalIgnoreCase))
{
return(null);
}
var stateToApply = context.NewState;
try
{
jobData.EnsureLoaded();
}
catch (JobLoadException ex)
{
// This happens when Hangfire couldn't find the target method,
// serialized within a background job. There are many reasons
// for this case, including refactored code, or a missing
// assembly reference due to a mistake or erroneous deployment.
//
// The problem is that in this case we can't get any filters,
// applied at a method or a class level, and we can't proceed
// with the state change without breaking a consistent behavior:
// in some cases our filters will be applied, and in other ones
// will not.
if (!stateToApply.IgnoreJobLoadException)
{
stateToApply = new FailedState(ex.InnerException)
{
Reason = $"Can not change the state to '{stateToApply.Name}': target method was not found."
};
}
}
using (var transaction = context.Connection.CreateWriteTransaction())
{
var applyContext = new ApplyStateContext(
context.Storage,
context.Connection,
transaction,
new BackgroundJob(context.BackgroundJobId, jobData.Job, jobData.CreatedAt),
stateToApply,
jobData.State,
context.Profiler);
// State changing process can fail due to an exception in state filters themselves,
// and DisableFilters property will cause state machine to perform a state transition
// without calling any filters. This is required when all the other state change
// attempts failed and we need to remove such a job from the processing pipeline.
// In this case all the filters are ignored, which may lead to confusion, so it's
// highly recommended to use the DisableFilters property only when changing state
// to the FailedState.
var stateMachine = context.DisableFilters ? _innerStateMachine : _stateMachine;
var appliedState = stateMachine.ApplyState(applyContext);
transaction.Commit();
return(appliedState);
}
}
}
public void IgnoreExceptions_ReturnsFalse()
{
var state = new FailedState(new Exception());
Assert.False(state.IgnoreJobLoadException);
}
public void IsFinal_ReturnsFalse()
{
var state = new FailedState(new Exception());
Assert.False(state.IsFinal);
}
/// <summary>
/// Transition the candidate state to the deleted state.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void TransitionToDeleted(ElectStateContext context, FailedState failedState)
{
context.CandidateState = new DeletedState
{
Reason = "Exceeded the maximum number of retry attempts."
};
if (LogEvents)
{
Logger.WarnException(
String.Format(
"Failed to process the job '{0}': an exception occured. Job was automatically deleted because the retry attempt count exceeded {1}.",
context.BackgroundJob.Id,
Attempts),
failedState.Exception);
}
}
/// <summary>
/// Transition the candidate state to the deleted state.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void TransitionToDeleted(ElectStateContext context, FailedState failedState)
{
context.CandidateState = new DeletedState
{
Reason = String.Format("Automatic deletion after retry count exceeded {0}", Attempts)
};
if (LogEvents)
{
Logger.WarnException(
String.Format(
"Failed to process the job '{0}': an exception occured. Job was automatically deleted because the retry attempt count exceeded {1}.",
context.JobId,
Attempts),
failedState.Exception);
}
}
private static void ExecuteContinuationsIfExist(ElectStateContext context)
{
// The following lines are being executed inside a distributed job lock,
// so it is safe to get continuation list here.
var continuations = GetContinuations(context.Connection, context.JobId);
var nextStates = new Dictionary<string, IState>();
// Getting continuation data for all continuations – state they are waiting
// for and their next state.
foreach (var continuation in continuations)
{
if (String.IsNullOrWhiteSpace(continuation.JobId)) continue;
var currentState = GetContinuaionState(context, continuation.JobId, ContinuationStateFetchTimeout);
if (currentState == null)
{
continue;
}
// All continuations should be in the awaiting state. If someone changed
// the state of a continuation, we should simply skip it.
if (currentState.Name != AwaitingState.StateName) continue;
if (continuation.Options.HasFlag(JobContinuationOptions.OnlyOnSucceededState) &&
context.CandidateState.Name != SucceededState.StateName)
{
nextStates.Add(continuation.JobId, new DeletedState { Reason = "Missed continuation" });
continue;
}
IState nextState;
try
{
nextState = JsonConvert.DeserializeObject<IState>(
currentState.Data["NextState"],
new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Objects });
}
catch (Exception ex)
{
nextState = new FailedState(ex)
{
Reason = "Can not start the continuation due to de-serialization error."
};
}
nextStates.Add(continuation.JobId, nextState);
}
foreach (var tuple in nextStates)
{
context.StateMachine.ChangeState(tuple.Key, tuple.Value, new[] { AwaitingState.StateName });
}
}
public void StateName_IsCorrect()
{
var state = new FailedState(new Exception());
Assert.Equal(FailedState.StateName, state.Name);
}
