public bool ChangeState(StateContext context, IState toState, string oldStateName)
{
try
{
var filterInfo = GetFilters(context.Job);
var electStateContext = new ElectStateContext(context, toState, oldStateName);
var electedState = ElectState(electStateContext, filterInfo.ElectStateFilters);
var applyStateContext = new ApplyStateContext(context, electedState, oldStateName);
ApplyState(applyStateContext, filterInfo.ApplyStateFilters);
// State transition was succeeded.
return true;
}
catch (Exception ex)
{
var failedState = new FailedState(ex)
{
Reason = "An exception occurred during the transition of job's state"
};
var applyStateContext = new ApplyStateContext(context, failedState, oldStateName);
// We should not use any state changed filters, because
// some of the could cause an exception.
ApplyState(applyStateContext, Enumerable.Empty<IApplyStateFilter>());
// State transition was failed due to exception.
return false;
}
}
public void OnStateElection(ElectStateContext context)
{
var failedState = context.CandidateState as FailedState;
if (failedState != null)
{
int retryCount = context.GetJobParameter <int>("RetryCount");
if (retryCount == 3)
{
var eventGridEventHandler = (IEventGridEventHandler)activator.ActivateJob(typeof(IEventGridEventHandler));
SubscriptionModel subscription = (SubscriptionModel)context.BackgroundJob.Job.Args[0];
EventModel @event = (EventModel)context.BackgroundJob.Job.Args[1];
var errorEvent = new EventModel
{
Data = @event,
EventType = "events.deliveryFailed",
Source = @event.Source,
Extensions = new Dictionary <string, object>
{
{ nameof(subscription), subscription }
}
};
eventGridEventHandler.HandleAsync(subscription.Topic, errorEvent).Wait();
}
//job has failed, stop application here
}
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState is EnqueuedState enqueuedState)
{
enqueuedState.Queue = String.Format(Queue, context.BackgroundJob.Job.Args.ToArray());
}
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState.Name == SucceededState.StateName)
{
context.Transaction.IncrementCounter(
String.Format(
"stats:succeeded:{0}",
DateTime.UtcNow.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)),
DateTime.UtcNow.AddMonths(1) - DateTime.UtcNow);
context.Transaction.IncrementCounter(
String.Format(
"stats:succeeded:{0}",
DateTime.UtcNow.ToString("yyyy-MM-dd-HH")),
TimeSpan.FromDays(1));
}
else if (context.CandidateState.Name == FailedState.StateName)
{
context.Transaction.IncrementCounter(
String.Format(
"stats:failed:{0}",
DateTime.UtcNow.ToString("yyyy-MM-dd")),
DateTime.UtcNow.AddMonths(1) - DateTime.UtcNow);
context.Transaction.IncrementCounter(
String.Format(
"stats:failed:{0}",
DateTime.UtcNow.ToString("yyyy-MM-dd-HH")),
TimeSpan.FromDays(1));
}
}
public void OnStateElection(ElectStateContext context)
{
var failedState = context.CandidateState as FailedState;
if (failedState == null)
{
// This filter accepts only failed job state.
return;
}
var retryAttempt = context.GetJobParameter<int>("RetryCount") + 1;
if (retryAttempt <= Attempts)
{
ScheduleAgainLater(context, retryAttempt, failedState);
}
else if (retryAttempt > Attempts && OnAttemptsExceeded == AttemptsExceededAction.Delete)
{
TransitionToDeleted(context, failedState);
}
else
{
if (LogEvents)
{
Logger.ErrorException(
String.Format(
"Failed to process the job '{0}': an exception occurred.",
context.JobId),
failedState.Exception);
}
}
}
private void SetState(ElectStateContext context, MonitorJobStatusDto dto)
{
var actionName = context.BackgroundJob.Job.Method.GetFullActionName();
var key = context.Connection.GetMonitorStateKey(DateTime.Today, actionName);
context.Connection.SetMonitorState(key, dto);
}
/// <summary>
/// Schedules the job to run again later. See <see cref="SecondsToDelay"/>.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="retryAttempt">The count of retry attempts made so far.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState)
{
context.SetJobParameter("RetryCount", retryAttempt);
var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt));
const int maxMessageLength = 50;
var exceptionMessage = failedState.Exception.Message;
// If attempt number is less than max attempts, we should
// schedule the job to run again later.
context.CandidateState = new ScheduledState(delay)
{
Reason = String.Format(
"Retry attempt {0} of {1}: {2}",
retryAttempt,
Attempts,
exceptionMessage.Length > maxMessageLength
? exceptionMessage.Substring(0, maxMessageLength - 1) + "…"
: exceptionMessage)
};
if (LogEvents)
{
Logger.WarnException(
String.Format(
"Failed to process the job '{0}': an exception occurred. Retry attempt {1} of {2} will be performed in {3}.",
context.JobId,
retryAttempt,
Attempts,
delay),
failedState.Exception);
}
}
/// <summary>
/// Schedules the job to run again later. See <see cref="SecondsToDelay"/>.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="retryAttempt">The count of retry attempts made so far.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState)
{
context.SetJobParameter("RetryCount", retryAttempt);
var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt));
const int maxMessageLength = 50;
var exceptionMessage = failedState.Exception.Message.Length > maxMessageLength
? failedState.Exception.Message.Substring(0, maxMessageLength - 1) + "…"
: failedState.Exception.Message;
// If attempt number is less than max attempts, we should
// schedule the job to run again later.
context.CandidateState = new ScheduledState(delay, context.BackgroundJob.Job.QueueName)
{
Reason = $"Retry attempt {retryAttempt} of {Attempts}: {exceptionMessage}"
};
if (LogEvents)
{
Logger.WarnException(
$"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred. Retry attempt {retryAttempt} of {Attempts} will be performed in {delay}.",
failedState.Exception);
}
}
public void OnStateElection(ElectStateContext context)
{
var failedState = context.CandidateState as FailedState;
if (failedState != null)
{
int retryCount = context.GetJobParameter <int>("RetryCount");
if (retryCount == 3)
{
var eventGridEventHandler = (IEventGridEventHandler)activator.ActivateJob(typeof(IEventGridEventHandler));
// var httpClientFactory = (IHttpClientFactory)serviceProvider.GetService(typeof(IHttpClientFactory));
var subscriber = (Subscription)context.BackgroundJob.Job.Args[0];
var @event = (EventModel)context.BackgroundJob.Job.Args[1];
var errorEvent = new EventModel
{
Data = @event,
EventType = "event.failed",
Source = @event.Source
};
eventGridEventHandler.HandleAsync(subscriber.Topic, errorEvent).Wait();
}
//job has failed, stop application here
}
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState is FailedState failedState)
{
Logger.WarnFormat($"Job `{context.BackgroundJob.Id}` has been failed due to an exception `{failedState.Exception}`");
}
}
/// <summary>
/// Schedules the job to run again later. See <see cref="SecondsToDelay"/>.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="retryAttempt">The count of retry attempts made so far.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState)
{
var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt));
context.SetJobParameter("RetryCount", retryAttempt);
// If attempt number is less than max attempts, we should
// schedule the job to run again later.
context.CandidateState = new ScheduledState(delay)
{
Reason = String.Format("Retry attempt {0} of {1}", retryAttempt, Attempts)
};
if (LogEvents)
{
Logger.WarnException(
String.Format(
"Failed to process the job '{0}': an exception occurred. Retry attempt {1} of {2} will be performed in {3}.",
context.JobId,
retryAttempt,
Attempts,
delay),
failedState.Exception);
}
}
/// <inheritdoc />
public void OnStateElection(ElectStateContext context)
{
var failedState = context.CandidateState as FailedState;
if (failedState == null)
{
// This filter accepts only failed job state.
return;
}
var retryAttempt = context.GetJobParameter <int>("RetryCount") + 1;
if (retryAttempt <= Attempts)
{
ScheduleAgainLater(context, retryAttempt, failedState);
}
else if (retryAttempt > Attempts && OnAttemptsExceeded == AttemptsExceededAction.Delete)
{
TransitionToDeleted(context, failedState);
}
else
{
if (LogEvents)
{
Logger.ErrorException(
$"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred.",
failedState.Exception);
}
}
}
private static IState ElectState(
ElectStateContext context,
IEnumerable<IElectStateFilter> filters)
{
var statesToAppend = new List<IState>();
foreach (var filter in filters)
{
var oldState = context.CandidateState;
filter.OnStateElection(context);
if (oldState != context.CandidateState)
{
statesToAppend.Add(oldState);
}
}
if (statesToAppend.Count > 0)
{
using (var transaction = context.Connection.CreateWriteTransaction())
{
foreach (var state in statesToAppend)
{
transaction.AddJobState(context.JobId, state);
}
transaction.Commit();
}
}
return context.CandidateState;
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState.Name == SucceededState.StateName)
{
context.Transaction.IncrementCounter(
String.Format(
"stats:succeeded:{0}",
DateTime.UtcNow.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)),
DateTime.UtcNow.AddMonths(1) - DateTime.UtcNow);
context.Transaction.IncrementCounter(
String.Format(
"stats:succeeded:{0}",
DateTime.UtcNow.ToString("yyyy-MM-dd-HH")),
TimeSpan.FromDays(1));
}
else if (context.CandidateState.Name == FailedState.StateName)
{
context.Transaction.IncrementCounter(
String.Format(
"stats:failed:{0}",
DateTime.UtcNow.ToString("yyyy-MM-dd")),
DateTime.UtcNow.AddMonths(1) - DateTime.UtcNow);
context.Transaction.IncrementCounter(
String.Format(
"stats:failed:{0}",
DateTime.UtcNow.ToString("yyyy-MM-dd-HH")),
TimeSpan.FromDays(1));
}
}
public void ElectState(IStorageConnection connection, ElectStateContext context)
{
var filterInfo = GetFilters(context.Job);
foreach (var filter in filterInfo.ElectStateFilters)
{
filter.OnStateElection(context);
}
}
public void OnStateElection(ElectStateContext context)
{
var enqueuedState = context.CandidateState as EnqueuedState;
if (enqueuedState != null)
{
enqueuedState.Queue = Queue;
}
}
public void OnStateElection(ElectStateContext context)
{
var enqueuedState = context.CandidateState as EnqueuedState;
if (enqueuedState != null)
{
enqueuedState.Queue = (context.BackgroundJob.Job.Args[ParameterIndex] as string).Replace("\"", string.Empty);
}
}
private void AddContinuation(ElectStateContext context, AwaitingState awaitingState)
{
var connection = context.Connection;
var parentId = awaitingState.ParentId;
// We store continuations as a json array in a job parameter. Since there
// is no way to add a continuation in an atomic way, we are placing a
// distributed lock on parent job to prevent race conditions, when
// multiple threads add continuation to the same parent job.
using (connection.AcquireDistributedJobLock(parentId, AddJobLockTimeout))
{
var continuations = GetContinuations(connection, parentId);
// Continuation may be already added. This may happen, when outer transaction
// was failed after adding a continuation last time, since the addition is
// performed outside of an outer transaction.
if (!continuations.Exists(x => x.JobId == context.BackgroundJob.Id))
{
continuations.Add(new Continuation {
JobId = context.BackgroundJob.Id, Options = awaitingState.Options
});
// Set continuation only after ensuring that parent job still
// exists. Otherwise we could create add non-expiring (garbage)
// parameter for the parent job.
SetContinuations(connection, parentId, continuations);
}
var jobData = connection.GetJobData(parentId);
if (jobData == null)
{
// When we try to add a continuation for a removed job,
// the system should throw an exception instead of creating
// corrupted state.
throw new InvalidOperationException(
$"Can not add a continuation: parent background job '{parentId}' does not exist.");
}
var currentState = connection.GetStateData(parentId);
if (currentState != null && _knownFinalStates.Contains(currentState.Name))
{
var startImmediately = !awaitingState.Options.HasFlag(JobContinuationOptions.OnlyOnSucceededState) ||
currentState.Name == SucceededState.StateName;
if (_pushResults && currentState.Data.TryGetValue("Result", out var antecedentResult))
{
context.Connection.SetJobParameter(context.BackgroundJob.Id, "AntecedentResult", antecedentResult);
}
context.CandidateState = startImmediately
? awaitingState.NextState
: new DeletedState {
Reason = "Continuation condition was not met"
};
}
}
}
public void OnStateElection(ElectStateContext context)
{
var enqueuedState = context.CandidateState as EnqueuedState;
if (enqueuedState != null)
{
enqueuedState.Queue = Queue;
}
}
public void OnStateElection(ElectStateContext context)
{
var jobServices = new JobServices();
if (context.CandidateState is FailedState failedState)
{
jobServices.UpdateJob(context.BackgroundJob.Id, Status.Failed, failedState.Exception.Message);
}
}
/// <inheritdoc />
public void OnStateElection(ElectStateContext context)
{
var contextCandidateState = context.CandidateState as SucceededState;
if (contextCandidateState?.Result is Result result)
{
result.OnFailure(error => SetFailedState(context, error));
}
}
public void OnStateElection(ElectStateContext context)
{
var jobService = new JobServices();
var failedState = context.CandidateState as FailedState;
if (failedState != null)
{
jobService.UpdateJob(context.BackgroundJob.Id, Status.Failed, failedState.Exception.Message);
}
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState is FailedState failedState)
{
logger.WarnFormat(
"[OnStateElection] BackgroundJob.Id `{0}` Failed，Exception `{1}`",
context.BackgroundJob.Id,
failedState.Exception.ToString());
}
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState is FailedState failedState)
{
Logger.WarnFormat(
"IElectStateFilter: Job `{0}` has been failed due to an exception `{1}`",
context.BackgroundJob.Id,
failedState.Exception);
}
}
public void OnStateElection(ElectStateContext context)
{
var failedState = context.CandidateState as FailedState;
if (failedState != null)
{
Logger.WarnFormat(
$"Job `{context.BackgroundJob?.Id}` has been failed due to an exception `{failedState.Exception?.Message}`");
}
}
public void OnStateElection(ElectStateContext context)
{
var jobService = new JobServices();
var failedState = context.CandidateState as FailedState;
if (failedState != null)
{
jobService.UpdateJob(context.BackgroundJob.Id, Status.Failed, failedState.Exception.Message);
}
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState is EnqueuedState enqueuedState)
{
if (context.BackgroundJob.Job.Args.Any() && context.BackgroundJob.Job.Args[0] is ISpecifiedQueueJob specifiedQueueJob)
{
enqueuedState.Queue = specifiedQueueJob.Queue;
}
}
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState is FailedState failedState)
{
logger.Warn(
"任务 `{0}` 执行失败，异常为 `{1}`",
context.BackgroundJob.Id,
failedState.Exception);
}
}
/// <inheritdoc />
public void OnStateElection(ElectStateContext context)
{
var jobdta = context.BackgroundJob.Job.Args.FirstOrDefault();
if (jobdta == null)
{
return;
}
var httpjob = jobdta as HttpJobItem;
if (httpjob != null && !httpjob.EnableRetry)
{
return;
}
var failedState = context.CandidateState as FailedState;
if (failedState == null)
{
// This filter accepts only failed job state.
return;
}
//var job = context.BackgroundJob.Job.Args.FirstOrDefault();
var retryAttempt = context.GetJobParameter <int>("RetryCount") + 1;
if (retryAttempt <= Attempts)
{
ScheduleAgainLater(context, retryAttempt, failedState);
return;
}
try
{
//删除Runtime数据 代表的是需要重试 但是已经重试到最大次数了
var hashKey = CodingUtil.MD5(context.BackgroundJob.Id + ".runtime");
context.Transaction.RemoveHash(hashKey);
}
catch (Exception)
{
//ignore
}
if (retryAttempt > Attempts && OnAttemptsExceeded == AttemptsExceededAction.Delete)
{
TransitionToDeleted(context, failedState);
}
else
{
if (LogEvents)
{
_logger.ErrorException(
$"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred.",
failedState.Exception);
}
}
}
public void OnStateElection(ElectStateContext context)
{
Assert.NotNull(context);
_results.Add(_name);
if (_changeState != null)
{
context.CandidateState = _changeState;
}
}
/// <inheritdoc />
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState is FailedState failedState)
{
this._telemetryClient.TrackException(new ExceptionTelemetry(failedState.Exception)
{
Message = $"Job `{context.BackgroundJob.Id}` has been failed due to an exception `{failedState.Exception}`",
SeverityLevel = SeverityLevel.Error
});
}
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState is FailedState failedState)
{
if (context.GetJobParameter <bool>(HangfireNoMoreRetriesKey))
{
if (IsSendAdvice(context))
{
var logger = _kernel.GetService <ILogger>();
try
{
// Get argument input to method. In this case it's only one argument and it is the Id of the advice.
int adviceId = Convert.ToInt32(context.BackgroundJob.Job.Args.First());
using (new HangfireNinjectResolutionScope(_kernel))
{
var advisService = _kernel.GetService <IAdviceService>();
var userNotificationService = _kernel.GetService <IUserNotificationService>();
var failedAdvice = advisService.GetAdviceById(adviceId);
if (failedAdvice.IsNone)
{
logger.Error($"Failed to create user notification for advis with Id: {adviceId} as it could not be found.");
return;
}
var advice = failedAdvice.Value;
if (advice.HasInvalidState())
{
logger.Error($"Failed to create user notification for advis with Id: {adviceId} as it has an invalid state.");
return;
}
var organizationIdOfRelatedEntityId = GetRelatedEntityOrganizationId(advice);
if (organizationIdOfRelatedEntityId.IsNone)
{
logger.Error($"Failed to create user notification as get root resolution for advis with Id: {adviceId} failed to resolve root.");
return;
}
var nameForNotification = advice.Name ?? "Ikke navngivet";
userNotificationService.AddUserNotification(organizationIdOfRelatedEntityId.Value, advice.ObjectOwnerId.Value, nameForNotification, $"Afsendelse af advis fejlede efter {KitosConstants.MaxHangfireRetries} forsøg. Undersøg gerne nærmere og rapportér evt. fejlen.", advice.RelationId.Value, advice.Type.Value, NotificationType.Advice);
}
}
catch (Exception e)
{
logger.Error(e, $"Failed to create user notification for failed hangfire job: {context.BackgroundJob.Job}");
}
}
}
else if (context.GetJobParameter <int>(HangfireRetryCountKey) >= (KitosConstants.MaxHangfireRetries - 1))
{
context.SetJobParameter <bool>(HangfireNoMoreRetriesKey, true);
}
}
}
/// <summary>
/// Chamado quando um status da ação está sendo setado.
/// Esta mudança de estado pode ser interceptada e o estado final pode ser alterado
/// através da definição do estado diferente no contexto em uma implementação deste método.
/// </summary>
/// <param name="context">Contexto do evento</param>
public void OnStateElection(ElectStateContext context)
{
var failedState = context.CandidateState as FailedState;
if (failedState != null)
{
var innerException = failedState.Exception.InnerException;
Logger.LogWarning($"Job `{context.BackgroundJob.Id}` has been failed due to an exception `{JsonConvert.SerializeObject(innerException)}`");
failedState.Reason = innerException.Message;
}
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState is EnqueuedState enqueuedState)
{
var queue = context.GetJobParameter <string>(JobParam.Queue);
if (!string.IsNullOrWhiteSpace(queue))
{
enqueuedState.Queue = queue;
}
}
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState.Name == SucceededState.StateName)
{
var scheduleAt = DateTime.UtcNow.AddSeconds(RepeatInterval);
context.CandidateState = new ScheduledState(scheduleAt)
{
Reason = "Scheduled as a recurring job"
};
}
}
/// <summary>
/// Action: OnStateElection
/// Description: It is the error filter to capture any error occur while performing the data sync. If any error occured then it will update sync status to 3.
/// Sync status 3 means failed.
/// </summary>
/// <param name="context"></param>
void IElectStateFilter.OnStateElection(ElectStateContext context)
{
Console.WriteLine(string.Format("Job `{0}` has been changed to state `{1}`", context.BackgroundJob?.Id, context.CandidateState.Name));
//Get current state
var failedState = context.CandidateState as FailedState;
if (failedState != null)
{
Console.WriteLine(string.Format("Job `{0}` has been failed due to an exception `{1}`", context.BackgroundJob.Id, failedState.Exception));
var ccid = context.BackgroundJob.Job.Args.ElementAt(2) as string;
int connectorId = (int)context.BackgroundJob.Job.Args.ElementAt(1);
if (!string.IsNullOrEmpty(ccid) && connectorId > 0)
{
if (GC.GetTotalMemory(false) >= 67108864)
{
Console.WriteLine($"GC.Generation: 2, max allocated memory: {GC.GetTotalMemory(false)}");
GC.Collect(2);
GC.WaitForPendingFinalizers();
GC.Collect(2);
Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}");
}
if (GC.GetTotalMemory(false) >= 33554432)
{
Console.WriteLine($"GC.Generation: 1, max allocated memory: {GC.GetTotalMemory(false)}");
GC.Collect(1);
GC.WaitForPendingFinalizers();
GC.Collect(1);
Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}");
}
if (GC.GetTotalMemory(false) >= 20971520)
{
Console.WriteLine($"GC.Generation: 0, max allocated memory: {GC.GetTotalMemory(false)}");
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}");
}
//set sync status to failed{3}
var connectorLogs = new ConnectorLogs()
{
sync_ended_at = DateTime.UtcNow,
sync_logs = new List <string>()
{
HttpUtility.UrlEncode($"{DateTime.UtcNow:yyyy-MM-dd HH:mm:ss.fff zzz} [{LogLevel.Error}]: {failedState.Exception} {Environment.NewLine}")
}
};
SyncRepository.UpdateSyncInfo(id: connectorId, ccid: ccid, status: 3, connectorLogs: connectorLogs);
}
}
}
public void IElectStateFilter.OnStateElection(ElectStateContext context)
{
var failedState = context.CandidateState as FailedState;
if (failedState != null)
{
//Job has filed
//Job ID => context.BackgroundJob.Id,
//Exception =>failedState.Exception
}
}
// All failed after retry will be catched here and I don't know if you still need this
// but it is up to you
public void OnStateElection(ElectStateContext context)
{
var failedState = context.CandidateState as FailedState;
if (failedState != null && failedState.Exception != null)
{
if (!string.IsNullOrEmpty(failedState.Exception.Message) && failedState.Exception.Message.Contains("Timeout expired. The timeout elapsed prior to obtaining a distributed lock on"))
{
}
}
}
public void OnStateElection(ElectStateContext context)
{
var failedState = context.CandidateState as FailedState;
if (failedState != null)
{
logger.Warn(
"任务 `{0}` 执行失败，异常为 `{1}`",
context.BackgroundJob.Id,
failedState.Exception);
}
}
public void OnStateElection(ElectStateContext context)
{
var failedState = context.CandidateState as FailedState;
if (failedState != null)
{
Console.WriteLine(
"Job `{0}` has been failed due to an exception `{1}`",
context.BackgroundJob.Id,
failedState.Exception);
}
}
/// <summary>
/// Schedules the job to run again later. See <see cref="DelayInSecondsByAttemptFunc"/>.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="retryAttempt">The count of retry attempts made so far.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState)
{
var jobdta = context.BackgroundJob.Job.Args.FirstOrDefault();
var httpjob = JsonConvert.DeserializeObject <HttpJobItem>(jobdta.ToString());
if (!httpjob.EnableRetry)
{
return;
}
context.SetJobParameter("RetryCount", retryAttempt);
int delayInSeconds;
if (_delaysInSeconds != null)
{
delayInSeconds = retryAttempt <= _delaysInSeconds.Length
? _delaysInSeconds[retryAttempt - 1]
: _delaysInSeconds.Last();
}
else
{
delayInSeconds = _delayInSecondsByAttemptFunc(retryAttempt);
}
var delay = TimeSpan.FromSeconds(delayInSeconds);
const int maxMessageLength = 50;
var exceptionMessage = failedState.Exception.Message.Length > maxMessageLength
? failedState.Exception.Message.Substring(0, maxMessageLength - 1) + "…"
: failedState.Exception.Message;
// If attempt number is less than max attempts, we should
// schedule the job to run again later.
var reason = $"Retry attempt {retryAttempt} of {Attempts}: {exceptionMessage}";
context.CandidateState = delay == TimeSpan.Zero
? (IState) new EnqueuedState {
Reason = reason
}
: new ScheduledState(delay)
{
Reason = reason
};
if (LogEvents)
{
_logger.WarnException(
$"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred. Retry attempt {retryAttempt} of {Attempts} will be performed in {delay}.",
failedState.Exception);
}
}
public void OnStateElection(ElectStateContext context)
{
var awaitingState = context.CandidateState as AwaitingState;
if (awaitingState != null)
{
// Branch for a child background job.
AddContinuation(context, awaitingState);
}
else if (_knownFinalStates.Contains(context.CandidateState.Name))
{
// Branch for a parent background job.
ExecuteContinuationsIfExist(context);
}
}
public void OnStateElection(ElectStateContext context)
{
var failedState = context.CandidateState as FailedState;
if (failedState == null)
{
// This filter accepts only failed job state.
return;
}
var retryAttempt = context.GetJobParameter<int>("RetryCount") + 1;
if (retryAttempt <= Attempts)
{
var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt));
context.SetJobParameter("RetryCount", retryAttempt);
// If attempt number is less than max attempts, we should
// schedule the job to run again later.
context.CandidateState = new ScheduledState(delay)
{
Reason = String.Format("Retry attempt {0} of {1}", retryAttempt, Attempts)
};
if (LogEvents)
{
Logger.WarnFormat(
"Failed to process the job '{0}': an exception occurred. Retry attempt {1} of {2} will be performed in {3}.",
failedState.Exception,
context.JobId,
retryAttempt,
Attempts,
delay);
}
}
else
{
if (LogEvents)
{
Logger.ErrorFormat(
"Failed to process the job '{0}': an exception occurred.",
failedState.Exception,
context.JobId);
}
}
}
private void AddContinuation(ElectStateContext context, AwaitingState awaitingState)
{
var connection = context.Connection;
var parentId = awaitingState.ParentId;
// We store continuations as a json array in a job parameter. Since there
// is no way to add a continuation in an atomic way, we are placing a
// distributed lock on parent job to prevent race conditions, when
// multiple threads add continuation to the same parent job.
using (connection.AcquireDistributedJobLock(parentId, AddJobLockTimeout))
{
var continuations = GetContinuations(connection, parentId);
continuations.Add(new Continuation { JobId = context.JobId, Options = awaitingState.Options });
var jobData = connection.GetJobData(parentId);
if (jobData == null)
{
// When we try to add a continuation for a removed job,
// the system should throw an exception instead of creating
// corrupted state.
throw new InvalidOperationException(
String.Format("Can not add a continuation: parent background job '{0}' does not exist.", parentId));
}
var currentState = connection.GetStateData(parentId);
// Set continuation only after ensuring that parent job still
// exists. Otherwise we could create add non-expiring (garbage)
// parameter for the parent job.
SetContinuations(connection, parentId, continuations);
if (currentState != null && _knownFinalStates.Contains(currentState.Name))
{
var startImmediately = !awaitingState.Options.HasFlag(JobContinuationOptions.OnlyOnSucceededState) ||
currentState.Name == SucceededState.StateName;
context.CandidateState = startImmediately
? awaitingState.NextState
: new DeletedState { Reason = "Missed continuation" };
}
}
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState.Name != SucceededState.StateName
&& context.CandidateState.Name != FailedState.StateName)
{
return;
}
using (var transaction = context.Connection.CreateWriteTransaction())
{
if (context.CandidateState.Name == SucceededState.StateName)
{
transaction.IncrementCounter(
String.Format(
"stats:succeeded:{0}",
DateTime.UtcNow.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)),
DateTime.UtcNow.AddMonths(1) - DateTime.UtcNow);
transaction.IncrementCounter(
String.Format(
"stats:succeeded:{0}",
DateTime.UtcNow.ToString("yyyy-MM-dd-HH")),
TimeSpan.FromDays(1));
}
else if (context.CandidateState.Name == FailedState.StateName)
{
transaction.IncrementCounter(
String.Format(
"stats:failed:{0}",
DateTime.UtcNow.ToString("yyyy-MM-dd")),
DateTime.UtcNow.AddMonths(1) - DateTime.UtcNow);
transaction.IncrementCounter(
String.Format(
"stats:failed:{0}",
DateTime.UtcNow.ToString("yyyy-MM-dd-HH")),
TimeSpan.FromDays(1));
}
transaction.Commit();
}
}
/// <inheritdoc />
public void OnStateElection(ElectStateContext context)
{
var state = context.CandidateState as ProcessingState;
if (state == null)
{
//this filter only accepts Processing
return;
}
var elapsedTime = DateTime.UtcNow - context.BackgroundJob.CreatedAt;
if (elapsedTime.TotalSeconds > _timeoutInSeconds)
{
context.CandidateState = new DeletedState
{
Reason = $"Background job has exceeded latency timeout of {_timeoutInSeconds} second(s)"
};
Logger.Log(
LogLevel,
() => $"Background job '{context.BackgroundJob.Id}' has exceeded latency timeout of {_timeoutInSeconds} second(s) and will be deleted");
}
}
public IState ApplyState(ApplyStateContext initialContext)
{
var filterInfo = GetFilters(initialContext.BackgroundJob.Job);
var electFilters = filterInfo.ElectStateFilters;
var applyFilters = filterInfo.ApplyStateFilters;
// Electing a a state
var electContext = new ElectStateContext(initialContext);
foreach (var filter in electFilters)
{
filter.OnStateElection(electContext);
}
foreach (var state in electContext.TraversedStates)
{
initialContext.Transaction.AddJobState(electContext.BackgroundJob.Id, state);
}
// Applying the elected state
var context = new ApplyStateContext(initialContext.Transaction, electContext)
{
JobExpirationTimeout = initialContext.JobExpirationTimeout
};
foreach (var filter in applyFilters)
{
filter.OnStateUnapplied(context, context.Transaction);
}
foreach (var filter in applyFilters)
{
filter.OnStateApplied(context, context.Transaction);
}
return _innerStateMachine.ApplyState(context);
}
/// <summary>
/// Schedules the job to run again later. See <see cref="SecondsToDelay"/>.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="retryAttempt">The count of retry attempts made so far.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState)
{
var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt));
context.SetJobParameter("RetryCount", retryAttempt);
// If attempt number is less than max attempts, we should
// schedule the job to run again later.
context.CandidateState = new ScheduledState(delay)
{
Reason = String.Format("Retry attempt {0} of {1}", retryAttempt, Attempts)
};
if (LogEvents)
{
Logger.WarnFormat(
"Failed to process the job '{0}': an exception occurred. Retry attempt {1} of {2} will be performed in {3}.",
failedState.Exception,
context.JobId,
retryAttempt,
Attempts,
delay);
}
}
public void OnStateElection(ElectStateContext context)
{
if (context.CandidateState.Name != FailedState.StateName)
{
// This filter accepts only failed job state.
return;
}
var retryCount = context.GetJobParameter<int>("RetryCount");
if (retryCount < Attempts)
{
var delay = DateTime.UtcNow.AddSeconds(SecondsToDelay(retryCount));
context.SetJobParameter("RetryCount", retryCount + 1);
// If attempt number is less than max attempts, we should
// schedule the job to run again later.
context.CandidateState = new ScheduledState(delay)
{
Reason = String.Format("Retry attempt {0} of {1}", retryCount + 1, Attempts)
};
}
}
private static void ExecuteContinuationsIfExist(ElectStateContext context)
{
// The following lines are being executed inside a distributed job lock,
// so it is safe to get continuation list here.
var continuations = GetContinuations(context.Connection, context.JobId);
var nextStates = new Dictionary<string, IState>();
// Getting continuation data for all continuations – state they are waiting
// for and their next state.
foreach (var continuation in continuations)
{
if (String.IsNullOrWhiteSpace(continuation.JobId)) continue;
var currentState = GetContinuaionState(context, continuation.JobId, ContinuationStateFetchTimeout);
if (currentState == null)
{
continue;
}
// All continuations should be in the awaiting state. If someone changed
// the state of a continuation, we should simply skip it.
if (currentState.Name != AwaitingState.StateName) continue;
if (continuation.Options.HasFlag(JobContinuationOptions.OnlyOnSucceededState) &&
context.CandidateState.Name != SucceededState.StateName)
{
nextStates.Add(continuation.JobId, new DeletedState { Reason = "Missed continuation" });
continue;
}
IState nextState;
try
{
nextState = JsonConvert.DeserializeObject<IState>(
currentState.Data["NextState"],
new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Objects });
}
catch (Exception ex)
{
nextState = new FailedState(ex)
{
Reason = "Can not start the continuation due to de-serialization error."
};
}
nextStates.Add(continuation.JobId, nextState);
}
foreach (var tuple in nextStates)
{
context.StateMachine.ChangeState(tuple.Key, tuple.Value, new[] { AwaitingState.StateName });
}
}
/// <summary>
/// Transition the candidate state to the deleted state.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void TransitionToDeleted(ElectStateContext context, FailedState failedState)
{
context.CandidateState = new DeletedState
{
Reason = String.Format("Automatic deletion after retry count exceeded {0}", Attempts)
};
if (LogEvents)
{
Logger.WarnException(
String.Format(
"Failed to process the job '{0}': an exception occured. Job was automatically deleted because the retry attempt count exceeded {1}.",
context.JobId,
Attempts),
failedState.Exception);
}
}
private static StateData GetContinuaionState(ElectStateContext context, string continuationJobId, TimeSpan timeout)
{
StateData currentState = null;
var started = DateTime.UtcNow;
var firstAttempt = true;
while (true)
{
var continuationData = context.Connection.GetJobData(continuationJobId);
if (continuationData == null)
{
Logger.Warn(String.Format(
"Can not start continuation '{0}' for background job '{1}': continuation does not exist.",
continuationJobId,
context.JobId));
break;
}
currentState = context.Connection.GetStateData(continuationJobId);
if (currentState != null)
{
break;
}
if (DateTime.UtcNow >= started.Add(timeout))
{
throw new TimeoutException(String.Format(
"Can not start continuation '{0}' for background job '{1}': timeout expired while trying to fetch continuation state.",
continuationJobId,
context.JobId));
}
Thread.Sleep(firstAttempt ? 0 : 1);
firstAttempt = false;
}
return currentState;
}
internal virtual State ElectState(
StateContext stateContext,
State toState,
string fromStateName,
IEnumerable<IElectStateFilter> filters)
{
var context = new ElectStateContext(
stateContext, toState, fromStateName, _connection);
return context.ElectState(filters);
}
private void ChangeState(StateContext context, IState toState, string oldStateName)
{
var electStateContext = new ElectStateContext(context, _connection, this, toState, oldStateName);
_stateChangeProcess.ElectState(_connection, electStateContext);
var applyStateContext = new ApplyStateContext(
context,
electStateContext.CandidateState,
oldStateName,
electStateContext.TraversedStates);
ApplyState(applyStateContext);
}
/// <summary>
/// Transition the candidate state to the deleted state.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void TransitionToDeleted(ElectStateContext context, FailedState failedState)
{
context.CandidateState = new DeletedState
{
Reason = "Exceeded the maximum number of retry attempts."
};
if (LogEvents)
{
Logger.WarnException(
String.Format(
"Failed to process the job '{0}': an exception occured. Job was automatically deleted because the retry attempt count exceeded {1}.",
context.BackgroundJob.Id,
Attempts),
failedState.Exception);
}
}
/// <summary>
/// Transition the candidate state to the deleted state.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void TransitionToDeleted(ElectStateContext context, FailedState failedState)
{
context.CandidateState = new DeletedState
{
Reason = Attempts > 0
? "Exceeded the maximum number of retry attempts."
: "Retries were disabled for this job."
};
if (LogEvents)
{
Logger.WarnException(
$"Failed to process the job '{context.BackgroundJob.Id}': an exception occured. Job was automatically deleted because the retry attempt count exceeded {Attempts}.",
failedState.Exception);
}
}
/// <summary>
/// Schedules the job to run again later. See <see cref="SecondsToDelay"/>.
/// </summary>
/// <param name="context">The state context.</param>
/// <param name="retryAttempt">The count of retry attempts made so far.</param>
/// <param name="failedState">Object which contains details about the current failed state.</param>
private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState)
{
context.SetJobParameter("RetryCount", retryAttempt);
var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt));
const int maxMessageLength = 50;
var exceptionMessage = failedState.Exception.Message.Length > maxMessageLength
? failedState.Exception.Message.Substring(0, maxMessageLength - 1) + "…"
: failedState.Exception.Message;
// If attempt number is less than max attempts, we should
// schedule the job to run again later.
context.CandidateState = new ScheduledState(delay)
{
Reason = $"Retry attempt {retryAttempt} of {Attempts}: {exceptionMessage}"
};
if (LogEvents)
{
Logger.WarnException(
$"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred. Retry attempt {retryAttempt} of {Attempts} will be performed in {delay}.",
failedState.Exception);
}
}
