public bool ChangeState(StateContext context, IState toState, string oldStateName) { try { var filterInfo = GetFilters(context.Job); var electStateContext = new ElectStateContext(context, toState, oldStateName); var electedState = ElectState(electStateContext, filterInfo.ElectStateFilters); var applyStateContext = new ApplyStateContext(context, electedState, oldStateName); ApplyState(applyStateContext, filterInfo.ApplyStateFilters); // State transition was succeeded. return true; } catch (Exception ex) { var failedState = new FailedState(ex) { Reason = "An exception occurred during the transition of job's state" }; var applyStateContext = new ApplyStateContext(context, failedState, oldStateName); // We should not use any state changed filters, because // some of the could cause an exception. ApplyState(applyStateContext, Enumerable.Empty<IApplyStateFilter>()); // State transition was failed due to exception. return false; } }
public void OnStateElection(ElectStateContext context) { var failedState = context.CandidateState as FailedState; if (failedState != null) { int retryCount = context.GetJobParameter <int>("RetryCount"); if (retryCount == 3) { var eventGridEventHandler = (IEventGridEventHandler)activator.ActivateJob(typeof(IEventGridEventHandler)); SubscriptionModel subscription = (SubscriptionModel)context.BackgroundJob.Job.Args[0]; EventModel @event = (EventModel)context.BackgroundJob.Job.Args[1]; var errorEvent = new EventModel { Data = @event, EventType = "events.deliveryFailed", Source = @event.Source, Extensions = new Dictionary <string, object> { { nameof(subscription), subscription } } }; eventGridEventHandler.HandleAsync(subscription.Topic, errorEvent).Wait(); } //job has failed, stop application here } }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState is EnqueuedState enqueuedState) { enqueuedState.Queue = String.Format(Queue, context.BackgroundJob.Job.Args.ToArray()); } }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState.Name == SucceededState.StateName) { context.Transaction.IncrementCounter( String.Format( "stats:succeeded:{0}", DateTime.UtcNow.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)), DateTime.UtcNow.AddMonths(1) - DateTime.UtcNow); context.Transaction.IncrementCounter( String.Format( "stats:succeeded:{0}", DateTime.UtcNow.ToString("yyyy-MM-dd-HH")), TimeSpan.FromDays(1)); } else if (context.CandidateState.Name == FailedState.StateName) { context.Transaction.IncrementCounter( String.Format( "stats:failed:{0}", DateTime.UtcNow.ToString("yyyy-MM-dd")), DateTime.UtcNow.AddMonths(1) - DateTime.UtcNow); context.Transaction.IncrementCounter( String.Format( "stats:failed:{0}", DateTime.UtcNow.ToString("yyyy-MM-dd-HH")), TimeSpan.FromDays(1)); } }
public void OnStateElection(ElectStateContext context) { var failedState = context.CandidateState as FailedState; if (failedState == null) { // This filter accepts only failed job state. return; } var retryAttempt = context.GetJobParameter<int>("RetryCount") + 1; if (retryAttempt <= Attempts) { ScheduleAgainLater(context, retryAttempt, failedState); } else if (retryAttempt > Attempts && OnAttemptsExceeded == AttemptsExceededAction.Delete) { TransitionToDeleted(context, failedState); } else { if (LogEvents) { Logger.ErrorException( String.Format( "Failed to process the job '{0}': an exception occurred.", context.JobId), failedState.Exception); } } }
private void SetState(ElectStateContext context, MonitorJobStatusDto dto) { var actionName = context.BackgroundJob.Job.Method.GetFullActionName(); var key = context.Connection.GetMonitorStateKey(DateTime.Today, actionName); context.Connection.SetMonitorState(key, dto); }
/// <summary> /// Schedules the job to run again later. See <see cref="SecondsToDelay"/>. /// </summary> /// <param name="context">The state context.</param> /// <param name="retryAttempt">The count of retry attempts made so far.</param> /// <param name="failedState">Object which contains details about the current failed state.</param> private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState) { context.SetJobParameter("RetryCount", retryAttempt); var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt)); const int maxMessageLength = 50; var exceptionMessage = failedState.Exception.Message; // If attempt number is less than max attempts, we should // schedule the job to run again later. context.CandidateState = new ScheduledState(delay) { Reason = String.Format( "Retry attempt {0} of {1}: {2}", retryAttempt, Attempts, exceptionMessage.Length > maxMessageLength ? exceptionMessage.Substring(0, maxMessageLength - 1) + "…" : exceptionMessage) }; if (LogEvents) { Logger.WarnException( String.Format( "Failed to process the job '{0}': an exception occurred. Retry attempt {1} of {2} will be performed in {3}.", context.JobId, retryAttempt, Attempts, delay), failedState.Exception); } }
/// <summary> /// Schedules the job to run again later. See <see cref="SecondsToDelay"/>. /// </summary> /// <param name="context">The state context.</param> /// <param name="retryAttempt">The count of retry attempts made so far.</param> /// <param name="failedState">Object which contains details about the current failed state.</param> private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState) { context.SetJobParameter("RetryCount", retryAttempt); var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt)); const int maxMessageLength = 50; var exceptionMessage = failedState.Exception.Message.Length > maxMessageLength ? failedState.Exception.Message.Substring(0, maxMessageLength - 1) + "…" : failedState.Exception.Message; // If attempt number is less than max attempts, we should // schedule the job to run again later. context.CandidateState = new ScheduledState(delay, context.BackgroundJob.Job.QueueName) { Reason = $"Retry attempt {retryAttempt} of {Attempts}: {exceptionMessage}" }; if (LogEvents) { Logger.WarnException( $"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred. Retry attempt {retryAttempt} of {Attempts} will be performed in {delay}.", failedState.Exception); } }
public void OnStateElection(ElectStateContext context) { var failedState = context.CandidateState as FailedState; if (failedState != null) { int retryCount = context.GetJobParameter <int>("RetryCount"); if (retryCount == 3) { var eventGridEventHandler = (IEventGridEventHandler)activator.ActivateJob(typeof(IEventGridEventHandler)); // var httpClientFactory = (IHttpClientFactory)serviceProvider.GetService(typeof(IHttpClientFactory)); var subscriber = (Subscription)context.BackgroundJob.Job.Args[0]; var @event = (EventModel)context.BackgroundJob.Job.Args[1]; var errorEvent = new EventModel { Data = @event, EventType = "event.failed", Source = @event.Source }; eventGridEventHandler.HandleAsync(subscriber.Topic, errorEvent).Wait(); } //job has failed, stop application here } }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState is FailedState failedState) { Logger.WarnFormat($"Job `{context.BackgroundJob.Id}` has been failed due to an exception `{failedState.Exception}`"); } }
/// <summary> /// Schedules the job to run again later. See <see cref="SecondsToDelay"/>. /// </summary> /// <param name="context">The state context.</param> /// <param name="retryAttempt">The count of retry attempts made so far.</param> /// <param name="failedState">Object which contains details about the current failed state.</param> private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState) { var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt)); context.SetJobParameter("RetryCount", retryAttempt); // If attempt number is less than max attempts, we should // schedule the job to run again later. context.CandidateState = new ScheduledState(delay) { Reason = String.Format("Retry attempt {0} of {1}", retryAttempt, Attempts) }; if (LogEvents) { Logger.WarnException( String.Format( "Failed to process the job '{0}': an exception occurred. Retry attempt {1} of {2} will be performed in {3}.", context.JobId, retryAttempt, Attempts, delay), failedState.Exception); } }
/// <inheritdoc /> public void OnStateElection(ElectStateContext context) { var failedState = context.CandidateState as FailedState; if (failedState == null) { // This filter accepts only failed job state. return; } var retryAttempt = context.GetJobParameter <int>("RetryCount") + 1; if (retryAttempt <= Attempts) { ScheduleAgainLater(context, retryAttempt, failedState); } else if (retryAttempt > Attempts && OnAttemptsExceeded == AttemptsExceededAction.Delete) { TransitionToDeleted(context, failedState); } else { if (LogEvents) { Logger.ErrorException( $"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred.", failedState.Exception); } } }
private static IState ElectState( ElectStateContext context, IEnumerable<IElectStateFilter> filters) { var statesToAppend = new List<IState>(); foreach (var filter in filters) { var oldState = context.CandidateState; filter.OnStateElection(context); if (oldState != context.CandidateState) { statesToAppend.Add(oldState); } } if (statesToAppend.Count > 0) { using (var transaction = context.Connection.CreateWriteTransaction()) { foreach (var state in statesToAppend) { transaction.AddJobState(context.JobId, state); } transaction.Commit(); } } return context.CandidateState; }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState.Name == SucceededState.StateName) { context.Transaction.IncrementCounter( String.Format( "stats:succeeded:{0}", DateTime.UtcNow.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)), DateTime.UtcNow.AddMonths(1) - DateTime.UtcNow); context.Transaction.IncrementCounter( String.Format( "stats:succeeded:{0}", DateTime.UtcNow.ToString("yyyy-MM-dd-HH")), TimeSpan.FromDays(1)); } else if (context.CandidateState.Name == FailedState.StateName) { context.Transaction.IncrementCounter( String.Format( "stats:failed:{0}", DateTime.UtcNow.ToString("yyyy-MM-dd")), DateTime.UtcNow.AddMonths(1) - DateTime.UtcNow); context.Transaction.IncrementCounter( String.Format( "stats:failed:{0}", DateTime.UtcNow.ToString("yyyy-MM-dd-HH")), TimeSpan.FromDays(1)); } }
public void ElectState(IStorageConnection connection, ElectStateContext context) { var filterInfo = GetFilters(context.Job); foreach (var filter in filterInfo.ElectStateFilters) { filter.OnStateElection(context); } }
public void OnStateElection(ElectStateContext context) { var enqueuedState = context.CandidateState as EnqueuedState; if (enqueuedState != null) { enqueuedState.Queue = Queue; } }
public void OnStateElection(ElectStateContext context) { var enqueuedState = context.CandidateState as EnqueuedState; if (enqueuedState != null) { enqueuedState.Queue = (context.BackgroundJob.Job.Args[ParameterIndex] as string).Replace("\"", string.Empty); } }
private void AddContinuation(ElectStateContext context, AwaitingState awaitingState) { var connection = context.Connection; var parentId = awaitingState.ParentId; // We store continuations as a json array in a job parameter. Since there // is no way to add a continuation in an atomic way, we are placing a // distributed lock on parent job to prevent race conditions, when // multiple threads add continuation to the same parent job. using (connection.AcquireDistributedJobLock(parentId, AddJobLockTimeout)) { var continuations = GetContinuations(connection, parentId); // Continuation may be already added. This may happen, when outer transaction // was failed after adding a continuation last time, since the addition is // performed outside of an outer transaction. if (!continuations.Exists(x => x.JobId == context.BackgroundJob.Id)) { continuations.Add(new Continuation { JobId = context.BackgroundJob.Id, Options = awaitingState.Options }); // Set continuation only after ensuring that parent job still // exists. Otherwise we could create add non-expiring (garbage) // parameter for the parent job. SetContinuations(connection, parentId, continuations); } var jobData = connection.GetJobData(parentId); if (jobData == null) { // When we try to add a continuation for a removed job, // the system should throw an exception instead of creating // corrupted state. throw new InvalidOperationException( $"Can not add a continuation: parent background job '{parentId}' does not exist."); } var currentState = connection.GetStateData(parentId); if (currentState != null && _knownFinalStates.Contains(currentState.Name)) { var startImmediately = !awaitingState.Options.HasFlag(JobContinuationOptions.OnlyOnSucceededState) || currentState.Name == SucceededState.StateName; if (_pushResults && currentState.Data.TryGetValue("Result", out var antecedentResult)) { context.Connection.SetJobParameter(context.BackgroundJob.Id, "AntecedentResult", antecedentResult); } context.CandidateState = startImmediately ? awaitingState.NextState : new DeletedState { Reason = "Continuation condition was not met" }; } } }
public void OnStateElection(ElectStateContext context) { var enqueuedState = context.CandidateState as EnqueuedState; if (enqueuedState != null) { enqueuedState.Queue = Queue; } }
public void OnStateElection(ElectStateContext context) { var jobServices = new JobServices(); if (context.CandidateState is FailedState failedState) { jobServices.UpdateJob(context.BackgroundJob.Id, Status.Failed, failedState.Exception.Message); } }
/// <inheritdoc /> public void OnStateElection(ElectStateContext context) { var contextCandidateState = context.CandidateState as SucceededState; if (contextCandidateState?.Result is Result result) { result.OnFailure(error => SetFailedState(context, error)); } }
public void OnStateElection(ElectStateContext context) { var jobService = new JobServices(); var failedState = context.CandidateState as FailedState; if (failedState != null) { jobService.UpdateJob(context.BackgroundJob.Id, Status.Failed, failedState.Exception.Message); } }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState is FailedState failedState) { logger.WarnFormat( "[OnStateElection] BackgroundJob.Id `{0}` Failed,Exception `{1}`", context.BackgroundJob.Id, failedState.Exception.ToString()); } }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState is FailedState failedState) { Logger.WarnFormat( "IElectStateFilter: Job `{0}` has been failed due to an exception `{1}`", context.BackgroundJob.Id, failedState.Exception); } }
public void OnStateElection(ElectStateContext context) { var failedState = context.CandidateState as FailedState; if (failedState != null) { Logger.WarnFormat( $"Job `{context.BackgroundJob?.Id}` has been failed due to an exception `{failedState.Exception?.Message}`"); } }
public void OnStateElection(ElectStateContext context) { var jobService = new JobServices(); var failedState = context.CandidateState as FailedState; if (failedState != null) { jobService.UpdateJob(context.BackgroundJob.Id, Status.Failed, failedState.Exception.Message); } }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState is EnqueuedState enqueuedState) { if (context.BackgroundJob.Job.Args.Any() && context.BackgroundJob.Job.Args[0] is ISpecifiedQueueJob specifiedQueueJob) { enqueuedState.Queue = specifiedQueueJob.Queue; } } }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState is FailedState failedState) { logger.Warn( "任务 `{0}` 执行失败,异常为 `{1}`", context.BackgroundJob.Id, failedState.Exception); } }
/// <inheritdoc /> public void OnStateElection(ElectStateContext context) { var jobdta = context.BackgroundJob.Job.Args.FirstOrDefault(); if (jobdta == null) { return; } var httpjob = jobdta as HttpJobItem; if (httpjob != null && !httpjob.EnableRetry) { return; } var failedState = context.CandidateState as FailedState; if (failedState == null) { // This filter accepts only failed job state. return; } //var job = context.BackgroundJob.Job.Args.FirstOrDefault(); var retryAttempt = context.GetJobParameter <int>("RetryCount") + 1; if (retryAttempt <= Attempts) { ScheduleAgainLater(context, retryAttempt, failedState); return; } try { //删除Runtime数据 代表的是需要重试 但是已经重试到最大次数了 var hashKey = CodingUtil.MD5(context.BackgroundJob.Id + ".runtime"); context.Transaction.RemoveHash(hashKey); } catch (Exception) { //ignore } if (retryAttempt > Attempts && OnAttemptsExceeded == AttemptsExceededAction.Delete) { TransitionToDeleted(context, failedState); } else { if (LogEvents) { _logger.ErrorException( $"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred.", failedState.Exception); } } }
public void OnStateElection(ElectStateContext context) { Assert.NotNull(context); _results.Add(_name); if (_changeState != null) { context.CandidateState = _changeState; } }
/// <inheritdoc /> public void OnStateElection(ElectStateContext context) { if (context.CandidateState is FailedState failedState) { this._telemetryClient.TrackException(new ExceptionTelemetry(failedState.Exception) { Message = $"Job `{context.BackgroundJob.Id}` has been failed due to an exception `{failedState.Exception}`", SeverityLevel = SeverityLevel.Error }); } }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState is FailedState failedState) { if (context.GetJobParameter <bool>(HangfireNoMoreRetriesKey)) { if (IsSendAdvice(context)) { var logger = _kernel.GetService <ILogger>(); try { // Get argument input to method. In this case it's only one argument and it is the Id of the advice. int adviceId = Convert.ToInt32(context.BackgroundJob.Job.Args.First()); using (new HangfireNinjectResolutionScope(_kernel)) { var advisService = _kernel.GetService <IAdviceService>(); var userNotificationService = _kernel.GetService <IUserNotificationService>(); var failedAdvice = advisService.GetAdviceById(adviceId); if (failedAdvice.IsNone) { logger.Error($"Failed to create user notification for advis with Id: {adviceId} as it could not be found."); return; } var advice = failedAdvice.Value; if (advice.HasInvalidState()) { logger.Error($"Failed to create user notification for advis with Id: {adviceId} as it has an invalid state."); return; } var organizationIdOfRelatedEntityId = GetRelatedEntityOrganizationId(advice); if (organizationIdOfRelatedEntityId.IsNone) { logger.Error($"Failed to create user notification as get root resolution for advis with Id: {adviceId} failed to resolve root."); return; } var nameForNotification = advice.Name ?? "Ikke navngivet"; userNotificationService.AddUserNotification(organizationIdOfRelatedEntityId.Value, advice.ObjectOwnerId.Value, nameForNotification, $"Afsendelse af advis fejlede efter {KitosConstants.MaxHangfireRetries} forsøg. Undersøg gerne nærmere og rapportér evt. fejlen.", advice.RelationId.Value, advice.Type.Value, NotificationType.Advice); } } catch (Exception e) { logger.Error(e, $"Failed to create user notification for failed hangfire job: {context.BackgroundJob.Job}"); } } } else if (context.GetJobParameter <int>(HangfireRetryCountKey) >= (KitosConstants.MaxHangfireRetries - 1)) { context.SetJobParameter <bool>(HangfireNoMoreRetriesKey, true); } } }
/// <summary> /// Chamado quando um status da ação está sendo setado. /// Esta mudança de estado pode ser interceptada e o estado final pode ser alterado /// através da definição do estado diferente no contexto em uma implementação deste método. /// </summary> /// <param name="context">Contexto do evento</param> public void OnStateElection(ElectStateContext context) { var failedState = context.CandidateState as FailedState; if (failedState != null) { var innerException = failedState.Exception.InnerException; Logger.LogWarning($"Job `{context.BackgroundJob.Id}` has been failed due to an exception `{JsonConvert.SerializeObject(innerException)}`"); failedState.Reason = innerException.Message; } }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState is EnqueuedState enqueuedState) { var queue = context.GetJobParameter <string>(JobParam.Queue); if (!string.IsNullOrWhiteSpace(queue)) { enqueuedState.Queue = queue; } } }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState.Name == SucceededState.StateName) { var scheduleAt = DateTime.UtcNow.AddSeconds(RepeatInterval); context.CandidateState = new ScheduledState(scheduleAt) { Reason = "Scheduled as a recurring job" }; } }
/// <summary> /// Action: OnStateElection /// Description: It is the error filter to capture any error occur while performing the data sync. If any error occured then it will update sync status to 3. /// Sync status 3 means failed. /// </summary> /// <param name="context"></param> void IElectStateFilter.OnStateElection(ElectStateContext context) { Console.WriteLine(string.Format("Job `{0}` has been changed to state `{1}`", context.BackgroundJob?.Id, context.CandidateState.Name)); //Get current state var failedState = context.CandidateState as FailedState; if (failedState != null) { Console.WriteLine(string.Format("Job `{0}` has been failed due to an exception `{1}`", context.BackgroundJob.Id, failedState.Exception)); var ccid = context.BackgroundJob.Job.Args.ElementAt(2) as string; int connectorId = (int)context.BackgroundJob.Job.Args.ElementAt(1); if (!string.IsNullOrEmpty(ccid) && connectorId > 0) { if (GC.GetTotalMemory(false) >= 67108864) { Console.WriteLine($"GC.Generation: 2, max allocated memory: {GC.GetTotalMemory(false)}"); GC.Collect(2); GC.WaitForPendingFinalizers(); GC.Collect(2); Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}"); } if (GC.GetTotalMemory(false) >= 33554432) { Console.WriteLine($"GC.Generation: 1, max allocated memory: {GC.GetTotalMemory(false)}"); GC.Collect(1); GC.WaitForPendingFinalizers(); GC.Collect(1); Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}"); } if (GC.GetTotalMemory(false) >= 20971520) { Console.WriteLine($"GC.Generation: 0, max allocated memory: {GC.GetTotalMemory(false)}"); GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}"); } //set sync status to failed{3} var connectorLogs = new ConnectorLogs() { sync_ended_at = DateTime.UtcNow, sync_logs = new List <string>() { HttpUtility.UrlEncode($"{DateTime.UtcNow:yyyy-MM-dd HH:mm:ss.fff zzz} [{LogLevel.Error}]: {failedState.Exception} {Environment.NewLine}") } }; SyncRepository.UpdateSyncInfo(id: connectorId, ccid: ccid, status: 3, connectorLogs: connectorLogs); } } }
public void IElectStateFilter.OnStateElection(ElectStateContext context) { var failedState = context.CandidateState as FailedState; if (failedState != null) { //Job has filed //Job ID => context.BackgroundJob.Id, //Exception =>failedState.Exception } }
// All failed after retry will be catched here and I don't know if you still need this // but it is up to you public void OnStateElection(ElectStateContext context) { var failedState = context.CandidateState as FailedState; if (failedState != null && failedState.Exception != null) { if (!string.IsNullOrEmpty(failedState.Exception.Message) && failedState.Exception.Message.Contains("Timeout expired. The timeout elapsed prior to obtaining a distributed lock on")) { } } }
public void OnStateElection(ElectStateContext context) { var failedState = context.CandidateState as FailedState; if (failedState != null) { logger.Warn( "任务 `{0}` 执行失败,异常为 `{1}`", context.BackgroundJob.Id, failedState.Exception); } }
public void OnStateElection(ElectStateContext context) { var failedState = context.CandidateState as FailedState; if (failedState != null) { Console.WriteLine( "Job `{0}` has been failed due to an exception `{1}`", context.BackgroundJob.Id, failedState.Exception); } }
/// <summary> /// Schedules the job to run again later. See <see cref="DelayInSecondsByAttemptFunc"/>. /// </summary> /// <param name="context">The state context.</param> /// <param name="retryAttempt">The count of retry attempts made so far.</param> /// <param name="failedState">Object which contains details about the current failed state.</param> private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState) { var jobdta = context.BackgroundJob.Job.Args.FirstOrDefault(); var httpjob = JsonConvert.DeserializeObject <HttpJobItem>(jobdta.ToString()); if (!httpjob.EnableRetry) { return; } context.SetJobParameter("RetryCount", retryAttempt); int delayInSeconds; if (_delaysInSeconds != null) { delayInSeconds = retryAttempt <= _delaysInSeconds.Length ? _delaysInSeconds[retryAttempt - 1] : _delaysInSeconds.Last(); } else { delayInSeconds = _delayInSecondsByAttemptFunc(retryAttempt); } var delay = TimeSpan.FromSeconds(delayInSeconds); const int maxMessageLength = 50; var exceptionMessage = failedState.Exception.Message.Length > maxMessageLength ? failedState.Exception.Message.Substring(0, maxMessageLength - 1) + "…" : failedState.Exception.Message; // If attempt number is less than max attempts, we should // schedule the job to run again later. var reason = $"Retry attempt {retryAttempt} of {Attempts}: {exceptionMessage}"; context.CandidateState = delay == TimeSpan.Zero ? (IState) new EnqueuedState { Reason = reason } : new ScheduledState(delay) { Reason = reason }; if (LogEvents) { _logger.WarnException( $"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred. Retry attempt {retryAttempt} of {Attempts} will be performed in {delay}.", failedState.Exception); } }
public void OnStateElection(ElectStateContext context) { var awaitingState = context.CandidateState as AwaitingState; if (awaitingState != null) { // Branch for a child background job. AddContinuation(context, awaitingState); } else if (_knownFinalStates.Contains(context.CandidateState.Name)) { // Branch for a parent background job. ExecuteContinuationsIfExist(context); } }
public void OnStateElection(ElectStateContext context) { var failedState = context.CandidateState as FailedState; if (failedState == null) { // This filter accepts only failed job state. return; } var retryAttempt = context.GetJobParameter<int>("RetryCount") + 1; if (retryAttempt <= Attempts) { var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt)); context.SetJobParameter("RetryCount", retryAttempt); // If attempt number is less than max attempts, we should // schedule the job to run again later. context.CandidateState = new ScheduledState(delay) { Reason = String.Format("Retry attempt {0} of {1}", retryAttempt, Attempts) }; if (LogEvents) { Logger.WarnFormat( "Failed to process the job '{0}': an exception occurred. Retry attempt {1} of {2} will be performed in {3}.", failedState.Exception, context.JobId, retryAttempt, Attempts, delay); } } else { if (LogEvents) { Logger.ErrorFormat( "Failed to process the job '{0}': an exception occurred.", failedState.Exception, context.JobId); } } }
private void AddContinuation(ElectStateContext context, AwaitingState awaitingState) { var connection = context.Connection; var parentId = awaitingState.ParentId; // We store continuations as a json array in a job parameter. Since there // is no way to add a continuation in an atomic way, we are placing a // distributed lock on parent job to prevent race conditions, when // multiple threads add continuation to the same parent job. using (connection.AcquireDistributedJobLock(parentId, AddJobLockTimeout)) { var continuations = GetContinuations(connection, parentId); continuations.Add(new Continuation { JobId = context.JobId, Options = awaitingState.Options }); var jobData = connection.GetJobData(parentId); if (jobData == null) { // When we try to add a continuation for a removed job, // the system should throw an exception instead of creating // corrupted state. throw new InvalidOperationException( String.Format("Can not add a continuation: parent background job '{0}' does not exist.", parentId)); } var currentState = connection.GetStateData(parentId); // Set continuation only after ensuring that parent job still // exists. Otherwise we could create add non-expiring (garbage) // parameter for the parent job. SetContinuations(connection, parentId, continuations); if (currentState != null && _knownFinalStates.Contains(currentState.Name)) { var startImmediately = !awaitingState.Options.HasFlag(JobContinuationOptions.OnlyOnSucceededState) || currentState.Name == SucceededState.StateName; context.CandidateState = startImmediately ? awaitingState.NextState : new DeletedState { Reason = "Missed continuation" }; } } }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState.Name != SucceededState.StateName && context.CandidateState.Name != FailedState.StateName) { return; } using (var transaction = context.Connection.CreateWriteTransaction()) { if (context.CandidateState.Name == SucceededState.StateName) { transaction.IncrementCounter( String.Format( "stats:succeeded:{0}", DateTime.UtcNow.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)), DateTime.UtcNow.AddMonths(1) - DateTime.UtcNow); transaction.IncrementCounter( String.Format( "stats:succeeded:{0}", DateTime.UtcNow.ToString("yyyy-MM-dd-HH")), TimeSpan.FromDays(1)); } else if (context.CandidateState.Name == FailedState.StateName) { transaction.IncrementCounter( String.Format( "stats:failed:{0}", DateTime.UtcNow.ToString("yyyy-MM-dd")), DateTime.UtcNow.AddMonths(1) - DateTime.UtcNow); transaction.IncrementCounter( String.Format( "stats:failed:{0}", DateTime.UtcNow.ToString("yyyy-MM-dd-HH")), TimeSpan.FromDays(1)); } transaction.Commit(); } }
/// <inheritdoc /> public void OnStateElection(ElectStateContext context) { var state = context.CandidateState as ProcessingState; if (state == null) { //this filter only accepts Processing return; } var elapsedTime = DateTime.UtcNow - context.BackgroundJob.CreatedAt; if (elapsedTime.TotalSeconds > _timeoutInSeconds) { context.CandidateState = new DeletedState { Reason = $"Background job has exceeded latency timeout of {_timeoutInSeconds} second(s)" }; Logger.Log( LogLevel, () => $"Background job '{context.BackgroundJob.Id}' has exceeded latency timeout of {_timeoutInSeconds} second(s) and will be deleted"); } }
public IState ApplyState(ApplyStateContext initialContext) { var filterInfo = GetFilters(initialContext.BackgroundJob.Job); var electFilters = filterInfo.ElectStateFilters; var applyFilters = filterInfo.ApplyStateFilters; // Electing a a state var electContext = new ElectStateContext(initialContext); foreach (var filter in electFilters) { filter.OnStateElection(electContext); } foreach (var state in electContext.TraversedStates) { initialContext.Transaction.AddJobState(electContext.BackgroundJob.Id, state); } // Applying the elected state var context = new ApplyStateContext(initialContext.Transaction, electContext) { JobExpirationTimeout = initialContext.JobExpirationTimeout }; foreach (var filter in applyFilters) { filter.OnStateUnapplied(context, context.Transaction); } foreach (var filter in applyFilters) { filter.OnStateApplied(context, context.Transaction); } return _innerStateMachine.ApplyState(context); }
/// <summary> /// Schedules the job to run again later. See <see cref="SecondsToDelay"/>. /// </summary> /// <param name="context">The state context.</param> /// <param name="retryAttempt">The count of retry attempts made so far.</param> /// <param name="failedState">Object which contains details about the current failed state.</param> private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState) { var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt)); context.SetJobParameter("RetryCount", retryAttempt); // If attempt number is less than max attempts, we should // schedule the job to run again later. context.CandidateState = new ScheduledState(delay) { Reason = String.Format("Retry attempt {0} of {1}", retryAttempt, Attempts) }; if (LogEvents) { Logger.WarnFormat( "Failed to process the job '{0}': an exception occurred. Retry attempt {1} of {2} will be performed in {3}.", failedState.Exception, context.JobId, retryAttempt, Attempts, delay); } }
public void OnStateElection(ElectStateContext context) { if (context.CandidateState.Name != FailedState.StateName) { // This filter accepts only failed job state. return; } var retryCount = context.GetJobParameter<int>("RetryCount"); if (retryCount < Attempts) { var delay = DateTime.UtcNow.AddSeconds(SecondsToDelay(retryCount)); context.SetJobParameter("RetryCount", retryCount + 1); // If attempt number is less than max attempts, we should // schedule the job to run again later. context.CandidateState = new ScheduledState(delay) { Reason = String.Format("Retry attempt {0} of {1}", retryCount + 1, Attempts) }; } }
private static void ExecuteContinuationsIfExist(ElectStateContext context) { // The following lines are being executed inside a distributed job lock, // so it is safe to get continuation list here. var continuations = GetContinuations(context.Connection, context.JobId); var nextStates = new Dictionary<string, IState>(); // Getting continuation data for all continuations – state they are waiting // for and their next state. foreach (var continuation in continuations) { if (String.IsNullOrWhiteSpace(continuation.JobId)) continue; var currentState = GetContinuaionState(context, continuation.JobId, ContinuationStateFetchTimeout); if (currentState == null) { continue; } // All continuations should be in the awaiting state. If someone changed // the state of a continuation, we should simply skip it. if (currentState.Name != AwaitingState.StateName) continue; if (continuation.Options.HasFlag(JobContinuationOptions.OnlyOnSucceededState) && context.CandidateState.Name != SucceededState.StateName) { nextStates.Add(continuation.JobId, new DeletedState { Reason = "Missed continuation" }); continue; } IState nextState; try { nextState = JsonConvert.DeserializeObject<IState>( currentState.Data["NextState"], new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Objects }); } catch (Exception ex) { nextState = new FailedState(ex) { Reason = "Can not start the continuation due to de-serialization error." }; } nextStates.Add(continuation.JobId, nextState); } foreach (var tuple in nextStates) { context.StateMachine.ChangeState(tuple.Key, tuple.Value, new[] { AwaitingState.StateName }); } }
/// <summary> /// Transition the candidate state to the deleted state. /// </summary> /// <param name="context">The state context.</param> /// <param name="failedState">Object which contains details about the current failed state.</param> private void TransitionToDeleted(ElectStateContext context, FailedState failedState) { context.CandidateState = new DeletedState { Reason = String.Format("Automatic deletion after retry count exceeded {0}", Attempts) }; if (LogEvents) { Logger.WarnException( String.Format( "Failed to process the job '{0}': an exception occured. Job was automatically deleted because the retry attempt count exceeded {1}.", context.JobId, Attempts), failedState.Exception); } }
private static StateData GetContinuaionState(ElectStateContext context, string continuationJobId, TimeSpan timeout) { StateData currentState = null; var started = DateTime.UtcNow; var firstAttempt = true; while (true) { var continuationData = context.Connection.GetJobData(continuationJobId); if (continuationData == null) { Logger.Warn(String.Format( "Can not start continuation '{0}' for background job '{1}': continuation does not exist.", continuationJobId, context.JobId)); break; } currentState = context.Connection.GetStateData(continuationJobId); if (currentState != null) { break; } if (DateTime.UtcNow >= started.Add(timeout)) { throw new TimeoutException(String.Format( "Can not start continuation '{0}' for background job '{1}': timeout expired while trying to fetch continuation state.", continuationJobId, context.JobId)); } Thread.Sleep(firstAttempt ? 0 : 1); firstAttempt = false; } return currentState; }
internal virtual State ElectState( StateContext stateContext, State toState, string fromStateName, IEnumerable<IElectStateFilter> filters) { var context = new ElectStateContext( stateContext, toState, fromStateName, _connection); return context.ElectState(filters); }
private void ChangeState(StateContext context, IState toState, string oldStateName) { var electStateContext = new ElectStateContext(context, _connection, this, toState, oldStateName); _stateChangeProcess.ElectState(_connection, electStateContext); var applyStateContext = new ApplyStateContext( context, electStateContext.CandidateState, oldStateName, electStateContext.TraversedStates); ApplyState(applyStateContext); }
/// <summary> /// Transition the candidate state to the deleted state. /// </summary> /// <param name="context">The state context.</param> /// <param name="failedState">Object which contains details about the current failed state.</param> private void TransitionToDeleted(ElectStateContext context, FailedState failedState) { context.CandidateState = new DeletedState { Reason = "Exceeded the maximum number of retry attempts." }; if (LogEvents) { Logger.WarnException( String.Format( "Failed to process the job '{0}': an exception occured. Job was automatically deleted because the retry attempt count exceeded {1}.", context.BackgroundJob.Id, Attempts), failedState.Exception); } }
/// <summary> /// Transition the candidate state to the deleted state. /// </summary> /// <param name="context">The state context.</param> /// <param name="failedState">Object which contains details about the current failed state.</param> private void TransitionToDeleted(ElectStateContext context, FailedState failedState) { context.CandidateState = new DeletedState { Reason = Attempts > 0 ? "Exceeded the maximum number of retry attempts." : "Retries were disabled for this job." }; if (LogEvents) { Logger.WarnException( $"Failed to process the job '{context.BackgroundJob.Id}': an exception occured. Job was automatically deleted because the retry attempt count exceeded {Attempts}.", failedState.Exception); } }
/// <summary> /// Schedules the job to run again later. See <see cref="SecondsToDelay"/>. /// </summary> /// <param name="context">The state context.</param> /// <param name="retryAttempt">The count of retry attempts made so far.</param> /// <param name="failedState">Object which contains details about the current failed state.</param> private void ScheduleAgainLater(ElectStateContext context, int retryAttempt, FailedState failedState) { context.SetJobParameter("RetryCount", retryAttempt); var delay = TimeSpan.FromSeconds(SecondsToDelay(retryAttempt)); const int maxMessageLength = 50; var exceptionMessage = failedState.Exception.Message.Length > maxMessageLength ? failedState.Exception.Message.Substring(0, maxMessageLength - 1) + "…" : failedState.Exception.Message; // If attempt number is less than max attempts, we should // schedule the job to run again later. context.CandidateState = new ScheduledState(delay) { Reason = $"Retry attempt {retryAttempt} of {Attempts}: {exceptionMessage}" }; if (LogEvents) { Logger.WarnException( $"Failed to process the job '{context.BackgroundJob.Id}': an exception occurred. Retry attempt {retryAttempt} of {Attempts} will be performed in {delay}.", failedState.Exception); } }