/// <inheritdoc /> public void Execute(BackgroundProcessContext context) { if (context == null) { throw new ArgumentNullException(nameof(context)); } var jobsEnqueued = 0; while (EnqueueNextScheduledJob(context)) { jobsEnqueued++; if (context.IsShutdownRequested) { break; } } if (jobsEnqueued != 0) { Logger.Info($"{jobsEnqueued} scheduled job(s) enqueued."); } if (_schedulerEvent == null) { context.Wait(_pollingDelay); } else { WaitHandle.WaitAny(new[] { context.CancellationToken.WaitHandle, _schedulerEvent }, _pollingDelay); } }
private static void RunProcess(IServerProcess process, BackgroundProcessContext context) { // Long-running tasks are based on custom threads (not threadpool ones) as in // .NET Framework 4.5, so we can try to set custom thread name to simplify the // debugging experience. TrySetThreadName(process.ToString()); // LogProvider.GetLogger does not throw any exception, that is why we are not // using the `try` statement here. It does not return `null` value as well. var logger = LogProvider.GetLogger(process.GetProcessType()); logger.Debug($"Background process '{process}' started."); try { process.Execute(context); } catch (Exception ex) { if (ex is OperationCanceledException && context.IsShutdownRequested) { // Graceful shutdown logger.Trace($"Background process '{process}' was stopped due to a shutdown request."); } else { logger.FatalException( $"Fatal error occurred during execution of '{process}' process. It will be stopped. See the exception for details.", ex); } } logger.Debug($"Background process '{process}' stopped."); }
private static void RunProcess(IServerProcess process, BackgroundProcessContext context) { // Long-running tasks are based on custom threads (not threadpool ones) as in // .NET Framework 4.5, so we can try to set custom thread name to simplify the // debugging experience. TrySetThreadName(process.ToString()); // LogProvider.GetLogger does not throw any exception, that is why we are not // using the `try` statement here. It does not return `null` value as well. var logger = LogProvider.GetLogger(process.GetProcessType()); logger.DebugFormat("Background process '{0}' started.", process); try { process.Execute(context); } catch (OperationCanceledException) { } catch (Exception ex) { logger.FatalException( String.Format( "Fatal error occurred during execution of '{0}' process. It will be stopped. See the exception for details.", process), ex); } logger.DebugFormat("Background process '{0}' stopped.", process); }
/// <inheritdoc /> public void Execute(BackgroundProcessContext context) { if (context == null) { throw new ArgumentNullException(nameof(context)); } int jobsProcessed; do { jobsProcessed = EnqueueNextRecurringJobs(context); if (jobsProcessed != 0) { _logger.Debug($"{jobsProcessed} recurring job(s) processed by scheduler."); } } while (jobsProcessed > 0 && !context.IsStopping); if (_pollingDelay > TimeSpan.Zero) { context.Wait(_pollingDelay); } else { var now = _nowFactory(); context.Wait(now.AddMilliseconds(-now.Millisecond).AddSeconds(-now.Second).AddMinutes(1) - now); } }
public void Execute(BackgroundProcessContext context) { while (!context.IsShutdownRequested) { InnerProcess.Execute(context); } }
private static void RunProcess(IBackgroundProcess process, BackgroundProcessContext context) { // Long-running tasks are based on custom threads (not threadpool ones) as in // .NET Framework 4.5, so we can try to set custom thread name to simplify the // debugging experience. TrySetThreadName(process.ToString()); // LogProvider.GetLogger does not throw any exception, that is why we are not // using the `try` statement here. It does not return `null` value as well. var logger = LogProvider.GetLogger(process.GetProcessType()); logger.Debug($"Background process '{process}' started."); try { process.Execute(context); } catch (Exception ex) { if (ex is OperationCanceledException && context.IsShutdownRequested) { // Graceful shutdown logger.Trace($"Background process '{process}' was stopped due to a shutdown request."); } else { logger.FatalException( $"Fatal error occurred during execution of '{process}' process. It will be stopped. See the exception for details.", ex); } } logger.Debug($"Background process '{process}' stopped."); }
private void ScheduleJobsForQueue(BackgroundProcessContext context, string queueName, IStorageConnection connection) { var recurringJobIds = connection.GetAllItemsFromSetQueue("recurring-jobs", queueName); foreach (var recurringJobId in recurringJobIds) { var recurringJob = connection.GetAllEntriesFromHash( $"recurring-job:{recurringJobId}"); if (recurringJob == null) { continue; } try { TryScheduleJob(context.Storage, connection, recurringJobId, recurringJob, queueName); } catch (JobLoadException ex) { Logger.WarnException( $"Recurring job '{recurringJobId}' can not be scheduled due to job load exception.", ex); } } _throttler.Delay(context.CancellationToken); }
private void EnqueueBackgroundJob(BackgroundProcessContext context, IStorageConnection connection, string jobId) { var appliedState = _stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, jobId, new EnqueuedState { Reason = $"Triggered by {ToString()}" }, new [] { ScheduledState.StateName }, CancellationToken.None, _profiler)); if (appliedState == null) { // When a background job with the given id does not exist, we should // remove its id from a schedule manually. This may happen when someone // modifies a storage bypassing Hangfire API. using (var transaction = connection.CreateWriteTransaction()) { transaction.RemoveFromSet("schedule", jobId); transaction.Commit(); } } }
void IBackgroundProcess.Execute(BackgroundProcessContext context) { using (var connection = context.Storage.GetConnection()) { var serverContext = GetServerContext(context.Properties); connection.AnnounceServer(context.ServerId, serverContext); } try { var tasks = _processes .Select(WrapProcess) .Select(process => process.CreateTask(context)) .ToArray(); Task.WaitAll(tasks); } finally { using (var connection = context.Storage.GetConnection()) { connection.RemoveServer(context.ServerId); } } }
public void Execute(BackgroundProcessContext context) { _logger.Trace("Checking for aborted jobs..."); using (var connection = context.Storage.GetConnection()) { var abortedJobIds = ServerJobCancellationToken.CheckAllCancellationTokens( context.ServerId, connection, context.StoppedToken); var aborted = false; foreach (var abortedJobId in abortedJobIds) { _logger.Debug($"Job {abortedJobId.Item1} was aborted on worker {abortedJobId.Item2}."); aborted = true; } if (!aborted) { _logger.Trace("No newly aborted jobs found."); } } context.Wait(_checkInterval); }
/// <inheritdoc /> public void Execute(BackgroundProcessContext context) { if (context == null) { throw new ArgumentNullException(nameof(context)); } var jobsEnqueued = 0; while (EnqueueNextScheduledJob(context)) { jobsEnqueued++; if (context.IsShutdownRequested) { break; } } if (jobsEnqueued != 0) { _logger.Info($"{jobsEnqueued} scheduled job(s) enqueued."); } context.Wait(_pollingDelay); }
private void TryEnqueueBackgroundJob( BackgroundProcessContext context, IStorageConnection connection, string recurringJobId, DateTime now) { using (connection.AcquireDistributedRecurringJobLock(recurringJobId, LockTimeout)) { var recurringJob = connection.GetRecurringJob(recurringJobId, _timeZoneResolver, now); if (recurringJob == null) { RemoveRecurringJob(connection, recurringJobId); return; } Exception exception; try { ScheduleRecurringJob(context, connection, recurringJobId, recurringJob, now); return; } catch (BackgroundJobClientException ex) { exception = ex.InnerException; } RetryRecurringJob(connection, recurringJobId, recurringJob, exception); } }
/// <inheritdoc /> public void Execute(BackgroundProcessContext context) { if (context == null) { throw new ArgumentNullException(nameof(context)); } var jobsEnqueued = 0; while (EnqueueNextRecurringJobs(context)) { jobsEnqueued++; if (context.IsStopping) { break; } } if (jobsEnqueued != 0) { _logger.Debug($"{jobsEnqueued} recurring job(s) enqueued."); } if (_pollingDelay > TimeSpan.Zero) { context.Wait(_pollingDelay); } else { var now = _nowFactory(); context.Wait(now.AddMilliseconds(-now.Millisecond).AddSeconds(-now.Second).AddMinutes(1) - now); } }
private IState PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId) { try { var jobData = connection.GetJobData(jobId); if (jobData == null) { // Job expired just after moving to a processing state. This is an // unreal scenario, but shit happens. Returning null instead of throwing // an exception and rescuing from en-queueing a poisoned jobId back // to a queue. return(null); } jobData.EnsureLoaded(); var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt); using (var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, context.ExecutionId.ToString(), context.StoppedToken)) { var performContext = new PerformContext(context.Storage, connection, backgroundJob, jobToken, _profiler); var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds; var duration = Stopwatch.StartNew(); var result = _performer.Perform(performContext); duration.Stop(); return(new SucceededState(result, (long)latency, duration.ElapsedMilliseconds)); } } catch (JobAbortedException) { // Background job performance was aborted due to a // state change, so its identifier should be removed // from a queue. return(null); } catch (JobPerformanceException ex) { return(new FailedState(ex.InnerException) { Reason = ex.Message }); } catch (Exception ex) { if (ex is OperationCanceledException && context.IsStopped) { throw; } return(new FailedState(ex) { Reason = "An exception occurred during processing of a background job." }); } }
private bool EnqueueNextRecurringJobs(BackgroundProcessContext context) { return(UseConnectionDistributedLock(context.Storage, connection => { var result = false; if (IsBatchingAvailable(connection)) { var now = _nowFactory(); var timestamp = JobHelper.ToTimestamp(now); var recurringJobIds = ((JobStorageConnection)connection).GetFirstByLowestScoreFromSet("recurring-jobs", 0, timestamp, BatchSize); if (recurringJobIds == null || recurringJobIds.Count == 0) { return false; } foreach (var recurringJobId in recurringJobIds) { if (context.IsStopping) { return false; } if (TryEnqueueBackgroundJob(context, connection, recurringJobId, now)) { result = true; } } } else { for (var i = 0; i < BatchSize; i++) { if (context.IsStopping) { return false; } var now = _nowFactory(); var timestamp = JobHelper.ToTimestamp(now); var recurringJobId = connection.GetFirstByLowestScoreFromSet("recurring-jobs", 0, timestamp); if (recurringJobId == null) { return false; } if (!TryEnqueueBackgroundJob(context, connection, recurringJobId, now)) { return false; } } } return result; })); }
private IState TryChangeState( BackgroundProcessContext context, IStorageConnection connection, IFetchedJob fetchedJob, IState state, string[] expectedStates, CancellationToken initializeToken, CancellationToken abortToken) { Exception exception = null; abortToken.ThrowIfCancellationRequested(); for (var retryAttempt = 0; retryAttempt < _maxStateChangeAttempts; retryAttempt++) { try { return(_stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, fetchedJob.JobId, state, expectedStates, disableFilters: false, initializeToken, _profiler))); } catch (Exception ex) { _logger.DebugException( $"State change attempt {retryAttempt + 1} of {_maxStateChangeAttempts} failed due to an error, see inner exception for details", ex); exception = ex; } abortToken.Wait(TimeSpan.FromSeconds(retryAttempt)); abortToken.ThrowIfCancellationRequested(); } _logger.ErrorException( $"{_maxStateChangeAttempts} state change attempt(s) failed due to an exception, moving job to the FailedState", exception); return(_stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, fetchedJob.JobId, new FailedState(exception) { Reason = $"Failed to change state to a '{state.Name}' one due to an exception after {_maxStateChangeAttempts} retry attempts" }, expectedStates, disableFilters: true, initializeToken, _profiler))); }
public void Ctor_CorrectlyInitializes_AllTheProperties() { var context = new BackgroundProcessContext(_serverId, _storage.Object, _properties, _cts.Token); Assert.Equal(_serverId, context.ServerId); Assert.True(_properties.SequenceEqual(context.Properties)); Assert.Same(_storage.Object, context.Storage); Assert.Equal(_cts.Token, context.CancellationToken); }
public void Execute(BackgroundProcessContext context) { using (var connection = context.Storage.GetConnection()) { connection.Heartbeat(context.ServerId); } context.Wait(_heartbeatInterval); }
public void Execute(BackgroundProcessContext context) { using (var connection = context.Storage.GetConnection()) { connection.Heartbeat(context.ServerId); } context.Wait(_heartbeatInterval);; }
private int EnqueueNextRecurringJobs(BackgroundProcessContext context) { return(UseConnectionDistributedLock(context.Storage, connection => { var jobsProcessed = 0; if (IsBatchingAvailable(connection)) { var now = _nowFactory(); var timestamp = JobHelper.ToTimestamp(now); var recurringJobIds = ((JobStorageConnection)connection).GetFirstByLowestScoreFromSet("recurring-jobs", 0, timestamp, BatchSize); if (recurringJobIds != null) { foreach (var recurringJobId in recurringJobIds) { if (context.IsStopping) { break; } TryEnqueueBackgroundJob(context, connection, recurringJobId, now); jobsProcessed++; } } } else { for (var i = 0; i < BatchSize; i++) { if (context.IsStopping) { break; } var now = _nowFactory(); var timestamp = JobHelper.ToTimestamp(now); var recurringJobId = connection.GetFirstByLowestScoreFromSet("recurring-jobs", 0, timestamp); if (recurringJobId == null) { break; } TryEnqueueBackgroundJob(context, connection, recurringJobId, now); jobsProcessed++; } } return jobsProcessed; })); }
public void Execute(BackgroundProcessContext context) { using (var connection = context.Storage.GetConnection()) { var serversRemoved = connection.RemoveTimedOutServers(_serverTimeout); if (serversRemoved != 0) { Logger.Info($"{serversRemoved} servers were removed due to timeout"); } } context.Wait(_checkInterval); }
public static Task CreateTask([NotNull] this IServerProcess process, BackgroundProcessContext context) { if (process == null) throw new ArgumentNullException(nameof(process)); if (!(process is IServerComponent || process is IBackgroundProcess)) { throw new ArgumentOutOfRangeException(nameof(process), "Long-running process must be of type IServerComponent or IBackgroundProcess."); } return Task.Factory.StartNew( () => RunProcess(process, context), TaskCreationOptions.LongRunning); }
public void Execute(BackgroundProcessContext context) { _logger.Trace($"{BackgroundServerProcess.GetServerTemplate(context.ServerId)} waiting for {_interval} delay before sending a heartbeat"); context.ShutdownToken.Wait(_interval); context.ShutdownToken.ThrowIfCancellationRequested(); try { using (var connection = context.Storage.GetConnection()) { connection.Heartbeat(context.ServerId); } if (_faultedSince == null) { _logger.Debug($"{BackgroundServerProcess.GetServerTemplate(context.ServerId)} heartbeat successfully sent"); } else { _logger.Info($"{BackgroundServerProcess.GetServerTemplate(context.ServerId)} is now able to continue sending heartbeats"); _faultedSince = null; } } catch (BackgroundServerGoneException) { if (!context.ShutdownToken.IsCancellationRequested) { _logger.Warn($"{BackgroundServerProcess.GetServerTemplate(context.ServerId)} was considered dead by other servers, restarting..."); _requestRestart(); } return; } catch (Exception ex) { _logger.WarnException($"{BackgroundServerProcess.GetServerTemplate(context.ServerId)} encountered an exception while sending heartbeat", ex); if (_faultedSince == null) { _faultedSince = Stopwatch.StartNew(); } if (_faultedSince.Elapsed >= _serverTimeout) { _logger.Error($"{BackgroundServerProcess.GetServerTemplate(context.ServerId)} will be restarted due to server time out"); _requestRestart(); return; } } }
private int EnqueueNextScheduledJobs(BackgroundProcessContext context) { return(UseConnectionDistributedLock(context.Storage, connection => { var jobsProcessed = 0; if (IsBatchingAvailable(connection)) { var timestamp = JobHelper.ToTimestamp(DateTime.UtcNow); var jobIds = ((JobStorageConnection)connection).GetFirstByLowestScoreFromSet("schedule", 0, timestamp, BatchSize); if (jobIds != null) { foreach (var jobId in jobIds) { if (context.IsStopping) { break; } EnqueueBackgroundJob(context, connection, jobId); jobsProcessed++; } } } else { for (var i = 0; i < BatchSize; i++) { if (context.IsStopping) { break; } var timestamp = JobHelper.ToTimestamp(DateTime.UtcNow); var jobId = connection.GetFirstByLowestScoreFromSet("schedule", 0, timestamp); if (jobId == null) { break; } EnqueueBackgroundJob(context, connection, jobId); jobsProcessed++; } } return jobsProcessed; })); }
public static void Execute(this IBackgroundProcess process, BackgroundProcessContext context) { if (!(process is IBackgroundProcess)) { throw new ArgumentOutOfRangeException(nameof(process), "Long-running process must be of type IServerComponent or IBackgroundProcess."); } var backgroundProcess = process as IBackgroundProcess; if (backgroundProcess != null) { backgroundProcess.Execute(context); } }
private IState TryChangeState( BackgroundProcessContext context, IStorageConnection connection, IFetchedJob fetchedJob, IState state, string[] expectedStates, CancellationToken cancellationToken) { Exception exception = null; for (var retryAttempt = 0; retryAttempt < MaxStateChangeAttempts; retryAttempt++) { try { return(_stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, fetchedJob.JobId, state, expectedStates, cancellationToken, _profiler))); } catch (Exception ex) { _logger.DebugException( String.Format("State change attempt {0} of {1} failed due to an error, see inner exception for details", retryAttempt + 1, MaxStateChangeAttempts), ex); exception = ex; } context.CancellationToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(retryAttempt)); context.CancellationToken.ThrowIfCancellationRequested(); } return(_stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, fetchedJob.JobId, new FailedState(exception) { Reason = $"Failed to change state to a '{state.Name}' one due to an exception after {MaxStateChangeAttempts} retry attempts" }, expectedStates, cancellationToken, _profiler))); }
/// <summary> /// 创建.Net运行时任务 /// </summary> /// <param name="process"></param> /// <param name="context"></param> /// <returns></returns> public static Task CreateTask([NotNull] this IServerProcess process, BackgroundProcessContext context) { if (process == null) { throw new ArgumentNullException(nameof(process)); } if (!(process is IServerComponent || process is IBackgroundProcess)) { throw new ArgumentOutOfRangeException(nameof(process), "Long-running process must be of type IServerComponent or IBackgroundProcess."); } return(Task.Factory.StartNew( () => RunProcess(process, context), TaskCreationOptions.LongRunning)); }
/// <inheritdoc /> public void Execute(BackgroundProcessContext context) { if (context == null) { throw new ArgumentNullException(nameof(context)); } _throttler.Throttle(context.CancellationToken); UseConnectionDistributedLock(context.Storage, connection => { var recurringJobIds = connection.GetAllItemsFromSet("recurring-jobs"); foreach (var recurringJobId in recurringJobIds) { if (context.IsShutdownRequested) { return; } var recurringJob = connection.GetAllEntriesFromHash( $"recurring-job:{recurringJobId}"); if (recurringJob == null) { continue; } try { TryScheduleJob(context.Storage, connection, recurringJobId, recurringJob); } catch (JobLoadException ex) { _logger.WarnException( $"Recurring job '{recurringJobId}' can not be scheduled due to job load exception.", ex); } } }); // The code above may be completed in less than a second. Default throttler use // the second resolution, and without an extra delay, CPU and DB bursts may happen. _throttler.Delay(context.CancellationToken); }
public static void Execute(this IServerProcess process, BackgroundProcessContext context) { if (!(process is IServerComponent || process is IBackgroundProcess)) { throw new ArgumentOutOfRangeException(nameof(process), "Long-running process must be of type IServerComponent or IBackgroundProcess."); } var backgroundProcess = process as IBackgroundProcess; if (backgroundProcess != null) { backgroundProcess.Execute(context); } else { var component = (IServerComponent) process; component.Execute(context.CancellationToken); } }
/// <inheritdoc /> public void Execute(BackgroundProcessContext context) { if (context == null) { throw new ArgumentNullException(nameof(context)); } _throttler.Throttle(context.CancellationToken); foreach (var queueName in _queues) { using (var connection = context.Storage.GetConnection()) using (connection.AcquireDistributedLock($"recurring-jobs:lock:{ queueName }", LockTimeout)) { ScheduleJobsForQueue(context, queueName, connection); } } }
/// <inheritdoc /> public void Execute(BackgroundProcessContext context) { if (context == null) throw new ArgumentNullException(nameof(context)); int jobsProcessed; do { jobsProcessed = EnqueueNextScheduledJobs(context); if (jobsProcessed != 0) { _logger.Debug($"{jobsProcessed} scheduled job(s) processed by scheduler."); } } while (jobsProcessed > 0 && !context.IsStopping); context.Wait(_pollingDelay); }
private void TryEnqueueBackgroundJob( BackgroundProcessContext context, IStorageConnection connection, string recurringJobId, DateTime now) { try { using (connection.AcquireDistributedRecurringJobLock(recurringJobId, LockTimeout)) { var recurringJob = connection.GetRecurringJob(recurringJobId, _timeZoneResolver, now); if (recurringJob == null) { RemoveRecurringJob(connection, recurringJobId); return; } Exception exception; try { ScheduleRecurringJob(context, connection, recurringJobId, recurringJob, now); return; } catch (BackgroundJobClientException ex) { exception = ex.InnerException; } RetryRecurringJob(connection, recurringJobId, recurringJob, exception); } } catch (DistributedLockTimeoutException e) when(e.Resource.EndsWith($"lock:recurring-job:{recurringJobId}")) { // DistributedLockTimeoutException here doesn't mean that others weren't scheduled. // It just means another Hangfire server did this work. _logger.Log( LogLevel.Debug, () => $@"An exception was thrown during acquiring distributed lock the {recurringJobId} resource within {LockTimeout.TotalSeconds} seconds. The recurring job has not been handled this time.", e); } }
private bool TryEnqueueBackgroundJob( BackgroundProcessContext context, IStorageConnection connection, string recurringJobId, DateTime now) { try { return(EnqueueBackgroundJob(context, connection, recurringJobId, now)); } catch (Exception ex) { _logger.WarnException( $"Recurring job '{recurringJobId}' can not be scheduled due to an exception.", ex); } return(false); }
public void Execute(BackgroundProcessContext context) { for (var i = 0; i <= MaxRetryAttempts; i++) { try { _innerProcess.Execute(context); return; } catch (OperationCanceledException) { throw; } catch (Exception ex) { // Break the loop after the retry attempts number exceeded. if (i >= MaxRetryAttempts - 1) { throw; } var nextTry = DelayCallback(i); var logLevel = GetLogLevel(i); _logger.Log( logLevel, () => String.Format( "Error occurred during execution of '{0}' process. Execution will be retried (attempt {1} of {2}) in {3} seconds.", _innerProcess, i + 1, MaxRetryAttempts, nextTry), ex); context.Wait(nextTry); if (context.IsShutdownRequested) { break; } } } }
private static void ExecuteProcess(Guid executionId, object state) { var tuple = (Tuple <IBackgroundProcess, BackgroundServerContext, BackgroundExecution>)state; var serverContext = tuple.Item2; var context = new BackgroundProcessContext( serverContext.ServerId, serverContext.Storage, serverContext.Properties.ToDictionary(x => x.Key, x => x.Value), executionId, serverContext.StoppingToken, serverContext.StoppedToken, serverContext.ShutdownToken); while (!context.IsStopping) { tuple.Item1.Execute(context); tuple.Item3.NotifySucceeded(); } }
public void Execute(BackgroundProcessContext context) { for (var i = 0; i <= MaxRetryAttempts; i++) { try { _innerProcess.Execute(context); return; } catch (Exception ex) { if (ex is OperationCanceledException && context.IsShutdownRequested) { throw; } // Break the loop after the retry attempts number exceeded. if (i >= MaxRetryAttempts - 1) throw; var nextTry = DelayCallback(i); var logLevel = GetLogLevel(i); _logger.Log( logLevel, () => String.Format( "Error occurred during execution of '{0}' process. Execution will be retried (attempt {1} of {2}) in {3} seconds.", _innerProcess, i + 1, MaxRetryAttempts, nextTry), ex); context.Wait(nextTry); if (context.IsShutdownRequested) { break; } } } }
public void Execute(BackgroundProcessContext context) { for (var i = 0; i <= MaxRetryAttempts; i++) { try { _innerProcess.Execute(context); return; } catch (Exception ex) { if (ex is OperationCanceledException && context.IsShutdownRequested) { throw; } // Break the loop after the retry attempts number exceeded. if (i >= MaxRetryAttempts - 1) throw; var nextTry = DelayCallback(i); var logLevel = GetLogLevel(i); _logger.Log( logLevel, // ReSharper disable once AccessToModifiedClosure () => $"Error occurred during execution of '{_innerProcess}' process. Execution will be retried (attempt {i + 1} of {MaxRetryAttempts}) in {nextTry} seconds.", ex); context.Wait(nextTry); if (context.IsShutdownRequested) { break; } } } }
/// <inheritdoc /> public void Execute(BackgroundProcessContext context) { if (context == null) throw new ArgumentNullException("context"); _throttler.Throttle(context.CancellationToken); using (var connection = context.Storage.GetConnection()) using (connection.AcquireDistributedLock("recurring-jobs:lock", LockTimeout)) { var recurringJobIds = connection.GetAllItemsFromSet("recurring-jobs"); foreach (var recurringJobId in recurringJobIds) { var recurringJob = connection.GetAllEntriesFromHash( String.Format("recurring-job:{0}", recurringJobId)); if (recurringJob == null) { continue; } try { TryScheduleJob(context.Storage, connection, recurringJobId, recurringJob); } catch (JobLoadException ex) { Logger.WarnException( String.Format( "Recurring job '{0}' can not be scheduled due to job load exception.", recurringJobId), ex); } } _throttler.Delay(context.CancellationToken); } }
private bool EnqueueNextScheduledJob(BackgroundProcessContext context) { using (var connection = context.Storage.GetConnection()) using (connection.AcquireDistributedLock("locks:schedulepoller", DefaultLockTimeout)) { var timestamp = JobHelper.ToTimestamp(DateTime.UtcNow); // TODO: it is very slow. Add batching. var jobId = connection.GetFirstByLowestScoreFromSet("schedule", 0, timestamp); if (jobId == null) { // No more scheduled jobs pending. return false; } var appliedState = _stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, jobId, new EnqueuedState { Reason = String.Format("Triggered by {0}", ToString()) }, ScheduledState.StateName)); if (appliedState == null) { // When a background job with the given id does not exist, we should // remove its id from a schedule manually. This may happen when someone // modifies a storage bypassing Hangfire API. using (var transaction = connection.CreateWriteTransaction()) { transaction.RemoveFromSet("schedule", jobId); transaction.Commit(); } } return true; } }
/// <summary> /// Initializes a new instance of the <see cref="BackgroundProcessingServer"/> /// class and immediately starts all the given background processes. /// </summary> /// <param name="storage"></param> /// <param name="processes"></param> /// <param name="properties"></param> /// <param name="options"></param> public BackgroundProcessingServer( [NotNull] JobStorage storage, [NotNull] IEnumerable<IBackgroundProcess> processes, [NotNull] IDictionary<string, object> properties, [NotNull] BackgroundProcessingServerOptions options) { if (storage == null) throw new ArgumentNullException(nameof(storage)); if (processes == null) throw new ArgumentNullException(nameof(processes)); if (properties == null) throw new ArgumentNullException(nameof(properties)); if (options == null) throw new ArgumentNullException(nameof(options)); _options = options; _processes.AddRange(GetRequiredProcesses()); _processes.AddRange(storage.GetComponents()); _processes.AddRange(processes); var context = new BackgroundProcessContext( GetGloballyUniqueServerId(), storage, properties, _cts.Token); _bootstrapTask = WrapProcess(this).CreateTask(context); }
/// <inheritdoc /> public void Execute(BackgroundProcessContext context) { if (context == null) throw new ArgumentNullException("context"); using (var connection = context.Storage.GetConnection()) using (var fetchedJob = connection.FetchNextJob(_queues, context.CancellationToken)) { context.CancellationToken.ThrowIfCancellationRequested(); try { using (var timeoutCts = new CancellationTokenSource(JobInitializationWaitTimeout)) using (var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( context.CancellationToken, timeoutCts.Token)) { var processingState = new ProcessingState(context.ServerId, _workerId); var appliedState = _stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, fetchedJob.JobId, processingState, new[] { EnqueuedState.StateName, ProcessingState.StateName }, linkedCts.Token)); // Cancel job processing if the job could not be loaded, was not in the initial state expected // or if a job filter changed the state to something other than processing state if (appliedState == null || !appliedState.Name.Equals(ProcessingState.StateName, StringComparison.OrdinalIgnoreCase)) { // We should re-queue a job identifier only when graceful shutdown // initiated. context.CancellationToken.ThrowIfCancellationRequested(); // We should forget a job in a wrong state, or when timeout exceeded. fetchedJob.RemoveFromQueue(); return; } } // Checkpoint #3. Job is in the Processing state. However, there are // no guarantees that it was performed. We need to re-queue it even // it was performed to guarantee that it was performed AT LEAST once. // It will be re-queued after the JobTimeout was expired. var state = PerformJob(context, connection, fetchedJob.JobId); if (state != null) { // Ignore return value, because we should not do anything when current state is not Processing. _stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, fetchedJob.JobId, state, ProcessingState.StateName)); } // Checkpoint #4. The job was performed, and it is in the one // of the explicit states (Succeeded, Scheduled and so on). // It should not be re-queued, but we still need to remove its // processing information. fetchedJob.RemoveFromQueue(); // Success point. No things must be done after previous command // was succeeded. } catch (JobAbortedException) { fetchedJob.RemoveFromQueue(); } catch (Exception ex) { Logger.DebugException("An exception occurred while processing a job. It will be re-queued.", ex); fetchedJob.Requeue(); throw; } } }
private IState PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId) { try { var jobData = connection.GetJobData(jobId); if (jobData == null) { // Job expired just after moving to a processing state. This is an // unreal scenario, but shit happens. Returning null instead of throwing // an exception and rescuing from en-queueing a poisoned jobId back // to a queue. return null; } jobData.EnsureLoaded(); var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt); var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, _workerId, context.CancellationToken); var performContext = new PerformContext(connection, backgroundJob, jobToken); var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds; var duration = Stopwatch.StartNew(); var result = _performer.Perform(performContext); duration.Stop(); return new SucceededState(result, (long) latency, duration.ElapsedMilliseconds); } catch (OperationCanceledException) { throw; } catch (JobPerformanceException ex) { return new FailedState(ex.InnerException) { Reason = ex.Message }; } catch (Exception ex) { return new FailedState(ex) { Reason = "An exception occurred during processing of a background job." }; } }
/// <inheritdoc /> public void Execute(BackgroundProcessContext context) { if (context == null) throw new ArgumentNullException("context"); var jobsEnqueued = 0; while (EnqueueNextScheduledJob(context)) { jobsEnqueued++; if (context.IsShutdownRequested) { break; } } if (jobsEnqueued != 0) { Logger.InfoFormat("{0} scheduled job(s) enqueued.", jobsEnqueued); } context.Wait(_pollingDelay); }
/// <summary> /// Runs aggregator /// </summary> /// <param name="context">Background processing context</param> public void Execute(BackgroundProcessContext context) { Execute(context.CancellationToken); }