/// <inheritdoc />
public void Execute(BackgroundProcessContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
var jobsEnqueued = 0;
while (EnqueueNextScheduledJob(context))
{
jobsEnqueued++;
if (context.IsShutdownRequested)
{
break;
}
}
if (jobsEnqueued != 0)
{
Logger.Info($"{jobsEnqueued} scheduled job(s) enqueued.");
}
if (_schedulerEvent == null)
{
context.Wait(_pollingDelay);
}
else
{
WaitHandle.WaitAny(new[] { context.CancellationToken.WaitHandle, _schedulerEvent }, _pollingDelay);
}
}
private static void RunProcess(IServerProcess process, BackgroundProcessContext context)
{
// Long-running tasks are based on custom threads (not threadpool ones) as in
// .NET Framework 4.5, so we can try to set custom thread name to simplify the
// debugging experience.
TrySetThreadName(process.ToString());
// LogProvider.GetLogger does not throw any exception, that is why we are not
// using the `try` statement here. It does not return `null` value as well.
var logger = LogProvider.GetLogger(process.GetProcessType());
logger.Debug($"Background process '{process}' started.");
try
{
process.Execute(context);
}
catch (Exception ex)
{
if (ex is OperationCanceledException && context.IsShutdownRequested)
{
// Graceful shutdown
logger.Trace($"Background process '{process}' was stopped due to a shutdown request.");
}
else
{
logger.FatalException(
$"Fatal error occurred during execution of '{process}' process. It will be stopped. See the exception for details.",
ex);
}
}
logger.Debug($"Background process '{process}' stopped.");
}
private static void RunProcess(IServerProcess process, BackgroundProcessContext context)
{
// Long-running tasks are based on custom threads (not threadpool ones) as in
// .NET Framework 4.5, so we can try to set custom thread name to simplify the
// debugging experience.
TrySetThreadName(process.ToString());
// LogProvider.GetLogger does not throw any exception, that is why we are not
// using the `try` statement here. It does not return `null` value as well.
var logger = LogProvider.GetLogger(process.GetProcessType());
logger.DebugFormat("Background process '{0}' started.", process);
try
{
process.Execute(context);
}
catch (OperationCanceledException)
{
}
catch (Exception ex)
{
logger.FatalException(
String.Format(
"Fatal error occurred during execution of '{0}' process. It will be stopped. See the exception for details.",
process),
ex);
}
logger.DebugFormat("Background process '{0}' stopped.", process);
}
/// <inheritdoc />
public void Execute(BackgroundProcessContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
int jobsProcessed;
do
{
jobsProcessed = EnqueueNextRecurringJobs(context);
if (jobsProcessed != 0)
{
_logger.Debug($"{jobsProcessed} recurring job(s) processed by scheduler.");
}
} while (jobsProcessed > 0 && !context.IsStopping);
if (_pollingDelay > TimeSpan.Zero)
{
context.Wait(_pollingDelay);
}
else
{
var now = _nowFactory();
context.Wait(now.AddMilliseconds(-now.Millisecond).AddSeconds(-now.Second).AddMinutes(1) - now);
}
}
public void Execute(BackgroundProcessContext context)
{
while (!context.IsShutdownRequested)
{
InnerProcess.Execute(context);
}
}
private static void RunProcess(IBackgroundProcess process, BackgroundProcessContext context)
{
// Long-running tasks are based on custom threads (not threadpool ones) as in
// .NET Framework 4.5, so we can try to set custom thread name to simplify the
// debugging experience.
TrySetThreadName(process.ToString());
// LogProvider.GetLogger does not throw any exception, that is why we are not
// using the `try` statement here. It does not return `null` value as well.
var logger = LogProvider.GetLogger(process.GetProcessType());
logger.Debug($"Background process '{process}' started.");
try
{
process.Execute(context);
}
catch (Exception ex)
{
if (ex is OperationCanceledException && context.IsShutdownRequested)
{
// Graceful shutdown
logger.Trace($"Background process '{process}' was stopped due to a shutdown request.");
}
else
{
logger.FatalException(
$"Fatal error occurred during execution of '{process}' process. It will be stopped. See the exception for details.",
ex);
}
}
logger.Debug($"Background process '{process}' stopped.");
}
private void ScheduleJobsForQueue(BackgroundProcessContext context, string queueName, IStorageConnection connection)
{
var recurringJobIds = connection.GetAllItemsFromSetQueue("recurring-jobs", queueName);
foreach (var recurringJobId in recurringJobIds)
{
var recurringJob = connection.GetAllEntriesFromHash(
$"recurring-job:{recurringJobId}");
if (recurringJob == null)
{
continue;
}
try
{
TryScheduleJob(context.Storage, connection, recurringJobId, recurringJob, queueName);
}
catch (JobLoadException ex)
{
Logger.WarnException(
$"Recurring job '{recurringJobId}' can not be scheduled due to job load exception.",
ex);
}
}
_throttler.Delay(context.CancellationToken);
}
public void Execute(BackgroundProcessContext context)
{
while (!context.IsShutdownRequested)
{
InnerProcess.Execute(context);
}
}
private void EnqueueBackgroundJob(BackgroundProcessContext context, IStorageConnection connection, string jobId)
{
var appliedState = _stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
jobId,
new EnqueuedState {
Reason = $"Triggered by {ToString()}"
},
new [] { ScheduledState.StateName },
CancellationToken.None,
_profiler));
if (appliedState == null)
{
// When a background job with the given id does not exist, we should
// remove its id from a schedule manually. This may happen when someone
// modifies a storage bypassing Hangfire API.
using (var transaction = connection.CreateWriteTransaction())
{
transaction.RemoveFromSet("schedule", jobId);
transaction.Commit();
}
}
}
void IBackgroundProcess.Execute(BackgroundProcessContext context)
{
using (var connection = context.Storage.GetConnection())
{
var serverContext = GetServerContext(context.Properties);
connection.AnnounceServer(context.ServerId, serverContext);
}
try
{
var tasks = _processes
.Select(WrapProcess)
.Select(process => process.CreateTask(context))
.ToArray();
Task.WaitAll(tasks);
}
finally
{
using (var connection = context.Storage.GetConnection())
{
connection.RemoveServer(context.ServerId);
}
}
}
public void Execute(BackgroundProcessContext context)
{
_logger.Trace("Checking for aborted jobs...");
using (var connection = context.Storage.GetConnection())
{
var abortedJobIds = ServerJobCancellationToken.CheckAllCancellationTokens(
context.ServerId,
connection,
context.StoppedToken);
var aborted = false;
foreach (var abortedJobId in abortedJobIds)
{
_logger.Debug($"Job {abortedJobId.Item1} was aborted on worker {abortedJobId.Item2}.");
aborted = true;
}
if (!aborted)
{
_logger.Trace("No newly aborted jobs found.");
}
}
context.Wait(_checkInterval);
}
/// <inheritdoc />
public void Execute(BackgroundProcessContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
var jobsEnqueued = 0;
while (EnqueueNextScheduledJob(context))
{
jobsEnqueued++;
if (context.IsShutdownRequested)
{
break;
}
}
if (jobsEnqueued != 0)
{
_logger.Info($"{jobsEnqueued} scheduled job(s) enqueued.");
}
context.Wait(_pollingDelay);
}
private void TryEnqueueBackgroundJob(
BackgroundProcessContext context,
IStorageConnection connection,
string recurringJobId,
DateTime now)
{
using (connection.AcquireDistributedRecurringJobLock(recurringJobId, LockTimeout))
{
var recurringJob = connection.GetRecurringJob(recurringJobId, _timeZoneResolver, now);
if (recurringJob == null)
{
RemoveRecurringJob(connection, recurringJobId);
return;
}
Exception exception;
try
{
ScheduleRecurringJob(context, connection, recurringJobId, recurringJob, now);
return;
}
catch (BackgroundJobClientException ex)
{
exception = ex.InnerException;
}
RetryRecurringJob(connection, recurringJobId, recurringJob, exception);
}
}
/// <inheritdoc />
public void Execute(BackgroundProcessContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
var jobsEnqueued = 0;
while (EnqueueNextRecurringJobs(context))
{
jobsEnqueued++;
if (context.IsStopping)
{
break;
}
}
if (jobsEnqueued != 0)
{
_logger.Debug($"{jobsEnqueued} recurring job(s) enqueued.");
}
if (_pollingDelay > TimeSpan.Zero)
{
context.Wait(_pollingDelay);
}
else
{
var now = _nowFactory();
context.Wait(now.AddMilliseconds(-now.Millisecond).AddSeconds(-now.Second).AddMinutes(1) - now);
}
}
private IState PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId)
{
try
{
var jobData = connection.GetJobData(jobId);
if (jobData == null)
{
// Job expired just after moving to a processing state. This is an
// unreal scenario, but shit happens. Returning null instead of throwing
// an exception and rescuing from en-queueing a poisoned jobId back
// to a queue.
return(null);
}
jobData.EnsureLoaded();
var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt);
using (var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, context.ExecutionId.ToString(), context.StoppedToken))
{
var performContext = new PerformContext(context.Storage, connection, backgroundJob, jobToken, _profiler);
var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
var duration = Stopwatch.StartNew();
var result = _performer.Perform(performContext);
duration.Stop();
return(new SucceededState(result, (long)latency, duration.ElapsedMilliseconds));
}
}
catch (JobAbortedException)
{
// Background job performance was aborted due to a
// state change, so its identifier should be removed
// from a queue.
return(null);
}
catch (JobPerformanceException ex)
{
return(new FailedState(ex.InnerException)
{
Reason = ex.Message
});
}
catch (Exception ex)
{
if (ex is OperationCanceledException && context.IsStopped)
{
throw;
}
return(new FailedState(ex)
{
Reason = "An exception occurred during processing of a background job."
});
}
}
private bool EnqueueNextRecurringJobs(BackgroundProcessContext context)
{
return(UseConnectionDistributedLock(context.Storage, connection =>
{
var result = false;
if (IsBatchingAvailable(connection))
{
var now = _nowFactory();
var timestamp = JobHelper.ToTimestamp(now);
var recurringJobIds = ((JobStorageConnection)connection).GetFirstByLowestScoreFromSet("recurring-jobs", 0, timestamp, BatchSize);
if (recurringJobIds == null || recurringJobIds.Count == 0)
{
return false;
}
foreach (var recurringJobId in recurringJobIds)
{
if (context.IsStopping)
{
return false;
}
if (TryEnqueueBackgroundJob(context, connection, recurringJobId, now))
{
result = true;
}
}
}
else
{
for (var i = 0; i < BatchSize; i++)
{
if (context.IsStopping)
{
return false;
}
var now = _nowFactory();
var timestamp = JobHelper.ToTimestamp(now);
var recurringJobId = connection.GetFirstByLowestScoreFromSet("recurring-jobs", 0, timestamp);
if (recurringJobId == null)
{
return false;
}
if (!TryEnqueueBackgroundJob(context, connection, recurringJobId, now))
{
return false;
}
}
}
return result;
}));
}
private IState TryChangeState(
BackgroundProcessContext context,
IStorageConnection connection,
IFetchedJob fetchedJob,
IState state,
string[] expectedStates,
CancellationToken initializeToken,
CancellationToken abortToken)
{
Exception exception = null;
abortToken.ThrowIfCancellationRequested();
for (var retryAttempt = 0; retryAttempt < _maxStateChangeAttempts; retryAttempt++)
{
try
{
return(_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
state,
expectedStates,
disableFilters: false,
initializeToken,
_profiler)));
}
catch (Exception ex)
{
_logger.DebugException(
$"State change attempt {retryAttempt + 1} of {_maxStateChangeAttempts} failed due to an error, see inner exception for details",
ex);
exception = ex;
}
abortToken.Wait(TimeSpan.FromSeconds(retryAttempt));
abortToken.ThrowIfCancellationRequested();
}
_logger.ErrorException(
$"{_maxStateChangeAttempts} state change attempt(s) failed due to an exception, moving job to the FailedState",
exception);
return(_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
new FailedState(exception)
{
Reason = $"Failed to change state to a '{state.Name}' one due to an exception after {_maxStateChangeAttempts} retry attempts"
},
expectedStates,
disableFilters: true,
initializeToken,
_profiler)));
}
public void Ctor_CorrectlyInitializes_AllTheProperties()
{
var context = new BackgroundProcessContext(_serverId, _storage.Object, _properties, _cts.Token);
Assert.Equal(_serverId, context.ServerId);
Assert.True(_properties.SequenceEqual(context.Properties));
Assert.Same(_storage.Object, context.Storage);
Assert.Equal(_cts.Token, context.CancellationToken);
}
public void Execute(BackgroundProcessContext context)
{
using (var connection = context.Storage.GetConnection())
{
connection.Heartbeat(context.ServerId);
}
context.Wait(_heartbeatInterval);
}
public void Execute(BackgroundProcessContext context)
{
using (var connection = context.Storage.GetConnection())
{
connection.Heartbeat(context.ServerId);
}
context.Wait(_heartbeatInterval);;
}
private int EnqueueNextRecurringJobs(BackgroundProcessContext context)
{
return(UseConnectionDistributedLock(context.Storage, connection =>
{
var jobsProcessed = 0;
if (IsBatchingAvailable(connection))
{
var now = _nowFactory();
var timestamp = JobHelper.ToTimestamp(now);
var recurringJobIds = ((JobStorageConnection)connection).GetFirstByLowestScoreFromSet("recurring-jobs", 0, timestamp, BatchSize);
if (recurringJobIds != null)
{
foreach (var recurringJobId in recurringJobIds)
{
if (context.IsStopping)
{
break;
}
TryEnqueueBackgroundJob(context, connection, recurringJobId, now);
jobsProcessed++;
}
}
}
else
{
for (var i = 0; i < BatchSize; i++)
{
if (context.IsStopping)
{
break;
}
var now = _nowFactory();
var timestamp = JobHelper.ToTimestamp(now);
var recurringJobId = connection.GetFirstByLowestScoreFromSet("recurring-jobs", 0, timestamp);
if (recurringJobId == null)
{
break;
}
TryEnqueueBackgroundJob(context, connection, recurringJobId, now);
jobsProcessed++;
}
}
return jobsProcessed;
}));
}
public void Execute(BackgroundProcessContext context)
{
using (var connection = context.Storage.GetConnection())
{
var serversRemoved = connection.RemoveTimedOutServers(_serverTimeout);
if (serversRemoved != 0)
{
Logger.Info($"{serversRemoved} servers were removed due to timeout");
}
}
context.Wait(_checkInterval);
}
public static Task CreateTask([NotNull] this IServerProcess process, BackgroundProcessContext context)
{
if (process == null) throw new ArgumentNullException(nameof(process));
if (!(process is IServerComponent || process is IBackgroundProcess))
{
throw new ArgumentOutOfRangeException(nameof(process), "Long-running process must be of type IServerComponent or IBackgroundProcess.");
}
return Task.Factory.StartNew(
() => RunProcess(process, context),
TaskCreationOptions.LongRunning);
}
public void Execute(BackgroundProcessContext context)
{
using (var connection = context.Storage.GetConnection())
{
var serversRemoved = connection.RemoveTimedOutServers(_serverTimeout);
if (serversRemoved != 0)
{
Logger.Info($"{serversRemoved} servers were removed due to timeout");
}
}
context.Wait(_checkInterval);
}
public void Execute(BackgroundProcessContext context)
{
_logger.Trace($"{BackgroundServerProcess.GetServerTemplate(context.ServerId)} waiting for {_interval} delay before sending a heartbeat");
context.ShutdownToken.Wait(_interval);
context.ShutdownToken.ThrowIfCancellationRequested();
try
{
using (var connection = context.Storage.GetConnection())
{
connection.Heartbeat(context.ServerId);
}
if (_faultedSince == null)
{
_logger.Debug($"{BackgroundServerProcess.GetServerTemplate(context.ServerId)} heartbeat successfully sent");
}
else
{
_logger.Info($"{BackgroundServerProcess.GetServerTemplate(context.ServerId)} is now able to continue sending heartbeats");
_faultedSince = null;
}
}
catch (BackgroundServerGoneException)
{
if (!context.ShutdownToken.IsCancellationRequested)
{
_logger.Warn($"{BackgroundServerProcess.GetServerTemplate(context.ServerId)} was considered dead by other servers, restarting...");
_requestRestart();
}
return;
}
catch (Exception ex)
{
_logger.WarnException($"{BackgroundServerProcess.GetServerTemplate(context.ServerId)} encountered an exception while sending heartbeat", ex);
if (_faultedSince == null)
{
_faultedSince = Stopwatch.StartNew();
}
if (_faultedSince.Elapsed >= _serverTimeout)
{
_logger.Error($"{BackgroundServerProcess.GetServerTemplate(context.ServerId)} will be restarted due to server time out");
_requestRestart();
return;
}
}
}
private int EnqueueNextScheduledJobs(BackgroundProcessContext context)
{
return(UseConnectionDistributedLock(context.Storage, connection =>
{
var jobsProcessed = 0;
if (IsBatchingAvailable(connection))
{
var timestamp = JobHelper.ToTimestamp(DateTime.UtcNow);
var jobIds = ((JobStorageConnection)connection).GetFirstByLowestScoreFromSet("schedule", 0, timestamp, BatchSize);
if (jobIds != null)
{
foreach (var jobId in jobIds)
{
if (context.IsStopping)
{
break;
}
EnqueueBackgroundJob(context, connection, jobId);
jobsProcessed++;
}
}
}
else
{
for (var i = 0; i < BatchSize; i++)
{
if (context.IsStopping)
{
break;
}
var timestamp = JobHelper.ToTimestamp(DateTime.UtcNow);
var jobId = connection.GetFirstByLowestScoreFromSet("schedule", 0, timestamp);
if (jobId == null)
{
break;
}
EnqueueBackgroundJob(context, connection, jobId);
jobsProcessed++;
}
}
return jobsProcessed;
}));
}
public static void Execute(this IBackgroundProcess process, BackgroundProcessContext context)
{
if (!(process is IBackgroundProcess))
{
throw new ArgumentOutOfRangeException(nameof(process), "Long-running process must be of type IServerComponent or IBackgroundProcess.");
}
var backgroundProcess = process as IBackgroundProcess;
if (backgroundProcess != null)
{
backgroundProcess.Execute(context);
}
}
private IState TryChangeState(
BackgroundProcessContext context,
IStorageConnection connection,
IFetchedJob fetchedJob,
IState state,
string[] expectedStates,
CancellationToken cancellationToken)
{
Exception exception = null;
for (var retryAttempt = 0; retryAttempt < MaxStateChangeAttempts; retryAttempt++)
{
try
{
return(_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
state,
expectedStates,
cancellationToken,
_profiler)));
}
catch (Exception ex)
{
_logger.DebugException(
String.Format("State change attempt {0} of {1} failed due to an error, see inner exception for details", retryAttempt + 1, MaxStateChangeAttempts),
ex);
exception = ex;
}
context.CancellationToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(retryAttempt));
context.CancellationToken.ThrowIfCancellationRequested();
}
return(_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
new FailedState(exception)
{
Reason = $"Failed to change state to a '{state.Name}' one due to an exception after {MaxStateChangeAttempts} retry attempts"
},
expectedStates,
cancellationToken,
_profiler)));
}
/// <summary>
/// 创建.Net运行时任务
/// </summary>
/// <param name="process"></param>
/// <param name="context"></param>
/// <returns></returns>
public static Task CreateTask([NotNull] this IServerProcess process, BackgroundProcessContext context)
{
if (process == null)
{
throw new ArgumentNullException(nameof(process));
}
if (!(process is IServerComponent || process is IBackgroundProcess))
{
throw new ArgumentOutOfRangeException(nameof(process), "Long-running process must be of type IServerComponent or IBackgroundProcess.");
}
return(Task.Factory.StartNew(
() => RunProcess(process, context),
TaskCreationOptions.LongRunning));
}
/// <inheritdoc />
public void Execute(BackgroundProcessContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_throttler.Throttle(context.CancellationToken);
UseConnectionDistributedLock(context.Storage, connection =>
{
var recurringJobIds = connection.GetAllItemsFromSet("recurring-jobs");
foreach (var recurringJobId in recurringJobIds)
{
if (context.IsShutdownRequested)
{
return;
}
var recurringJob = connection.GetAllEntriesFromHash(
$"recurring-job:{recurringJobId}");
if (recurringJob == null)
{
continue;
}
try
{
TryScheduleJob(context.Storage, connection, recurringJobId, recurringJob);
}
catch (JobLoadException ex)
{
_logger.WarnException(
$"Recurring job '{recurringJobId}' can not be scheduled due to job load exception.",
ex);
}
}
});
// The code above may be completed in less than a second. Default throttler use
// the second resolution, and without an extra delay, CPU and DB bursts may happen.
_throttler.Delay(context.CancellationToken);
}
public static void Execute(this IServerProcess process, BackgroundProcessContext context)
{
if (!(process is IServerComponent || process is IBackgroundProcess))
{
throw new ArgumentOutOfRangeException(nameof(process), "Long-running process must be of type IServerComponent or IBackgroundProcess.");
}
var backgroundProcess = process as IBackgroundProcess;
if (backgroundProcess != null)
{
backgroundProcess.Execute(context);
}
else
{
var component = (IServerComponent) process;
component.Execute(context.CancellationToken);
}
}
/// <inheritdoc />
public void Execute(BackgroundProcessContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_throttler.Throttle(context.CancellationToken);
foreach (var queueName in _queues)
{
using (var connection = context.Storage.GetConnection())
using (connection.AcquireDistributedLock($"recurring-jobs:lock:{ queueName }", LockTimeout))
{
ScheduleJobsForQueue(context, queueName, connection);
}
}
}
/// <inheritdoc />
public void Execute(BackgroundProcessContext context)
{
if (context == null) throw new ArgumentNullException(nameof(context));
int jobsProcessed;
do
{
jobsProcessed = EnqueueNextScheduledJobs(context);
if (jobsProcessed != 0)
{
_logger.Debug($"{jobsProcessed} scheduled job(s) processed by scheduler.");
}
} while (jobsProcessed > 0 && !context.IsStopping);
context.Wait(_pollingDelay);
}
private void TryEnqueueBackgroundJob(
BackgroundProcessContext context,
IStorageConnection connection,
string recurringJobId,
DateTime now)
{
try
{
using (connection.AcquireDistributedRecurringJobLock(recurringJobId, LockTimeout))
{
var recurringJob = connection.GetRecurringJob(recurringJobId, _timeZoneResolver, now);
if (recurringJob == null)
{
RemoveRecurringJob(connection, recurringJobId);
return;
}
Exception exception;
try
{
ScheduleRecurringJob(context, connection, recurringJobId, recurringJob, now);
return;
}
catch (BackgroundJobClientException ex)
{
exception = ex.InnerException;
}
RetryRecurringJob(connection, recurringJobId, recurringJob, exception);
}
}
catch (DistributedLockTimeoutException e) when(e.Resource.EndsWith($"lock:recurring-job:{recurringJobId}"))
{
// DistributedLockTimeoutException here doesn't mean that others weren't scheduled.
// It just means another Hangfire server did this work.
_logger.Log(
LogLevel.Debug,
() => $@"An exception was thrown during acquiring distributed lock the {recurringJobId} resource within {LockTimeout.TotalSeconds} seconds. The recurring job has not been handled this time.",
e);
}
}
private bool TryEnqueueBackgroundJob(
BackgroundProcessContext context,
IStorageConnection connection,
string recurringJobId,
DateTime now)
{
try
{
return(EnqueueBackgroundJob(context, connection, recurringJobId, now));
}
catch (Exception ex)
{
_logger.WarnException(
$"Recurring job '{recurringJobId}' can not be scheduled due to an exception.",
ex);
}
return(false);
}
public void Execute(BackgroundProcessContext context)
{
for (var i = 0; i <= MaxRetryAttempts; i++)
{
try
{
_innerProcess.Execute(context);
return;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
// Break the loop after the retry attempts number exceeded.
if (i >= MaxRetryAttempts - 1)
{
throw;
}
var nextTry = DelayCallback(i);
var logLevel = GetLogLevel(i);
_logger.Log(
logLevel,
() => String.Format(
"Error occurred during execution of '{0}' process. Execution will be retried (attempt {1} of {2}) in {3} seconds.",
_innerProcess,
i + 1,
MaxRetryAttempts,
nextTry),
ex);
context.Wait(nextTry);
if (context.IsShutdownRequested)
{
break;
}
}
}
}
private static void ExecuteProcess(Guid executionId, object state)
{
var tuple = (Tuple <IBackgroundProcess, BackgroundServerContext, BackgroundExecution>)state;
var serverContext = tuple.Item2;
var context = new BackgroundProcessContext(
serverContext.ServerId,
serverContext.Storage,
serverContext.Properties.ToDictionary(x => x.Key, x => x.Value),
executionId,
serverContext.StoppingToken,
serverContext.StoppedToken,
serverContext.ShutdownToken);
while (!context.IsStopping)
{
tuple.Item1.Execute(context);
tuple.Item3.NotifySucceeded();
}
}
public void Execute(BackgroundProcessContext context)
{
for (var i = 0; i <= MaxRetryAttempts; i++)
{
try
{
_innerProcess.Execute(context);
return;
}
catch (Exception ex)
{
if (ex is OperationCanceledException && context.IsShutdownRequested)
{
throw;
}
// Break the loop after the retry attempts number exceeded.
if (i >= MaxRetryAttempts - 1) throw;
var nextTry = DelayCallback(i);
var logLevel = GetLogLevel(i);
_logger.Log(
logLevel,
() => String.Format(
"Error occurred during execution of '{0}' process. Execution will be retried (attempt {1} of {2}) in {3} seconds.",
_innerProcess,
i + 1,
MaxRetryAttempts,
nextTry),
ex);
context.Wait(nextTry);
if (context.IsShutdownRequested)
{
break;
}
}
}
}
public void Execute(BackgroundProcessContext context)
{
for (var i = 0; i <= MaxRetryAttempts; i++)
{
try
{
_innerProcess.Execute(context);
return;
}
catch (Exception ex)
{
if (ex is OperationCanceledException && context.IsShutdownRequested)
{
throw;
}
// Break the loop after the retry attempts number exceeded.
if (i >= MaxRetryAttempts - 1) throw;
var nextTry = DelayCallback(i);
var logLevel = GetLogLevel(i);
_logger.Log(
logLevel,
// ReSharper disable once AccessToModifiedClosure
() => $"Error occurred during execution of '{_innerProcess}' process. Execution will be retried (attempt {i + 1} of {MaxRetryAttempts}) in {nextTry} seconds.",
ex);
context.Wait(nextTry);
if (context.IsShutdownRequested)
{
break;
}
}
}
}
/// <inheritdoc />
public void Execute(BackgroundProcessContext context)
{
if (context == null) throw new ArgumentNullException("context");
_throttler.Throttle(context.CancellationToken);
using (var connection = context.Storage.GetConnection())
using (connection.AcquireDistributedLock("recurring-jobs:lock", LockTimeout))
{
var recurringJobIds = connection.GetAllItemsFromSet("recurring-jobs");
foreach (var recurringJobId in recurringJobIds)
{
var recurringJob = connection.GetAllEntriesFromHash(
String.Format("recurring-job:{0}", recurringJobId));
if (recurringJob == null)
{
continue;
}
try
{
TryScheduleJob(context.Storage, connection, recurringJobId, recurringJob);
}
catch (JobLoadException ex)
{
Logger.WarnException(
String.Format(
"Recurring job '{0}' can not be scheduled due to job load exception.",
recurringJobId),
ex);
}
}
_throttler.Delay(context.CancellationToken);
}
}
private bool EnqueueNextScheduledJob(BackgroundProcessContext context)
{
using (var connection = context.Storage.GetConnection())
using (connection.AcquireDistributedLock("locks:schedulepoller", DefaultLockTimeout))
{
var timestamp = JobHelper.ToTimestamp(DateTime.UtcNow);
// TODO: it is very slow. Add batching.
var jobId = connection.GetFirstByLowestScoreFromSet("schedule", 0, timestamp);
if (jobId == null)
{
// No more scheduled jobs pending.
return false;
}
var appliedState = _stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
jobId,
new EnqueuedState { Reason = String.Format("Triggered by {0}", ToString()) },
ScheduledState.StateName));
if (appliedState == null)
{
// When a background job with the given id does not exist, we should
// remove its id from a schedule manually. This may happen when someone
// modifies a storage bypassing Hangfire API.
using (var transaction = connection.CreateWriteTransaction())
{
transaction.RemoveFromSet("schedule", jobId);
transaction.Commit();
}
}
return true;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="BackgroundProcessingServer"/>
/// class and immediately starts all the given background processes.
/// </summary>
/// <param name="storage"></param>
/// <param name="processes"></param>
/// <param name="properties"></param>
/// <param name="options"></param>
public BackgroundProcessingServer(
[NotNull] JobStorage storage,
[NotNull] IEnumerable<IBackgroundProcess> processes,
[NotNull] IDictionary<string, object> properties,
[NotNull] BackgroundProcessingServerOptions options)
{
if (storage == null) throw new ArgumentNullException(nameof(storage));
if (processes == null) throw new ArgumentNullException(nameof(processes));
if (properties == null) throw new ArgumentNullException(nameof(properties));
if (options == null) throw new ArgumentNullException(nameof(options));
_options = options;
_processes.AddRange(GetRequiredProcesses());
_processes.AddRange(storage.GetComponents());
_processes.AddRange(processes);
var context = new BackgroundProcessContext(
GetGloballyUniqueServerId(),
storage,
properties,
_cts.Token);
_bootstrapTask = WrapProcess(this).CreateTask(context);
}
/// <inheritdoc />
public void Execute(BackgroundProcessContext context)
{
if (context == null) throw new ArgumentNullException("context");
using (var connection = context.Storage.GetConnection())
using (var fetchedJob = connection.FetchNextJob(_queues, context.CancellationToken))
{
context.CancellationToken.ThrowIfCancellationRequested();
try
{
using (var timeoutCts = new CancellationTokenSource(JobInitializationWaitTimeout))
using (var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(
context.CancellationToken,
timeoutCts.Token))
{
var processingState = new ProcessingState(context.ServerId, _workerId);
var appliedState = _stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
processingState,
new[] { EnqueuedState.StateName, ProcessingState.StateName },
linkedCts.Token));
// Cancel job processing if the job could not be loaded, was not in the initial state expected
// or if a job filter changed the state to something other than processing state
if (appliedState == null || !appliedState.Name.Equals(ProcessingState.StateName, StringComparison.OrdinalIgnoreCase))
{
// We should re-queue a job identifier only when graceful shutdown
// initiated.
context.CancellationToken.ThrowIfCancellationRequested();
// We should forget a job in a wrong state, or when timeout exceeded.
fetchedJob.RemoveFromQueue();
return;
}
}
// Checkpoint #3. Job is in the Processing state. However, there are
// no guarantees that it was performed. We need to re-queue it even
// it was performed to guarantee that it was performed AT LEAST once.
// It will be re-queued after the JobTimeout was expired.
var state = PerformJob(context, connection, fetchedJob.JobId);
if (state != null)
{
// Ignore return value, because we should not do anything when current state is not Processing.
_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
state,
ProcessingState.StateName));
}
// Checkpoint #4. The job was performed, and it is in the one
// of the explicit states (Succeeded, Scheduled and so on).
// It should not be re-queued, but we still need to remove its
// processing information.
fetchedJob.RemoveFromQueue();
// Success point. No things must be done after previous command
// was succeeded.
}
catch (JobAbortedException)
{
fetchedJob.RemoveFromQueue();
}
catch (Exception ex)
{
Logger.DebugException("An exception occurred while processing a job. It will be re-queued.", ex);
fetchedJob.Requeue();
throw;
}
}
}
private IState PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId)
{
try
{
var jobData = connection.GetJobData(jobId);
if (jobData == null)
{
// Job expired just after moving to a processing state. This is an
// unreal scenario, but shit happens. Returning null instead of throwing
// an exception and rescuing from en-queueing a poisoned jobId back
// to a queue.
return null;
}
jobData.EnsureLoaded();
var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt);
var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, _workerId, context.CancellationToken);
var performContext = new PerformContext(connection, backgroundJob, jobToken);
var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
var duration = Stopwatch.StartNew();
var result = _performer.Perform(performContext);
duration.Stop();
return new SucceededState(result, (long) latency, duration.ElapsedMilliseconds);
}
catch (OperationCanceledException)
{
throw;
}
catch (JobPerformanceException ex)
{
return new FailedState(ex.InnerException)
{
Reason = ex.Message
};
}
catch (Exception ex)
{
return new FailedState(ex)
{
Reason = "An exception occurred during processing of a background job."
};
}
}
/// <inheritdoc />
public void Execute(BackgroundProcessContext context)
{
if (context == null) throw new ArgumentNullException("context");
var jobsEnqueued = 0;
while (EnqueueNextScheduledJob(context))
{
jobsEnqueued++;
if (context.IsShutdownRequested)
{
break;
}
}
if (jobsEnqueued != 0)
{
Logger.InfoFormat("{0} scheduled job(s) enqueued.", jobsEnqueued);
}
context.Wait(_pollingDelay);
}
/// <summary>
/// Runs aggregator
/// </summary>
/// <param name="context">Background processing context</param>
public void Execute(BackgroundProcessContext context)
{
Execute(context.CancellationToken);
}
void IBackgroundProcess.Execute(BackgroundProcessContext context)
{
using (var connection = context.Storage.GetConnection())
{
var serverContext = GetServerContext(context.Properties);
connection.AnnounceServer(context.ServerId, serverContext);
}
try
{
var tasks = _processes
.Select(WrapProcess)
.Select(process => process.CreateTask(context))
.ToArray();
Task.WaitAll(tasks);
}
finally
{
using (var connection = context.Storage.GetConnection())
{
connection.RemoveServer(context.ServerId);
}
}
}
