public void Execute(CancellationToken cancellationToken)
{
using (var connection = _storage.GetConnection())
{
connection.AnnounceServer(_serverId, _context);
}
try
{
using (_supervisorFactory.Value)
{
Logger.Info("Starting server components...");
_supervisorFactory.Value.Start();
cancellationToken.WaitHandle.WaitOne();
Logger.Info("Stopping server components...");
}
}
finally
{
using (var connection = _storage.GetConnection())
{
connection.RemoveServer(_serverId);
}
}
}
public bool ChangeState(string jobId, IState state, string expectedState)
{
if (jobId == null)
{
throw new ArgumentNullException(nameof(jobId));
}
if (state == null)
{
throw new ArgumentNullException(nameof(state));
}
try
{
using (var connection = _storage.GetConnection())
{
var appliedState = _stateChanger.ChangeState(new StateChangeContext(
_storage,
connection,
jobId,
state,
expectedState != null ? new[] { expectedState } : null));
return(appliedState != null && appliedState.Name.Equals(state.Name, StringComparison.OrdinalIgnoreCase));
}
}
catch (Exception ex)
{
throw new BackgroundJobClientException("State change of a background job failed. See inner exception for details", ex);
}
}
public async Task Consume(ConsumeContext <CancelScheduledMessage> context)
{
using var connection = _jobStorage.GetConnection();
var correlationId = context.Message.TokenId.ToString("N");
if (TryRemoveJob(connection, correlationId, out var jobId))
{
LogContext.Debug?.Log("Canceled Scheduled Message: {Id} at {Timestamp}", jobId, context.Message.Timestamp);
}
else
{
LogContext.Debug?.Log("CancelScheduledMessage: no message found for {Id}", jobId);
}
}
private void CreateServer(BackgroundServerContext context)
{
_logger.Trace($"{GetServerTemplate(context.ServerId)} is announcing itself...");
var stopwatch = Stopwatch.StartNew();
using (var connection = _storage.GetConnection())
{
connection.AnnounceServer(context.ServerId, GetServerContext(_properties));
}
stopwatch.Stop();
ServerJobCancellationToken.AddServer(context.ServerId);
_logger.Info($"{GetServerTemplate(context.ServerId)} successfully announced in {stopwatch.Elapsed.TotalMilliseconds} ms");
}
private bool EnqueueNextScheduledJob()
{
using (var connection = _storage.GetConnection())
{
var timestamp = JobHelper.ToTimestamp(DateTime.UtcNow);
// TODO: it is very slow. Add batching.
var jobId = connection
.GetFirstByLowestScoreFromSet("schedule", 0, timestamp);
if (String.IsNullOrEmpty(jobId))
{
return(false);
}
var stateMachine = _stateMachineFactory.Create(connection);
var enqueuedState = new EnqueuedState
{
Reason = "Enqueued as a scheduled job"
};
stateMachine.TryToChangeState(jobId, enqueuedState, new[] { ScheduledState.StateName });
return(true);
}
}
public void Execute(CancellationToken cancellationToken)
{
while (_dateTimeProvider.CurrentDateTime.Second != 0)
{
cancellationToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(1));
cancellationToken.ThrowIfCancellationRequested();
}
using (var connection = _storage.GetConnection())
using (connection.AcquireDistributedLock("recurring-jobs:lock", LockTimeout))
{
var recurringJobIds = connection.GetAllItemsFromSet("recurring-jobs");
foreach (var recurringJobId in recurringJobIds)
{
var recurringJob = connection.GetAllEntriesFromHash(
String.Format("recurring-job:{0}", recurringJobId));
if (recurringJob == null)
{
continue;
}
try
{
TryScheduleJob(connection, recurringJobId, recurringJob);
}
catch (JobLoadException ex)
{
Logger.WarnFormat("Recurring job '{0}' can not be scheduled due to job load exception.", ex, recurringJobId);
}
}
}
}
public void Execute(CancellationToken cancellationToken)
{
_throttler.Throttle(cancellationToken);
using (var connection = _storage.GetConnection())
using (connection.AcquireDistributedLock("recurring-jobs:lock", LockTimeout))
{
var recurringJobIds = connection.GetAllItemsFromSet("recurring-jobs");
foreach (var recurringJobId in recurringJobIds)
{
var recurringJob = connection.GetAllEntriesFromHash(
String.Format("recurring-job:{0}", recurringJobId));
if (recurringJob == null)
{
continue;
}
try
{
TryScheduleJob(connection, recurringJobId, recurringJob);
}
catch (JobLoadException ex)
{
Logger.WarnFormat("Recurring job '{0}' can not be scheduled due to job load exception.", ex, recurringJobId);
}
}
_throttler.Delay(cancellationToken);
}
}
private void AddGrainState(string grainId)
{
using (var connection = _jobStorage.GetConnection())
{
connection.SetRangeInHash(_lockId, new[] { new KeyValuePair <string, string>(grainId, "false") });
}
}
public virtual List <RecurringJobDto> GetRecurringJobs()
{
using (var connection = jobStorage.GetConnection())
{
return(connection.GetRecurringJobs());
}
}
public void AddOrUpdate([NotNull] RecurringDateRangeJobOptions options)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
options.Validate();
ValidateCronExpression(options.CronExpression);
using (var connection = _storage.GetConnection())
{
var recurringJob = new Dictionary <string, string>();
var invocationData = InvocationData.Serialize(options.Job);
recurringJob["Job"] = JobHelper.ToJson(invocationData);
recurringJob["Cron"] = options.CronExpression;
recurringJob["TimeZoneId"] = options.RecurringJobOptions.TimeZone.Id;
recurringJob["Queue"] = options.RecurringJobOptions.QueueName;
var utcStartDate = options.StartDateTime.HasValue ? TimeZoneInfo.ConvertTime(options.StartDateTime.Value, options.RecurringJobOptions.TimeZone, TimeZoneInfo.Utc) : (DateTime?)null;
var utcEndDate = options.EndDateTime.HasValue ? TimeZoneInfo.ConvertTime(options.EndDateTime.Value, options.RecurringJobOptions.TimeZone, TimeZoneInfo.Utc) : (DateTime?)null;
recurringJob["StartDate"] = JobHelper.SerializeDateTime(utcStartDate ?? DateTime.MinValue);
recurringJob["EndDate"] = JobHelper.SerializeDateTime(utcEndDate ?? DateTime.MaxValue);
if (options.UseEndDateTimeComponent.HasValue)
{
recurringJob[HashKeys.UseEndDateTimeComponent] = options.UseEndDateTimeComponent.ToString();
}
var existingJob = connection.GetAllEntriesFromHash($"{PluginConstants.JobType}:{options.RecurringJobId}");
if (existingJob == null)
{
recurringJob["CreatedAt"] = JobHelper.SerializeDateTime(DateTime.UtcNow);
}
using (var transaction = connection.CreateWriteTransaction())
{
transaction.SetRangeInHash(
$"{PluginConstants.JobType}:{options.RecurringJobId}",
recurringJob);
transaction.AddToSet(PluginConstants.JobSet, options.RecurringJobId);
transaction.Commit();
}
}
}
public async Task Consume(ConsumeContext <ScheduleMessage> context)
{
var correlationId = context.Message.CorrelationId.ToString("N");
var message = HangfireScheduledMessageData.Create(context);
using var connection = _jobStorage.GetConnection();
using var transaction = connection.CreateWriteTransaction();
if (TryRemoveJob(connection, correlationId, out var jobId))
{
LogContext.Debug?.Log("Cancelled Scheduled Message: {Id}", jobId);
}
jobId = _backgroundJobClient.Schedule <ScheduleJob>(
x => x.SendMessage(message, null),
context.Message.ScheduledTime);
connection.SetRangeInHash(correlationId, new[] { new KeyValuePair <string, string>(JobIdKey, jobId) });
LogContext.Debug?.Log("Scheduled: {Id}", jobId);
}
private string CrmSyncNextExecutionAt()
{
var connection = _hangfireJobStorage.GetConnection();
var recurringJob = connection.GetAllEntriesFromHash(
$"recurring-job:{JobConfiguration.CrmSyncJobId}");
return(recurringJob?["LastExecution"]);
}
public void Execute(CancellationToken cancellationToken)
{
if (!RunningWithMono())
{
// Do not allow to run multiple servers with the same ServerId on same
// machine, fixes https://github.com/odinserj/HangFire/issues/112.
WaitHandle.WaitAny(new[] { _globalMutex, cancellationToken.WaitHandle });
cancellationToken.ThrowIfCancellationRequested();
}
try
{
using (var connection = _storage.GetConnection())
{
connection.AnnounceServer(_serverId, _context);
}
try
{
using (_supervisorFactory.Value)
{
Logger.Info("Starting server components...");
_supervisorFactory.Value.Start();
cancellationToken.WaitHandle.WaitOne();
Logger.Info("Stopping server components...");
}
}
finally
{
using (var connection = _storage.GetConnection())
{
connection.RemoveServer(_serverId);
}
}
}
finally
{
if (!RunningWithMono())
{
_globalMutex.ReleaseMutex();
}
}
}
/// <summary>
/// Returns an instance of <see cref="IConsoleApi"/>.
/// </summary>
/// <param name="storage">Job storage instance</param>
/// <returns>Console API instance</returns>
public static IConsoleApi GetConsoleApi(this JobStorage storage)
{
if (storage == null)
{
throw new ArgumentNullException(nameof(storage));
}
return(new ConsoleApi(storage.GetConnection()));
}
public void Execute(CancellationToken cancellationToken)
{
using (var connection = _storage.GetConnection())
{
connection.Heartbeat(_serverId);
}
cancellationToken.WaitHandle.WaitOne(HeartbeatInterval);
}
public TagsStorage(JobStorage jobStorage)
{
var connection = jobStorage.GetConnection();
connection = connection ?? throw new ArgumentNullException(nameof(connection));
if (!(connection is JobStorageConnection jobStorageConnection))
{
throw new NotSupportedException("Storage connection must implement JobStorageConnection");
}
Connection = jobStorageConnection;
}
public RecurringJobViewModel GetRecurringJobById(string id)
{
using var connection = _jobStorage.GetConnection();
var hash = connection.GetAllEntriesFromHash($"recurring-job:{id}");
if (hash == null)
{
return(null);
}
var _parameters = new List <KeyValuePair <string, string> >
{
KeyValuePair.Create(nameof(RecurringJobViewModel.Cron), hash["Cron"])
};
var model = new RecurringJobViewModel
{
Id = id,
Cron = hash["Cron"],
Parameters = _parameters
};
try
{
if (hash.TryGetValue("Job", out var payload) && !String.IsNullOrWhiteSpace(payload))
{
var invocationData = InvocationData.DeserializePayload(payload);
var Job = invocationData.DeserializeJob();
model.TypeName = invocationData.Type;
model.MethodName = invocationData.Method;
ParameterInfo[] parameters = Job.Method.GetParameters();
for (int i = 0; i < parameters.Length; i++)
{
_parameters.Add(KeyValuePair.Create(parameters[i].Name, Job.Args[i]?.ToString()));
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Could not load the job:{0}", id);
}
if (hash.ContainsKey("Queue"))
{
model.Queue = hash["Queue"];
_parameters.Add(KeyValuePair.Create(nameof(model.Queue), model.Queue));
}
if (hash.ContainsKey("TimeZoneId"))
{
model.TimeZoneId = hash["TimeZoneId"];
_parameters.Add(KeyValuePair.Create(nameof(model.TimeZoneId), model.TimeZoneId));
}
return(model);
}
public void Execute(CancellationToken cancellationToken)
{
using (var connection = (RedisConnection)_storage.GetConnection())
{
var queues = connection.Redis.SetMembers(_options.Prefix + "queues");
foreach (var queue in queues)
{
ProcessQueue(queue, connection);
}
}
cancellationToken.WaitHandle.WaitOne(_options.SleepTimeout);
}
private void CreateAtomState()
{
using (var connection = _jobStorage.GetConnection())
{
using var tr = connection.CreateWriteTransaction();
var atomRemainingKeys = Atom.GenerateSubAtomRemainingKeys(_atomId);
foreach (var activeSubatoms in _createdSubAtoms.Where(x => !x.Value.IsFinal))
{
tr.AddToSet(atomRemainingKeys, activeSubatoms.Key);
}
tr.Commit();
}
_client.ChangeState(_atomId, new AtomCreatedState(_atomId));
}
public HangfireJobStatistics GetJobStatistics()
{
StatisticsDto hangfireStats = _hangfireJobStorage.GetMonitoringApi().GetStatistics();
long retryJobs = _hangfireJobStorage.GetConnection().GetAllItemsFromSet(retrySetName).Count;
return(new HangfireJobStatistics
{
Failed = hangfireStats.Failed,
Enqueued = hangfireStats.Enqueued,
Scheduled = hangfireStats.Scheduled,
Processing = hangfireStats.Processing,
Succeeded = hangfireStats.Succeeded,
Retry = retryJobs
});
}
public void Execute(CancellationToken cancellationToken)
{
using (var connection = _storage.GetConnection())
{
var serversRemoved = connection.RemoveTimedOutServers(_options.ServerTimeout);
if (serversRemoved != 0)
{
Logger.Info(String.Format(
"{0} servers were removed due to timeout",
serversRemoved));
}
}
cancellationToken.WaitHandle.WaitOne(_options.CheckInterval);
}
public TagsStorage(JobStorage jobStorage)
{
_jobStorage = jobStorage;
var connection = jobStorage.GetConnection();
connection = connection ?? throw new ArgumentNullException(nameof(connection));
if (!(connection is JobStorageConnection jobStorageConnection))
{
throw new NotSupportedException("Storage connection must implement JobStorageConnection");
}
ServiceStorage = jobStorage.FindRegistration().Item2;
Connection = jobStorageConnection;
}
internal static MonitorJobStatusDto GetStatus(this MonitorJob job, JobStorage storage)
{
long currentTicks = DateTime.UtcNow.Ticks;
if (job.MonitorTime.Ticks > currentTicks)
{
return(new MonitorJobStatusDto(MonitorJobStatus.Unstarted));
}
var key = job.GetMonitorStateKey();
using (var connection = storage.GetConnection())
{
var source = connection.GetMonitorState(key);
if (source == null)
{
return(new MonitorJobStatusDto(MonitorJobStatus.Unstarted));
}
var startTick = job.MonitorTime.Ticks.ToString();
var endTick = job.NextTime.Ticks.ToString();
var lastKey = source.Keys.OrderBy(k => k)
.LastOrDefault(k => string.Compare(k, startTick) >= 0 && string.Compare(k, endTick) == -1);
if (string.IsNullOrEmpty(lastKey) && job.NextTime.Ticks < currentTicks)
{
return(new MonitorJobStatusDto(MonitorJobStatus.Unexecuted));
}
else if (!string.IsNullOrEmpty(lastKey))
{
var result = source[lastKey].Split(';');
return(new MonitorJobStatusDto
{
Status = (MonitorJobStatus)Enum.Parse(typeof(MonitorJobStatus), result[0]),
ExecutedTime = new DateTime(long.Parse(result[1])),
ExecutedJobId = result[2]
});
}
}
return(new MonitorJobStatusDto(MonitorJobStatus.Unstarted));
}
private T UseConnectionDistributedLock<T>(JobStorage storage, Func<IStorageConnection, T> action)
{
var resource = "locks:schedulepoller";
try
{
using (var connection = storage.GetConnection())
using (connection.AcquireDistributedLock(resource, DefaultLockTimeout))
{
return action(connection);
}
}
catch (DistributedLockTimeoutException e) when (e.Resource.EndsWith(resource))
{
// DistributedLockTimeoutException here doesn't mean that delayed jobs weren't enqueued.
// It just means another Hangfire server did this work.
_logger.DebugException(
$@"An exception was thrown during acquiring distributed lock on the {resource} resource within {DefaultLockTimeout.TotalSeconds} seconds. The scheduled jobs have not been handled this time.
It will be retried in {_pollingDelay.TotalSeconds} seconds",
e);
return default(T);
}
}
private bool EnqueueNextScheduledJob()
{
using (var connection = _storage.GetConnection())
using (connection.AcquireDistributedLock("locks:schedulepoller", DefaultLockTimeout))
{
var timestamp = JobHelper.ToTimestamp(DateTime.UtcNow);
// TODO: it is very slow. Add batching.
var jobId = connection
.GetFirstByLowestScoreFromSet("schedule", 0, timestamp);
if (String.IsNullOrEmpty(jobId))
{
// No more scheduled jobs pending.
return(false);
}
var stateMachine = _stateMachineFactory.Create(connection);
var enqueuedState = new EnqueuedState
{
Reason = "Triggered scheduled job"
};
if (!stateMachine.ChangeState(jobId, enqueuedState, new[] { ScheduledState.StateName }))
{
// When state change does not succeed, this means that background job
// was in a state other than Scheduled, or it was moved to a state other
// than Enqueued. We should remove the job identifier from the set in
// the first case only, but can't differentiate these cases yet.
using (var transaction = connection.CreateWriteTransaction())
{
transaction.RemoveFromSet("schedule", jobId);
transaction.Commit();
}
}
return(true);
}
}
private void UseConnectionDistributedLock(JobStorage storage, Action <IStorageConnection> action)
{
var resource = "recurring-jobs:lock";
try
{
using (var connection = storage.GetConnection())
using (connection.AcquireDistributedLock(resource, LockTimeout))
{
action(connection);
}
}
catch (DistributedLockTimeoutException e) when(e.Resource.EndsWith(resource))
{
// DistributedLockTimeoutException here doesn't mean that recurring jobs weren't scheduled.
// It just means another Hangfire server did this work.
_logger.Log(
LogLevel.Debug,
() => $@"An exception was thrown during acquiring distributed lock the {resource} resource within {LockTimeout.TotalSeconds} seconds. The recurring jobs have not been handled this time.",
e);
}
}
public void Register(CronJobBroadcast payload)
{
string MakeKey(CronJobDescription j, int i) => $"{payload.Application}@{payload.Environment}.{j.Name}#{i}";
var app = $"{payload.Application}@{payload.Environment}";
_logger.LogDebug("Looking for registered jobs for {Application}", app);
var existingJobs = _jobStorage.GetConnection().GetRecurringJobs()
.Where(j => j.Id.StartsWith(app))
.ToList();
_logger.LogDebug($"Found {existingJobs.Count} jobs. Replacing with new ones.");
foreach (var job in existingJobs)
{
_jobManager.RemoveIfExists(job.Id);
}
var jobs = payload.Jobs.ToList();
foreach (var job in jobs)
{
_logger.LogDebug("Registering {Job} for {Application}", job, payload.Application);
for (var i = 0; i < job.CronExpressions.Count; i++)
{
var cron = job.CronExpressions[i];
var jobKey = MakeKey(job, i);
_jobManager.AddOrUpdate <CronJobTriggerer>(
jobKey,
t => t.Trigger(job),
cron,
TimeZoneInfo.Local
);
}
}
_logger.LogInformation($"Registered {jobs.Count} jobs for {{Application}}.", app);
}
public List <RecurringJobDto> GetRecurringJobs()
{
var recurringJobs = _jobStorageCurrent.GetConnection().GetRecurringJobs();
return(recurringJobs);
}
private IEnumerable <RecurringJobDto> QueryRecurringJobs()
{
using var connection = _jobStorage.GetConnection();
return(connection.GetRecurringJobs().Where(x => x.Job.Type == typeof(RunHangfireWorkflowDefinitionJob)));
}
private IEnumerable <RecurringJobDto> QueryRecurringJobs() => _jobStorage.GetConnection().GetRecurringJobs().Where(x => x.Job.Type == typeof(RunHangfireWorkflowDefinitionJob));