private static JobQueueDto CreateJobQueueDto(HangfireDbContext connection, string queue, bool isFetched)
{
var state = new StateDto();
AsyncHelper.RunSync(() => connection.State.InsertOneAsync(state));
var job = new JobDto
{
CreatedAt = connection.GetServerTimeUtc(),
StateId = state.Id
};
AsyncHelper.RunSync(() => connection.Job.InsertOneAsync(job));
var jobQueue = new JobQueueDto
{
Queue = queue,
JobId = job.Id
};
if (isFetched)
{
jobQueue.FetchedAt = connection.GetServerTimeUtc().AddDays(-1);
}
AsyncHelper.RunSync(() => connection.JobQueue.InsertOneAsync(jobQueue));
return(jobQueue);
}
public IFetchedJob Dequeue(string[] queues, CancellationToken cancellationToken)
{
if (queues == null)
{
throw new ArgumentNullException(nameof(queues));
}
if (queues.Length == 0)
{
throw new ArgumentException("Queue array must be non-empty.", nameof(queues));
}
JobQueueDto fetchedJob = null;
var fetchConditions = new[]
{
Builders <JobQueueDto> .Filter.Eq(_ => _.FetchedAt, null),
Builders <JobQueueDto> .Filter.Lt(_ => _.FetchedAt, _connection.GetServerTimeUtc().AddSeconds(_options.InvisibilityTimeout.Negate().TotalSeconds))
};
var currentQueryIndex = 0;
do
{
cancellationToken.ThrowIfCancellationRequested();
FilterDefinition <JobQueueDto> fetchCondition = fetchConditions[currentQueryIndex];
foreach (var queue in queues)
{
fetchedJob = _connection.JobQueue.FindOneAndUpdate(
fetchCondition & Builders <JobQueueDto> .Filter.Eq(_ => _.Queue, queue),
Builders <JobQueueDto> .Update.Set(_ => _.FetchedAt, _connection.GetServerTimeUtc()),
new FindOneAndUpdateOptions <JobQueueDto>
{
IsUpsert = false,
ReturnDocument = ReturnDocument.After
}, cancellationToken);
if (fetchedJob != null)
{
break;
}
}
if (fetchedJob == null)
{
if (currentQueryIndex == fetchConditions.Length - 1)
{
cancellationToken.WaitHandle.WaitOne(_options.QueuePollInterval);
cancellationToken.ThrowIfCancellationRequested();
}
}
currentQueryIndex = (currentQueryIndex + 1) % fetchConditions.Length;
}while (fetchedJob == null);
return(new MongoFetchedJob(_connection, fetchedJob.Id, fetchedJob.JobId.ToString(CultureInfo.InvariantCulture), fetchedJob.Queue));
}
public IFetchedJob Dequeue(string[] queues, CancellationToken cancellationToken)
{
if (queues == null)
{
throw new ArgumentNullException("queues");
}
if (queues.Length == 0)
{
throw new ArgumentException("Queue array must be non-empty.", "queues");
}
JobQueueDto fetchedJob;
var fetchConditions = new[]
{
Query <JobQueueDto> .EQ(_ => _.FetchedAt, null),
Query <JobQueueDto> .LT(_ => _.FetchedAt, _connection.GetServerTimeUtc().AddSeconds(_options.InvisibilityTimeout.Negate().TotalSeconds))
};
var currentQueryIndex = 0;
do
{
cancellationToken.ThrowIfCancellationRequested();
fetchedJob = _connection.JobQueue
.FindAndModify(new FindAndModifyArgs
{
Query = Query.And(fetchConditions[currentQueryIndex], Query <JobQueueDto> .In(_ => _.Queue, queues)),
Update = Update <JobQueueDto> .Set(_ => _.FetchedAt, _connection.GetServerTimeUtc()),
VersionReturned = FindAndModifyDocumentVersion.Modified,
Upsert = false
})
.GetModifiedDocumentAs <JobQueueDto>();
if (fetchedJob == null)
{
if (currentQueryIndex == fetchConditions.Length - 1)
{
cancellationToken.WaitHandle.WaitOne(_options.QueuePollInterval);
cancellationToken.ThrowIfCancellationRequested();
}
}
currentQueryIndex = (currentQueryIndex + 1) % fetchConditions.Length;
}while (fetchedJob == null);
return(new MongoFetchedJob(_connection, fetchedJob.Id, fetchedJob.JobId.ToString(CultureInfo.InvariantCulture), fetchedJob.Queue));
}
public override void AnnounceServer(string serverId, ServerContext context)
{
if (serverId == null)
{
throw new ArgumentNullException("serverId");
}
if (context == null)
{
throw new ArgumentNullException("context");
}
var data = new ServerDataDto
{
WorkerCount = context.WorkerCount,
Queues = context.Queues,
StartedAt = _database.GetServerTimeUtc(),
};
_database.Server.UpdateManyAsync(Builders <ServerDto> .Filter.Eq(_ => _.Id, serverId),
Builders <ServerDto> .Update.Combine(Builders <ServerDto> .Update.Set(_ => _.Data, JobHelper.ToJson(data)), Builders <ServerDto> .Update.Set(_ => _.LastHeartbeat, _database.GetServerTimeUtc())),
new UpdateOptions {
IsUpsert = true
});
}
private Dictionary <DateTime, long> GetHourlyTimelineStats(HangfireDbContext connection, string type)
{
var endDate = connection.GetServerTimeUtc();
var dates = new List <DateTime>();
for (var i = 0; i < 24; i++)
{
dates.Add(endDate);
endDate = endDate.AddHours(-1);
}
var keys = dates.Select(x => $"stats:{type}:{x:yyyy-MM-dd-HH}").ToList();
var valuesMap = connection.Counter.Find(Builders <CounterDto> .Filter.In(_ => _.Key, keys))
.ToList()
.GroupBy(x => x.Key, x => x)
.ToDictionary(x => x.Key, x => (long)x.Count());
foreach (var key in keys.Where(key => !valuesMap.ContainsKey(key)))
{
valuesMap.Add(key, 0);
}
var result = new Dictionary <DateTime, long>();
for (var i = 0; i < dates.Count; i++)
{
var value = valuesMap[valuesMap.Keys.ElementAt(i)];
result.Add(dates[i], value);
}
return(result);
}
private JobDto CreateJobInState(HangfireDbContext database, int jobId, string stateName)
{
var job = Job.FromExpression(() => SampleMethod("wrong"));
var jobState = new StateDto
{
CreatedAt = database.GetServerTimeUtc(),
Data =
stateName == EnqueuedState.StateName
? new Dictionary <string, string> {
["EnqueuedAt"] = $"{database.GetServerTimeUtc():o}"
}
: new Dictionary <string, string>(),
JobId = jobId
};
database.State.InsertOne(jobState);
var jobDto = new JobDto
{
Id = jobId,
InvocationData = JobHelper.ToJson(InvocationData.Serialize(job)),
Arguments = "['Arguments']",
StateName = stateName,
CreatedAt = database.GetServerTimeUtc(),
StateId = jobState.Id
};
database.Job.InsertOne(jobDto);
var jobQueueDto = new JobQueueDto
{
FetchedAt = null,
Id = jobId * 10,
JobId = jobId,
Queue = DefaultQueue
};
if (stateName == FetchedStateName)
{
jobQueueDto.FetchedAt = database.GetServerTimeUtc();
}
database.JobQueue.InsertOne(jobQueueDto);
return(jobDto);
}
/// <summary>
/// Starts database heartbeat
/// </summary>
private void StartHeartBeat()
{
TimeSpan timerInterval = TimeSpan.FromMilliseconds(_options.DistributedLockLifetime.TotalMilliseconds / 5);
_heartbeatTimer = new Timer(state =>
{
try
{
_database.DistributedLock.FindOneAndUpdate(
Builders <DistributedLockDto> .Filter.Eq(_ => _.Resource, _resource) & Builders <DistributedLockDto> .Filter.Eq(_ => _.ClientId, _options.ClientId),
Builders <DistributedLockDto> .Update.Set(_ => _.Heartbeat, _database.GetServerTimeUtc()));
}
catch (Exception ex)
{
Logger.ErrorFormat("Unable to update heartbeat on the resource '{0}'", ex, _resource);
}
}, null, timerInterval, timerInterval);
}
private JobDto CreateJobInState(HangfireDbContext database, int jobId, string stateName)
{
var job = Job.FromExpression(() => SampleMethod("wrong"));
var jobState = new StateDto
{
CreatedAt = database.GetServerTimeUtc(),
Data = stateName == EnqueuedState.StateName
? string.Format(" {{ 'EnqueuedAt': '{0}' }}", database.GetServerTimeUtc())
: "{}",
JobId = jobId
};
AsyncHelper.RunSync(() => database.State.InsertOneAsync(jobState));
var jobDto = new JobDto
{
Id = jobId,
InvocationData = JobHelper.ToJson(InvocationData.Serialize(job)),
Arguments = "['Arguments']",
StateName = stateName,
CreatedAt = database.GetServerTimeUtc(),
StateId = jobState.Id
};
AsyncHelper.RunSync(() => database.Job.InsertOneAsync(jobDto));
var jobQueueDto = new JobQueueDto
{
FetchedAt = null,
Id = jobId * 10,
JobId = jobId,
Queue = DefaultQueue
};
if (stateName == FetchedStateName)
{
jobQueueDto.FetchedAt = database.GetServerTimeUtc();
}
AsyncHelper.RunSync(() => database.JobQueue.InsertOneAsync(jobQueueDto));
return(jobDto);
}
private void Acquire(TimeSpan timeout)
{
try
{
// If result is null, then it means we acquired the lock
var isLockAcquired = false;
var now = DateTime.Now;
var lockTimeoutTime = now.Add(timeout);
while (!isLockAcquired && (lockTimeoutTime >= now))
{
// Acquire the lock if it does not exist - Notice: ReturnDocument.Before
var filter = Builders <DistributedLockDto> .Filter.Eq(_ => _.Resource, _resource);
var update = Builders <DistributedLockDto> .Update.SetOnInsert(_ => _.ExpireAt, _database.GetServerTimeUtc().Add(_options.DistributedLockLifetime));
var options = new FindOneAndUpdateOptions <DistributedLockDto>
{
IsUpsert = true,
ReturnDocument = ReturnDocument.Before
};
var result = _database.DistributedLock.FindOneAndUpdate(filter, update, options);
// If result is null, then it means we acquired the lock
if (result == null)
{
isLockAcquired = true;
}
else
{
// Wait on the event. This allows us to be "woken" up sooner rather than later.
// We wait in chunks as we need to "wake-up" from time to time and poll mongo,
// in case that the lock was acquired on another machine or instance.
var eventWaitHandle = new EventWaitHandle(false, EventResetMode.AutoReset, EventWaitHandleName);
eventWaitHandle.WaitOne((int)timeout.TotalMilliseconds / 10);
now = DateTime.Now;
}
}
if (!isLockAcquired)
{
throw new DistributedLockTimeoutException($"Could not place a lock on the resource \'{_resource}\': The lock request timed out.");
}
}
catch (DistributedLockTimeoutException)
{
throw;
}
catch (Exception ex)
{
throw new MongoDistributedLockException($"Could not place a lock on the resource \'{_resource}\': Check inner exception for details.", ex);
}
}
private static int CreateJobQueueRecord(HangfireDbContext connection, string jobId, string queue)
{
var jobQueue = new JobQueueDto
{
JobId = int.Parse(jobId),
Queue = queue,
FetchedAt = connection.GetServerTimeUtc()
};
connection.JobQueue.InsertOne(jobQueue);
return(jobQueue.Id);
}
public void AnnounceServer(string serverId, ServerContext context)
{
if (serverId == null)
{
throw new ArgumentNullException("serverId");
}
if (context == null)
{
throw new ArgumentNullException("context");
}
var data = new ServerDataDto
{
WorkerCount = context.WorkerCount,
Queues = context.Queues,
StartedAt = _database.GetServerTimeUtc(),
};
_database.Server.Update(Query <ServerDto> .EQ(_ => _.Id, serverId),
Update.Combine(Update <ServerDto> .Set(_ => _.Data, JobHelper.ToJson(data)), Update <ServerDto> .Set(_ => _.LastHeartbeat, _database.GetServerTimeUtc())),
UpdateFlags.Upsert);
}
/// <summary>
/// Run expiration manager to remove outdated records
/// </summary>
/// <param name="cancellationToken">Cancellation token</param>
public void Execute(CancellationToken cancellationToken)
{
using (HangfireDbContext connection = _storage.CreateAndOpenConnection())
{
DateTime now = connection.GetServerTimeUtc();
RemoveExpiredRecord(connection.AggregatedCounter, _ => _.ExpireAt, now);
RemoveExpiredRecord(connection.Counter, _ => _.ExpireAt, now);
RemoveExpiredRecord(connection.Job, _ => _.ExpireAt, now);
RemoveExpiredRecord(connection.List, _ => _.ExpireAt, now);
RemoveExpiredRecord(connection.Set, _ => _.ExpireAt, now);
RemoveExpiredRecord(connection.Hash, _ => _.ExpireAt, now);
}
cancellationToken.WaitHandle.WaitOne(_checkInterval);
}
public void Dispose()
{
if (_completed)
{
return;
}
try
{
// Remove dead locks
_database.DistributedLock.Remove(Query.And(Query <DistributedLockDto> .EQ(_ => _.Resource, _resource),
Query <DistributedLockDto> .LT(_ => _.Heartbeat, _database.GetServerTimeUtc().Subtract(_options.DistributedLockLifetime))));
// Remove resource lock
_database.DistributedLock.FindAndModify(new FindAndModifyArgs
{
Query = Query.And(Query <DistributedLockDto> .EQ(_ => _.Resource, _resource),
Query <DistributedLockDto> .EQ(_ => _.ClientId, _options.ClientId)),
Update = Update.Combine(
Update <DistributedLockDto> .Inc(_ => _.LockCount, -1),
Update <DistributedLockDto> .Set(_ => _.Heartbeat, _database.GetServerTimeUtc())
),
Upsert = false
});
_database.DistributedLock.Remove(Query <DistributedLockDto> .LTE(_ => _.LockCount, 0));
if (_heartbeatTimer != null)
{
_heartbeatTimer.Dispose();
_heartbeatTimer = null;
}
_completed = true;
}
catch (Exception ex)
{
throw new MongoDistributedLockException(String.Format("Could not release a lock on the resource '{0}': {1}.", _resource, "Check inner exception for details"), ex);
}
}
private Dictionary <DateTime, long> GetTimelineStats(HangfireDbContext connection, string type)
{
var endDate = connection.GetServerTimeUtc().Date;
var startDate = endDate.AddDays(-7);
var dates = new List <DateTime>();
while (startDate <= endDate)
{
dates.Add(endDate);
endDate = endDate.AddDays(-1);
}
var stringDates = dates.Select(x => x.ToString("yyyy-MM-dd")).ToList();
var keys = stringDates.Select(x => $"stats:{type}:{x}").ToList();
var valuesMap = connection.AggregatedCounter
.Find(Builders <AggregatedCounterDto> .Filter.In(_ => _.Key, keys))
.ToList()
.GroupBy(x => x.Key)
.ToDictionary(x => x.Key, x => (long)x.Count());
foreach (var key in keys)
{
if (!valuesMap.ContainsKey(key))
{
valuesMap.Add(key, 0);
}
}
var result = new Dictionary <DateTime, long>();
for (var i = 0; i < stringDates.Count; i++)
{
var value = valuesMap[valuesMap.Keys.ElementAt(i)];
result.Add(dates[i], value);
}
return(result);
}
public void Execute(CancellationToken cancellationToken)
{
using (HangfireDbContext connection = _storage.CreateAndOpenConnection())
{
MongoCollection[] processedTables =
{
connection.Counter,
connection.Job,
connection.List,
connection.Set,
connection.Hash
};
DateTime now = connection.GetServerTimeUtc();
foreach (var table in processedTables)
{
Logger.DebugFormat("Removing outdated records from table '{0}'...", table.Name);
table.Remove(Query.LT("ExpireAt", now));
}
}
cancellationToken.WaitHandle.WaitOne(_checkInterval);
}
/// <summary>
/// Creates MongoDB distributed lock
/// </summary>
/// <param name="resource">Lock resource</param>
/// <param name="timeout">Lock timeout</param>
/// <param name="database">Lock database</param>
/// <param name="options">Database options</param>
/// <exception cref="MongoDistributedLockException"></exception>
public MongoDistributedLock(string resource, TimeSpan timeout, HangfireDbContext database, MongoStorageOptions options)
{
if (String.IsNullOrEmpty(resource))
{
throw new ArgumentNullException("resource");
}
if (database == null)
{
throw new ArgumentNullException("database");
}
if (options == null)
{
throw new ArgumentNullException("options");
}
_resource = resource;
_database = database;
_options = options;
try
{
// Remove dead locks
database.DistributedLock.DeleteMany(
Builders <DistributedLockDto> .Filter.Eq(_ => _.Resource, resource) &
Builders <DistributedLockDto> .Filter.Lt(_ => _.Heartbeat, database.GetServerTimeUtc().Subtract(options.DistributedLockLifetime)));
// Check lock
DateTime lockTimeoutTime = DateTime.Now.Add(timeout);
bool isLockedBySomeoneElse;
bool isFirstAttempt = true;
do
{
isLockedBySomeoneElse = database.DistributedLock
.Find(Builders <DistributedLockDto> .Filter.Eq(_ => _.Resource, resource) &
Builders <DistributedLockDto> .Filter.Ne(_ => _.ClientId, _options.ClientId))
.FirstOrDefault() != null;
if (isFirstAttempt)
{
isFirstAttempt = false;
}
else
{
Thread.Sleep((int)timeout.TotalMilliseconds / 10);
}
}while (isLockedBySomeoneElse && (lockTimeoutTime >= DateTime.Now));
// Set lock
if (isLockedBySomeoneElse == false)
{
database.DistributedLock.FindOneAndUpdate(
Builders <DistributedLockDto> .Filter.Eq(_ => _.Resource, resource),
Builders <DistributedLockDto> .Update.Combine(
Builders <DistributedLockDto> .Update.Set(_ => _.ClientId, _options.ClientId),
Builders <DistributedLockDto> .Update.Inc(_ => _.LockCount, 1),
Builders <DistributedLockDto> .Update.Set(_ => _.Heartbeat, database.GetServerTimeUtc())
),
new FindOneAndUpdateOptions <DistributedLockDto> {
IsUpsert = true
});
StartHeartBeat();
}
else
{
throw new MongoDistributedLockException(
String.Format("Could not place a lock on the resource '{0}': {1}.", _resource,
"The lock request timed out"));
}
}
catch (Exception ex)
{
if (ex is MongoDistributedLockException)
{
throw;
}
else
{
throw new MongoDistributedLockException(
String.Format("Could not place a lock on the resource '{0}': {1}.", _resource,
"Check inner exception for details"), ex);
}
}
}
public override void ExpireJob(string jobId, TimeSpan expireIn)
{
QueueCommand(x => x.Job.UpdateMany(Builders <JobDto> .Filter.Eq(_ => _.Id, int.Parse(jobId)),
Builders <JobDto> .Update.Set(_ => _.ExpireAt, _connection.GetServerTimeUtc().Add(expireIn))));
}
private void Acquire(TimeSpan timeout)
{
try
{
// If result is null, then it means we acquired the lock
var isLockAcquired = false;
var now = DateTime.Now;
var lockTimeoutTime = now.Add(timeout);
while (!isLockAcquired && (lockTimeoutTime >= now))
{
// Acquire the lock if it does not exist - Notice: ReturnDocument.Before
var filter = Builders <DistributedLockDto> .Filter.Eq(_ => _.Resource, _resource);
var update = Builders <DistributedLockDto> .Update.SetOnInsert(_ => _.ExpireAt, _database.GetServerTimeUtc().Add(_options.DistributedLockLifetime));
var options = new FindOneAndUpdateOptions <DistributedLockDto>
{
IsUpsert = true,
ReturnDocument = ReturnDocument.Before
};
var result = _database.DistributedLock.FindOneAndUpdate(filter, update, options);
// If result is null, then it means we acquired the lock
if (result == null)
{
isLockAcquired = true;
}
else
{
try
{
// Wait on the event. This allows us to be "woken" up sooner rather than later.
// We wait in chunks as we need to "wake-up" from time to time and poll mongo,
// in case that the lock was acquired on another machine or instance.
var eventWaitHandle = new EventWaitHandle(false, EventResetMode.AutoReset, EventWaitHandleName);
eventWaitHandle.WaitOne((int)timeout.TotalMilliseconds / 10);
}
catch (PlatformNotSupportedException)
{
// EventWaitHandle is not supported on UNIX systems
// https://github.com/dotnet/coreclr/pull/1387
// Instead of using a compiler directive, we catch the
// exception and handles it. This way, when EventWaitHandle
// becomes available on UNIX, we will start working.
// So in this case, we just sleep for a while and then
// check if the lock has been released.
Thread.Sleep((int)timeout.TotalMilliseconds / 10);
}
now = DateTime.Now;
}
}
if (!isLockAcquired)
{
throw new DistributedLockTimeoutException($"Could not place a lock on the resource \'{_resource}\': The lock request timed out.");
}
}
catch (DistributedLockTimeoutException)
{
throw;
}
catch (Exception ex)
{
throw new MongoDistributedLockException($"Could not place a lock on the resource \'{_resource}\': Check inner exception for details.", ex);
}
}
public void ExpireJob(string jobId, TimeSpan expireIn)
{
QueueCommand(x => x.Job.Update(Query <JobDto> .EQ(_ => _.Id, int.Parse(jobId)),
Update <JobDto> .Set(_ => _.ExpireAt, _connection.GetServerTimeUtc().Add(expireIn))));
}