private SqlServerTransactionJob DequeueUsingTransaction(string[] queues, CancellationToken cancellationToken)
{
FetchedJob fetchedJob = null;
DbTransaction transaction = null;
string fetchJobSqlTemplate =
$@"delete top (1) JQ
output DELETED.Id, DELETED.JobId, DELETED.Queue
from [{_storage.SchemaName}].JobQueue JQ with (readpast, updlock, rowlock, forceseek)
where Queue in @queues and (FetchedAt is null or FetchedAt < DATEADD(second, @timeout, GETUTCDATE()))";
using (var cancellationEvent = cancellationToken.GetCancellationEvent())
{
do
{
cancellationToken.ThrowIfCancellationRequested();
var connection = _storage.CreateAndOpenConnection();
try
{
transaction = connection.BeginTransaction(IsolationLevel.ReadCommitted);
fetchedJob = connection.Query <FetchedJob>(
fetchJobSqlTemplate,
#pragma warning disable 618
new { queues = queues, timeout = _options.InvisibilityTimeout.Negate().TotalSeconds },
#pragma warning restore 618
transaction,
commandTimeout: _storage.CommandTimeout).SingleOrDefault();
if (fetchedJob != null)
{
return(new SqlServerTransactionJob(
_storage,
connection,
transaction,
fetchedJob.JobId.ToString(CultureInfo.InvariantCulture),
fetchedJob.Queue));
}
}
finally
{
if (fetchedJob == null)
{
transaction?.Dispose();
transaction = null;
_storage.ReleaseConnection(connection);
}
}
var pollInterval = _options.QueuePollInterval > TimeSpan.Zero
? _options.QueuePollInterval
: TimeSpan.FromSeconds(1);
WaitHandle.WaitAny(new WaitHandle[] { cancellationEvent.WaitHandle, NewItemInQueueEvent }, pollInterval);
cancellationToken.ThrowIfCancellationRequested();
} while (true);
}
}
private async Task <IFetchedJob> FetchNextDelayedJobCoreAsync(string sql, object args = null)
{
FetchedJob fetchedJob = null;
var connection = _context.GetDbConnection();
var transaction = _context.Database.CurrentTransaction;
transaction = transaction ?? await _context.Database.BeginTransactionAsync(IsolationLevel.ReadCommitted);
try
{
fetchedJob =
(await connection.QueryAsync <FetchedJob>(sql, args, transaction.GetDbTransaction()))
.FirstOrDefault();
}
catch (SqlException)
{
transaction.Dispose();
throw;
}
if (fetchedJob == null)
{
transaction.Rollback();
transaction.Dispose();
return(null);
}
return(new SqlServerFetchedJob(
fetchedJob.JobId,
connection,
transaction));
}
public IFetchedJob Dequeue(string[] queues, CancellationToken cancellationToken)
{
if (queues == null)
{
throw new ArgumentNullException(nameof(queues));
}
if (queues.Length == 0)
{
throw new ArgumentException("Queue array must be non-empty.", nameof(queues));
}
FetchedJob fetchedJob = null;
DbTransaction transaction = null;
string fetchJobSqlTemplate =
$@"delete top (1) JQ
output DELETED.Id, DELETED.JobId, DELETED.Queue
from [{_storage.SchemaName}].JobQueue JQ with (readpast, updlock, rowlock, forceseek)
where Queue in @queues";
do
{
cancellationToken.ThrowIfCancellationRequested();
var connection = _storage.CreateAndOpenConnection();
try
{
transaction = connection.BeginTransaction(IsolationLevel.ReadCommitted);
fetchedJob = connection.Query <FetchedJob>(
fetchJobSqlTemplate,
new { queues = queues },
transaction,
commandTimeout: _storage.CommandTimeout).SingleOrDefault();
if (fetchedJob != null)
{
return(new SqlServerFetchedJob(
_storage,
connection,
transaction,
fetchedJob.JobId.ToString(CultureInfo.InvariantCulture),
fetchedJob.Queue));
}
}
finally
{
if (fetchedJob == null)
{
transaction?.Dispose();
transaction = null;
_storage.ReleaseConnection(connection);
}
}
WaitHandle.WaitAny(new[] { cancellationToken.WaitHandle, NewItemInQueueEvent }, _options.QueuePollInterval);
cancellationToken.ThrowIfCancellationRequested();
} while (true);
}
public IFetchedJob Dequeue(string[] queues, CancellationToken cancellationToken)
{
if (queues == null)
{
throw new ArgumentNullException("queues");
}
if (queues.Length == 0)
{
throw new ArgumentException("Queue array must be non-empty.", "queues");
}
FetchedJob fetchedJob = null;
// string fetchJobSqlTemplate = string.Format(@"
//delete top (1) from [{0}].JobQueue with (readpast, updlock, rowlock)
//output DELETED.Id, DELETED.JobId, DELETED.Queue
//where (FetchedAt is null or FetchedAt < DATEADD(second, @timeout, GETUTCDATE()))
//and Queue in @queues", _storage.GetSchemaName());
string fetchNextJobSqlTemplate = string.Format(@"
select * from [{0}.JobQueue]
where (FetchedAt is null or FetchedAt < @fetchedAt)
and Queue in @queues
limit 1", _storage.GetSchemaName());
string dequeueJobSqlTemplate = string.Format(@"
update [{0}.JobQueue] set FetchedAt = @fetchedAt where Id = @id", _storage.GetSchemaName());
do
{
cancellationToken.ThrowIfCancellationRequested();
_storage.UseConnection(connection =>
{
fetchedJob = connection.Query <FetchedJob>(
fetchNextJobSqlTemplate,
new { queues = queues, fetchedAt = DateTime.UtcNow })
.SingleOrDefault();
if (fetchedJob != null)
{
// update
connection.Execute(dequeueJobSqlTemplate,
new { id = fetchedJob.Id, fetchedAt = DateTime.UtcNow });
}
}, true);
if (fetchedJob == null)
{
cancellationToken.WaitHandle.WaitOne(_options.QueuePollInterval);
cancellationToken.ThrowIfCancellationRequested();
}
} while (fetchedJob == null);
return(new SQLiteFetchedJob(
_storage,
fetchedJob.Id,
fetchedJob.JobId.ToString(CultureInfo.InvariantCulture),
fetchedJob.Queue));
}
public void Dequeue_ShouldFetchNonTimedOutJobs_FromTheSpecifiedQueue(string queue, string jobId)
{
// clean all the documents for the container
ContainerFixture.Clean();
//arrange
Documents.Queue document = new()
{
Name = queue,
JobId = Guid.NewGuid().ToString(),
FetchedAt = DateTime.UtcNow.AddMinutes(-10),
CreatedOn = DateTime.UtcNow.AddMinutes(-1)
};
Storage.Container.CreateItemWithRetries(document, PartitionKeys.Queue);
JobQueue jobQueue = new(Storage);
jobQueue.Enqueue(queue, jobId);
//act
FetchedJob job = (FetchedJob)jobQueue.Dequeue(new[] { queue }, CancellationToken.None);
//assert
Assert.Equal(jobId, job.JobId);
Assert.Equal(queue, job.Queue);
Assert.True((job.FetchedAt !.Value - DateTime.UtcNow).TotalSeconds <= 5);
}
public void Ctor_CorrectlySets_AllInstanceProperties()
{
var fetchedJob = new FetchedJob(_connection.Object, 1, JobId, Queue);
Assert.Equal(1, fetchedJob.Id);
Assert.Equal(JobId, fetchedJob.JobId);
Assert.Equal(Queue, fetchedJob.Queue);
}
public override IFetchedJob FetchNextJob(string[] queues, CancellationToken cancellationToken)
{
if (queues == null)
{
throw new ArgumentNullException(nameof(queues));
}
if (queues.Length == 0)
{
throw new ArgumentException($"'{nameof(queues)}' cannot be an empty list", nameof(queues));
}
var jobFetchedCancellationToken = new CancellationTokenSource();
var compositeCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, jobFetchedCancellationToken.Token);
var pollingInterval = TimeSpan.FromSeconds(5); // TODO parameterize
IFetchedJob fetchedJob = null;
Observable.Interval(pollingInterval)
.Subscribe(_ =>
{
var timeout = DateTime.UtcNow.Add(_settings.FetchNextJobTimeout.Negate());
var searchResponse = _elasticClient.Search <JobDataDto>(descr => descr
.Version()
.Size(1)
.Sort(sort => sort.Field(j => j.CreatedAt, SortOrder.Descending))
.Query(q =>
q.Terms(terms => terms.Field(j => j.Queue).Terms(queues)) &&
(
q.Bool(b => b.MustNot(mq => mq.Exists(j => j.Field(f => f.FetchedAt)))) ||
q.DateRange(dr => dr.Field(j => j.FetchedAt).GreaterThan(timeout))
)))
.ThrowIfInvalid();
if (searchResponse.Total == 1)
{
var fetchedJobDataHit = searchResponse.Hits.Single();
var jobDataVersion = fetchedJobDataHit.Version.Value;
var jobData = fetchedJobDataHit.Source;
jobData.FetchedAt = DateTime.UtcNow;
_elasticClient
.Index(jobData, descr => descr.Version(jobDataVersion))
.ThrowIfInvalid();
fetchedJob = new FetchedJob(jobData, _elasticClient);
jobFetchedCancellationToken.Cancel();
}
},
token: compositeCancellationToken.Token);
WaitHandle.WaitAll(new[] { compositeCancellationToken.Token.WaitHandle });
return(fetchedJob);
}
public FluentNHibernateFetchedJobTests()
{
_fetchedJob = new FetchedJob {
Id = _id, JobId = JobId, Queue = Queue
};
var options = new FluentNHibernateStorageOptions {
UpdateSchema = false
};
_storage = new Mock <FluentNHibernateJobStorage>(ConnectionUtils.GetPersistenceConfigurer(), options);
}
public MySqlFetchedJobTests()
{
_fetchedJob = new FetchedJob()
{
Id = _id, JobId = JobId, Queue = Queue
};
_connection = new Mock <IDbConnection>();
_storageOptions = new MySqlStorageOptions {
PrepareSchemaIfNecessary = false
};
_storage = new Mock <ElasticStorage.ElasticStorage>(ConnectionUtils.GetConnectionString(), _storageOptions);
}
public OracleFetchedJobTests()
{
_fetchedJob = new FetchedJob()
{
Id = _id, JobId = JobId, Queue = Queue
};
_connection = new Mock <IDbConnection>();
var options = new OracleStorageOptions {
PrepareSchemaIfNecessary = false
};
_storage = new Mock <OracleStorage>(ConnectionUtils.GetConnectionString(), options);
}
private SqlServerTimeoutJob DequeueUsingSlidingInvisibilityTimeout(string[] queues, CancellationToken cancellationToken)
{
if (queues == null)
{
throw new ArgumentNullException(nameof(queues));
}
if (queues.Length == 0)
{
throw new ArgumentException("Queue array must be non-empty.", nameof(queues));
}
FetchedJob fetchedJob = null;
var fetchJobSqlTemplate = $@"
set transaction isolation level read committed
update top (1) JQ
set FetchedAt = GETUTCDATE()
output INSERTED.Id, INSERTED.JobId, INSERTED.Queue
from [{_storage.SchemaName}].JobQueue JQ with (readpast, updlock, rowlock, forceseek)
where Queue in @queues and
(FetchedAt is null or FetchedAt < DATEADD(second, @timeout, GETUTCDATE()))";
using (var cancellationEvent = cancellationToken.GetCancellationEvent())
{
do
{
cancellationToken.ThrowIfCancellationRequested();
_storage.UseConnection(null, connection =>
{
fetchedJob = connection
.Query <FetchedJob>(
fetchJobSqlTemplate,
new { queues = queues, timeout = _options.SlidingInvisibilityTimeout.Value.Negate().TotalSeconds })
.SingleOrDefault();
});
if (fetchedJob != null)
{
return(new SqlServerTimeoutJob(
_storage,
fetchedJob.Id,
fetchedJob.JobId.ToString(CultureInfo.InvariantCulture),
fetchedJob.Queue));
}
WaitHandle.WaitAny(new WaitHandle[] { cancellationEvent.WaitHandle, NewItemInQueueEvent }, _options.QueuePollInterval);
cancellationToken.ThrowIfCancellationRequested();
} while (true);
}
}
public void Dispose_SetsFetchedAtValueToNull_IfThereWereNoCallsToComplete()
{
UseConnection((provider, connection) =>
{
// Arrange
var id = CreateJobQueueRecord(connection, "1", "default");
var processingJob = new FetchedJob(provider, id, "1", "default");
// Act
processingJob.Dispose();
// Assert
var record = connection.Query(@"select * from """ + GetSchemaName() + @""".""jobqueue""").Single();
Assert.Null(record.fetchedat);
});
}
public void RemoveFromQueue_ReallyDeletesTheJobFromTheQueue()
{
UseConnection((provider, connection) =>
{
// Arrange
var id = CreateJobQueueRecord(connection, "1", "default");
var processingJob = new FetchedJob(provider, id, "1", "default");
// Act
processingJob.RemoveFromQueue();
// Assert
var count = connection.Query <long>(@"select count(*) from """ + GetSchemaName() + @""".""jobqueue""")
.Single();
Assert.Equal(0, count);
});
}
public void Dequeue_ShouldSetFetchedAtToNull_WhenTheJobRequeue()
{
// clean all the documents for the container
ContainerFixture.Clean();
//arrange
JobQueue jobQueue = new(Storage);
jobQueue.Enqueue("default", Guid.NewGuid().ToString());
//act
FetchedJob job = (FetchedJob)jobQueue.Dequeue(defaultQueues, CancellationToken.None);
job.Requeue();
//assert
Assert.Null(job.FetchedAt);
}
public void Dequeue_ShouldFetchAJob_FromTheSpecifiedQueue(string queue, string jobId)
{
// clean all the documents for the container
ContainerFixture.Clean();
//arrange
JobQueue jobQueue = new(Storage);
jobQueue.Enqueue("high", queue.Equals("high") ? jobId : Guid.NewGuid().ToString());
jobQueue.Enqueue("default", queue.Equals("default") ? jobId : Guid.NewGuid().ToString());
//act
FetchedJob job = (FetchedJob)jobQueue.Dequeue(new[] { queue }, CancellationToken.None);
//assert
Assert.Equal(jobId, job.JobId);
Assert.Equal(queue, job.Queue);
}
public void RemoveFromQueue_DoesNotDelete_UnrelatedJobs()
{
UseConnection((provider, connection) =>
{
// Arrange
CreateJobQueueRecord(connection, "1", "default");
CreateJobQueueRecord(connection, "1", "critical");
CreateJobQueueRecord(connection, "2", "default");
var fetchedJob = new FetchedJob(provider, 999, "1", "default");
// Act
fetchedJob.RemoveFromQueue();
// Assert
var count = connection.Query <long>(@"select count(*) from """ + GetSchemaName() + @""".""jobqueue""")
.Single();
Assert.Equal(3, count);
});
}
public void Dequeue_ShouldRemoveFromQueue_WhenTheJobIsRemovedFromQueue()
{
// clean all the documents for the container
ContainerFixture.Clean();
//arrange
JobQueue jobQueue = new(Storage);
jobQueue.Enqueue("default", Guid.NewGuid().ToString());
//act
FetchedJob job = (FetchedJob)jobQueue.Dequeue(defaultQueues, CancellationToken.None);
job.RemoveFromQueue();
JobQueueMonitoringApi monitoringApi = new(Storage);
(int?enqueuedCount, int?fetchedCount)data = monitoringApi.GetEnqueuedAndFetchedCount("default");
//assert
Assert.Equal(0, data.enqueuedCount);
Assert.Equal(0, data.fetchedCount);
}
public IFetchedJob Dequeue(string[] queues, CancellationToken cancellationToken)
{
if (queues == null)
{
throw new ArgumentNullException(nameof(queues));
}
if (queues.Length == 0)
{
throw new ArgumentException("Queue array must be non-empty.", nameof(queues));
}
FetchedJob fetchedJob = null;
DbTransaction transaction = null;
string fetchJobSqlTemplate =
$@"delete top (1) from [{_storage.SchemaName}].JobQueue with (readpast, updlock, rowlock)
output DELETED.Id, DELETED.JobId, DELETED.Queue
where (FetchedAt is null or FetchedAt < DATEADD(second, @timeout, GETUTCDATE()))
and Queue in @queues";
do
{
cancellationToken.ThrowIfCancellationRequested();
var connection = _storage.CreateAndOpenConnection();
try
{
transaction = connection.BeginTransaction(IsolationLevel.ReadCommitted);
fetchedJob = connection.Query <FetchedJob>(
fetchJobSqlTemplate,
#pragma warning disable 618
new { queues = queues, timeout = _options.InvisibilityTimeout.Negate().TotalSeconds },
#pragma warning restore 618
transaction).SingleOrDefault();
if (fetchedJob != null)
{
return(new SqlServerFetchedJob(
_storage,
connection,
transaction,
fetchedJob.JobId.ToString(CultureInfo.InvariantCulture),
fetchedJob.Queue));
}
}
finally
{
if (fetchedJob == null)
{
transaction?.Dispose();
transaction = null;
_storage.ReleaseConnection(connection);
}
}
WaitHandle.WaitAny(new[] { cancellationToken.WaitHandle, NewItemInQueueEvent }, _options.QueuePollInterval);
cancellationToken.ThrowIfCancellationRequested();
} while (true);
}
public IFetchedJob Dequeue(string[] queues, CancellationToken cancellationToken)
{
if (queues == null)
{
throw new ArgumentNullException("queues");
}
if (queues.Length == 0)
{
throw new ArgumentException("Queue array must be non-empty.", "queues");
}
FetchedJob fetchedJob = null;
SqlConnection connection = null;
SqlTransaction transaction = null;
string fetchJobSqlTemplate = string.Format(@"
delete top (1) from [{0}].JobQueue with (readpast, updlock, rowlock)
output DELETED.Id, DELETED.JobId, DELETED.Queue
where (FetchedAt is null or FetchedAt < DATEADD(second, @timeout, GETUTCDATE()))
and Queue in @queues {1}", _storage.GetSchemaName(), _options.EnsureJobOrder?" order by Id ASC":"");
do
{
cancellationToken.ThrowIfCancellationRequested();
connection = _storage.CreateAndOpenConnection();
transaction = connection.BeginTransaction(IsolationLevel.ReadCommitted);
try
{
fetchedJob = connection.Query <FetchedJob>(
fetchJobSqlTemplate,
new { queues = queues, timeout = _options.InvisibilityTimeout.Negate().TotalSeconds },
transaction)
.SingleOrDefault();
}
catch (SqlException)
{
transaction.Dispose();
_storage.ReleaseConnection(connection);
throw;
}
if (fetchedJob == null)
{
transaction.Rollback();
transaction.Dispose();
_storage.ReleaseConnection(connection);
cancellationToken.WaitHandle.WaitOne(_options.QueuePollInterval);
cancellationToken.ThrowIfCancellationRequested();
}
} while (fetchedJob == null);
return(new SqlServerFetchedJob(
_storage,
connection,
transaction,
fetchedJob.JobId.ToString(CultureInfo.InvariantCulture),
fetchedJob.Queue));
}
private SqlServerTransactionJob DequeueUsingTransaction(string[] queues, CancellationToken cancellationToken)
{
FetchedJob fetchedJob = null;
DbTransaction transaction = null;
string fetchJobSqlTemplate =
$@"delete top (1) JQ
output DELETED.Id, DELETED.JobId, DELETED.Queue
from [{_storage.SchemaName}].JobQueue JQ with (readpast, updlock, rowlock, forceseek)
where Queue in @queues and (FetchedAt is null or FetchedAt < DATEADD(second, @timeout, GETUTCDATE()))";
var pollInterval = _options.QueuePollInterval > TimeSpan.Zero
? _options.QueuePollInterval
: TimeSpan.FromSeconds(1);
using (var cancellationEvent = cancellationToken.GetCancellationEvent())
{
do
{
cancellationToken.ThrowIfCancellationRequested();
var connection = _storage.CreateAndOpenConnection();
try
{
transaction = connection.BeginTransaction(IsolationLevel.ReadCommitted);
fetchedJob = connection.Query <FetchedJob>(
fetchJobSqlTemplate,
#pragma warning disable 618
new { queues = queues, timeout = _options.InvisibilityTimeout.Negate().TotalSeconds },
#pragma warning restore 618
transaction,
commandTimeout: _storage.CommandTimeout).SingleOrDefault();
if (fetchedJob != null)
{
return(new SqlServerTransactionJob(
_storage,
connection,
transaction,
fetchedJob.JobId.ToString(CultureInfo.InvariantCulture),
fetchedJob.Queue));
}
else
{
// Nothing updated, just commit the empty transaction.
transaction.Commit();
}
}
catch
{
// Check connection isn't broken first, and that transaction
// can be rolled back without throwing InvalidOperationException
// on older System.Data.SqlClient in .NET Core.
// https://github.com/HangfireIO/Hangfire/issues/1494
// https://github.com/dotnet/efcore/issues/12864
if (transaction?.Connection != null)
{
transaction.Rollback();
}
throw;
}
finally
{
if (fetchedJob == null)
{
transaction?.Dispose();
transaction = null;
_storage.ReleaseConnection(connection);
}
}
WaitHandle.WaitAny(new WaitHandle[] { cancellationEvent.WaitHandle, NewItemInQueueEvent }, pollInterval);
cancellationToken.ThrowIfCancellationRequested();
} while (true);
}
}
public IFetchedJob Dequeue(string[] queues, CancellationToken cancellationToken)
{
if (queues == null)
{
throw new ArgumentNullException(nameof(queues));
}
if (queues.Length == 0)
{
throw new ArgumentException("Queue array must be non-empty.", nameof(queues));
}
FetchedJob fetchedJob = null;
// string fetchJobSqlTemplate = string.Format(@"
//delete top (1) from [{0}].JobQueue with (readpast, updlock, rowlock)
//output DELETED.Id, DELETED.JobId, DELETED.Queue
//where (FetchedAt is null or FetchedAt < DATEADD(second, @timeout, GETUTCDATE()))
//and Queue in @queues", _storage.GetSchemaName());
//Add Status flag to prevent long-run job from duplicated dequeue and execution
//ref: https://github.com/HangfireIO/Hangfire/issues/514
string fetchNextJobSqlTemplate =
$@"select * from [{_storage.SchemaName}.JobQueue]
where
(Status = 'W' and (FetchedAt is null or FetchedAt < @fetchedAt)) or
(Status = 'R' and FetchedAt < @execTimeoutChk)
and Queue in @queues
limit 1";
string dequeueJobSqlTemplate =
$@"update [{_storage.SchemaName}.JobQueue] set FetchedAt = @fetchedAt, Status = 'R' where Id = @id";
do
{
cancellationToken.ThrowIfCancellationRequested();
_storage.UseConnection(connection =>
{
using (var tran = connection.BeginTransaction())
{
fetchedJob = connection.Query <FetchedJob>(
fetchNextJobSqlTemplate,
new
{
queues = queues,
//implement FetchedAt < DATEADD(second, @timeout, GETUTCDATE())
fetchedAt = DateTime.UtcNow.AddSeconds(_options.SlidingInvisibilityTimeout.Negate()
.TotalSeconds),
execTimeoutChk =
DateTime.UtcNow.AddSeconds(_options.JobQueueExecutionTimeout.Negate().TotalSeconds)
}, tran)
.SingleOrDefault();
if (fetchedJob != null)
{
// update
connection.Execute(dequeueJobSqlTemplate,
new { id = fetchedJob.Id, fetchedAt = DateTime.UtcNow }, tran);
}
tran.Commit();
}
}, true);
if (fetchedJob == null)
{
cancellationToken.WaitHandle.WaitOne(_options.QueuePollInterval);
cancellationToken.ThrowIfCancellationRequested();
}
} while (fetchedJob == null);
return(new SQLiteFetchedJob(
_storage,
fetchedJob.Id,
fetchedJob.JobId.ToString(CultureInfo.InvariantCulture),
fetchedJob.Queue));
}
