public void Queues_Can_Dequeue_On_Signal()
{
UseConnection((connection, storage) =>
{
var queue = CreateJobQueue(storage, false);
IFetchedJob job = null;
//as UseConnection does not support async-await we have to work with Thread.Sleep
Task.Run(() =>
{
//dequeue the job asynchronously
job = queue.Dequeue(new[] { "default" }, CreateTimingOutCancellationToken());
});
//all sleeps are possibly way to high but this ensures that any race condition is unlikely
//to ensure that the task would run
Thread.Sleep(1000);
Assert.Null(job);
//enqueue a job that does not trigger the existing queue to reevaluate its state
queue.Enqueue(connection, "default", "1");
Thread.Sleep(1000);
//the job should still be unset
Assert.Null(job);
//trigger a reevaluation
queue.FetchNextJob();
//wait for the Dequeue to execute and return the next job
Thread.Sleep(1000);
Assert.NotNull(job);
});
}
public override IFetchedJob FetchNextJob(string[] queues, CancellationToken cancellationToken)
{
if (queues == null)
{
throw new ArgumentNullException(nameof(queues));
}
if (queues.Length == 0)
{
throw new ArgumentNullException(nameof(queues));
}
IPersistentJobQueueProvider[] providers = queues.Select(q => QueueProviders.GetProvider(q))
.Distinct()
.ToArray();
if (providers.Length != 1)
{
throw new InvalidOperationException($"Multiple provider instances registered for queues: [{string.Join(", ", queues)}]. You should choose only one type of persistent queues per server instance.");
}
IPersistentJobQueue persistentQueue = providers.Single().GetJobQueue();
IFetchedJob queue = persistentQueue.Dequeue(queues, cancellationToken);
return(queue);
}
private IState TryChangeState(
BackgroundProcessContext context,
IStorageConnection connection,
IFetchedJob fetchedJob,
IState state,
string[] expectedStates,
CancellationToken initializeToken,
CancellationToken abortToken)
{
Exception exception = null;
abortToken.ThrowIfCancellationRequested();
for (var retryAttempt = 0; retryAttempt < _maxStateChangeAttempts; retryAttempt++)
{
try
{
return(_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
state,
expectedStates,
disableFilters: false,
initializeToken,
_profiler)));
}
catch (Exception ex)
{
_logger.DebugException(
$"State change attempt {retryAttempt + 1} of {_maxStateChangeAttempts} failed due to an error, see inner exception for details",
ex);
exception = ex;
}
abortToken.Wait(TimeSpan.FromSeconds(retryAttempt));
abortToken.ThrowIfCancellationRequested();
}
_logger.ErrorException(
$"{_maxStateChangeAttempts} state change attempt(s) failed due to an exception, moving job to the FailedState",
exception);
return(_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
new FailedState(exception)
{
Reason = $"Failed to change state to a '{state.Name}' one due to an exception after {_maxStateChangeAttempts} retry attempts"
},
expectedStates,
disableFilters: true,
initializeToken,
_profiler)));
}
private void Requeue(IFetchedJob fetchedJob)
{
try
{
fetchedJob.Requeue();
}
catch (Exception ex)
{
_logger.WarnException($"Failed to immediately re-queue the background job '{fetchedJob.JobId}'. Next invocation may be delayed, if invisibility timeout is used", ex);
}
}
public override IFetchedJob FetchNextJob(string[] queues, CancellationToken cancellationToken)
{
if (queues == null)
{
throw new ArgumentNullException(nameof(queues));
}
if (queues.Length == 0)
{
throw new ArgumentException($"'{nameof(queues)}' cannot be an empty list", nameof(queues));
}
var jobFetchedCancellationToken = new CancellationTokenSource();
var compositeCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, jobFetchedCancellationToken.Token);
var pollingInterval = TimeSpan.FromSeconds(5); // TODO parameterize
IFetchedJob fetchedJob = null;
Observable.Interval(pollingInterval)
.Subscribe(_ =>
{
var timeout = DateTime.UtcNow.Add(_settings.FetchNextJobTimeout.Negate());
var searchResponse = _elasticClient.Search <JobDataDto>(descr => descr
.Version()
.Size(1)
.Sort(sort => sort.Field(j => j.CreatedAt, SortOrder.Descending))
.Query(q =>
q.Terms(terms => terms.Field(j => j.Queue).Terms(queues)) &&
(
q.Bool(b => b.MustNot(mq => mq.Exists(j => j.Field(f => f.FetchedAt)))) ||
q.DateRange(dr => dr.Field(j => j.FetchedAt).GreaterThan(timeout))
)))
.ThrowIfInvalid();
if (searchResponse.Total == 1)
{
var fetchedJobDataHit = searchResponse.Hits.Single();
var jobDataVersion = fetchedJobDataHit.Version.Value;
var jobData = fetchedJobDataHit.Source;
jobData.FetchedAt = DateTime.UtcNow;
_elasticClient
.Index(jobData, descr => descr.Version(jobDataVersion))
.ThrowIfInvalid();
fetchedJob = new FetchedJob(jobData, _elasticClient);
jobFetchedCancellationToken.Cancel();
}
},
token: compositeCancellationToken.Token);
WaitHandle.WaitAll(new[] { compositeCancellationToken.Token.WaitHandle });
return(fetchedJob);
}
private IState TryChangeState(
BackgroundProcessContext context,
IStorageConnection connection,
IFetchedJob fetchedJob,
IState state,
string[] expectedStates,
CancellationToken cancellationToken)
{
Exception exception = null;
for (var retryAttempt = 0; retryAttempt < MaxStateChangeAttempts; retryAttempt++)
{
try
{
return(_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
state,
expectedStates,
cancellationToken,
_profiler)));
}
catch (Exception ex)
{
_logger.DebugException(
String.Format("State change attempt {0} of {1} failed due to an error, see inner exception for details", retryAttempt + 1, MaxStateChangeAttempts),
ex);
exception = ex;
}
context.CancellationToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(retryAttempt));
context.CancellationToken.ThrowIfCancellationRequested();
}
return(_stateChanger.ChangeState(new StateChangeContext(
context.Storage,
connection,
fetchedJob.JobId,
new FailedState(exception)
{
Reason = $"Failed to change state to a '{state.Name}' one due to an exception after {MaxStateChangeAttempts} retry attempts"
},
expectedStates,
cancellationToken,
_profiler)));
}
public void FetchNextJob_JobWaiting()
{
var cancel = new CancellationTokenSource();
IFetchedJob Job = null;
UseConnections((redis, connection) =>
{
redis.ListRightPush(Prefix + "queue:1", "job1");
var t = new Thread(() => Job = connection.FetchNextJob(new string[] { "1" }, cancel.Token));
t.IsBackground = true;
t.Start();
t.Join();
Assert.Equal("job1", Job.JobId);
});
}
public async Task FetchNextJobAsync_ReadsPast()
{
// Arrange
using (CreateScope())
{
var fixture = Create();
var job1 = new Job("data");
var job2 = new Job("data");
await fixture.StoreJobAsync(job1);
await fixture.StoreJobAsync(job2);
fixture.Context.AddRange(new JobQueue {
Job = job1
}, new JobQueue {
Job = job2
});
fixture.Context.SaveChanges();
}
// Act
IFetchedJob fJob1 = null, fJob2 = null;
using (var scope1 = CreateScope(Provider))
using (var scope2 = CreateScope(Provider))
{
var fixture1 = Create(scope1.ServiceProvider);
fJob1 = await fixture1.FetchNextJobAsync();
var fixture2 = Create(scope2.ServiceProvider);
fJob2 = await fixture2.FetchNextJobAsync();
await fJob1.RemoveFromQueueAsync();
await fJob2.RemoveFromQueueAsync();
}
// Assert
fJob1.JobId.Should().NotBe(fJob2.JobId);
using (CreateScope())
{
var fixture = Create();
fixture.Context.JobQueue.Any().Should().BeFalse();
}
}
public void FetchNextJob_WithPub()
{
var cancel = new CancellationTokenSource();
IFetchedJob Job = null;
UseConnections((redis, connection) =>
{
var t = new Thread(() => Job = connection.FetchNextJob(new string[] { "1" }, cancel.Token));
t.IsBackground = true;
t.Start();
Thread.Sleep(10); //Enough time for Redis to respond that there are no jobs in queue, and the thread to start waiting
redis.ListRightPush(Prefix + "queue:1", "job2");
redis.Publish(Prefix + "announce", "1"); //Pub to wake up thread
t.Join();
Assert.Equal("job2", Job.JobId);
});
}
