public void Queues_Can_Dequeue_On_Signal() { UseConnection((connection, storage) => { var queue = CreateJobQueue(storage, false); IFetchedJob job = null; //as UseConnection does not support async-await we have to work with Thread.Sleep Task.Run(() => { //dequeue the job asynchronously job = queue.Dequeue(new[] { "default" }, CreateTimingOutCancellationToken()); }); //all sleeps are possibly way to high but this ensures that any race condition is unlikely //to ensure that the task would run Thread.Sleep(1000); Assert.Null(job); //enqueue a job that does not trigger the existing queue to reevaluate its state queue.Enqueue(connection, "default", "1"); Thread.Sleep(1000); //the job should still be unset Assert.Null(job); //trigger a reevaluation queue.FetchNextJob(); //wait for the Dequeue to execute and return the next job Thread.Sleep(1000); Assert.NotNull(job); }); }
public override IFetchedJob FetchNextJob(string[] queues, CancellationToken cancellationToken) { if (queues == null) { throw new ArgumentNullException(nameof(queues)); } if (queues.Length == 0) { throw new ArgumentNullException(nameof(queues)); } IPersistentJobQueueProvider[] providers = queues.Select(q => QueueProviders.GetProvider(q)) .Distinct() .ToArray(); if (providers.Length != 1) { throw new InvalidOperationException($"Multiple provider instances registered for queues: [{string.Join(", ", queues)}]. You should choose only one type of persistent queues per server instance."); } IPersistentJobQueue persistentQueue = providers.Single().GetJobQueue(); IFetchedJob queue = persistentQueue.Dequeue(queues, cancellationToken); return(queue); }
private IState TryChangeState( BackgroundProcessContext context, IStorageConnection connection, IFetchedJob fetchedJob, IState state, string[] expectedStates, CancellationToken initializeToken, CancellationToken abortToken) { Exception exception = null; abortToken.ThrowIfCancellationRequested(); for (var retryAttempt = 0; retryAttempt < _maxStateChangeAttempts; retryAttempt++) { try { return(_stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, fetchedJob.JobId, state, expectedStates, disableFilters: false, initializeToken, _profiler))); } catch (Exception ex) { _logger.DebugException( $"State change attempt {retryAttempt + 1} of {_maxStateChangeAttempts} failed due to an error, see inner exception for details", ex); exception = ex; } abortToken.Wait(TimeSpan.FromSeconds(retryAttempt)); abortToken.ThrowIfCancellationRequested(); } _logger.ErrorException( $"{_maxStateChangeAttempts} state change attempt(s) failed due to an exception, moving job to the FailedState", exception); return(_stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, fetchedJob.JobId, new FailedState(exception) { Reason = $"Failed to change state to a '{state.Name}' one due to an exception after {_maxStateChangeAttempts} retry attempts" }, expectedStates, disableFilters: true, initializeToken, _profiler))); }
private void Requeue(IFetchedJob fetchedJob) { try { fetchedJob.Requeue(); } catch (Exception ex) { _logger.WarnException($"Failed to immediately re-queue the background job '{fetchedJob.JobId}'. Next invocation may be delayed, if invisibility timeout is used", ex); } }
public override IFetchedJob FetchNextJob(string[] queues, CancellationToken cancellationToken) { if (queues == null) { throw new ArgumentNullException(nameof(queues)); } if (queues.Length == 0) { throw new ArgumentException($"'{nameof(queues)}' cannot be an empty list", nameof(queues)); } var jobFetchedCancellationToken = new CancellationTokenSource(); var compositeCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, jobFetchedCancellationToken.Token); var pollingInterval = TimeSpan.FromSeconds(5); // TODO parameterize IFetchedJob fetchedJob = null; Observable.Interval(pollingInterval) .Subscribe(_ => { var timeout = DateTime.UtcNow.Add(_settings.FetchNextJobTimeout.Negate()); var searchResponse = _elasticClient.Search <JobDataDto>(descr => descr .Version() .Size(1) .Sort(sort => sort.Field(j => j.CreatedAt, SortOrder.Descending)) .Query(q => q.Terms(terms => terms.Field(j => j.Queue).Terms(queues)) && ( q.Bool(b => b.MustNot(mq => mq.Exists(j => j.Field(f => f.FetchedAt)))) || q.DateRange(dr => dr.Field(j => j.FetchedAt).GreaterThan(timeout)) ))) .ThrowIfInvalid(); if (searchResponse.Total == 1) { var fetchedJobDataHit = searchResponse.Hits.Single(); var jobDataVersion = fetchedJobDataHit.Version.Value; var jobData = fetchedJobDataHit.Source; jobData.FetchedAt = DateTime.UtcNow; _elasticClient .Index(jobData, descr => descr.Version(jobDataVersion)) .ThrowIfInvalid(); fetchedJob = new FetchedJob(jobData, _elasticClient); jobFetchedCancellationToken.Cancel(); } }, token: compositeCancellationToken.Token); WaitHandle.WaitAll(new[] { compositeCancellationToken.Token.WaitHandle }); return(fetchedJob); }
private IState TryChangeState( BackgroundProcessContext context, IStorageConnection connection, IFetchedJob fetchedJob, IState state, string[] expectedStates, CancellationToken cancellationToken) { Exception exception = null; for (var retryAttempt = 0; retryAttempt < MaxStateChangeAttempts; retryAttempt++) { try { return(_stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, fetchedJob.JobId, state, expectedStates, cancellationToken, _profiler))); } catch (Exception ex) { _logger.DebugException( String.Format("State change attempt {0} of {1} failed due to an error, see inner exception for details", retryAttempt + 1, MaxStateChangeAttempts), ex); exception = ex; } context.CancellationToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(retryAttempt)); context.CancellationToken.ThrowIfCancellationRequested(); } return(_stateChanger.ChangeState(new StateChangeContext( context.Storage, connection, fetchedJob.JobId, new FailedState(exception) { Reason = $"Failed to change state to a '{state.Name}' one due to an exception after {MaxStateChangeAttempts} retry attempts" }, expectedStates, cancellationToken, _profiler))); }
public void FetchNextJob_JobWaiting() { var cancel = new CancellationTokenSource(); IFetchedJob Job = null; UseConnections((redis, connection) => { redis.ListRightPush(Prefix + "queue:1", "job1"); var t = new Thread(() => Job = connection.FetchNextJob(new string[] { "1" }, cancel.Token)); t.IsBackground = true; t.Start(); t.Join(); Assert.Equal("job1", Job.JobId); }); }
public async Task FetchNextJobAsync_ReadsPast() { // Arrange using (CreateScope()) { var fixture = Create(); var job1 = new Job("data"); var job2 = new Job("data"); await fixture.StoreJobAsync(job1); await fixture.StoreJobAsync(job2); fixture.Context.AddRange(new JobQueue { Job = job1 }, new JobQueue { Job = job2 }); fixture.Context.SaveChanges(); } // Act IFetchedJob fJob1 = null, fJob2 = null; using (var scope1 = CreateScope(Provider)) using (var scope2 = CreateScope(Provider)) { var fixture1 = Create(scope1.ServiceProvider); fJob1 = await fixture1.FetchNextJobAsync(); var fixture2 = Create(scope2.ServiceProvider); fJob2 = await fixture2.FetchNextJobAsync(); await fJob1.RemoveFromQueueAsync(); await fJob2.RemoveFromQueueAsync(); } // Assert fJob1.JobId.Should().NotBe(fJob2.JobId); using (CreateScope()) { var fixture = Create(); fixture.Context.JobQueue.Any().Should().BeFalse(); } }
public void FetchNextJob_WithPub() { var cancel = new CancellationTokenSource(); IFetchedJob Job = null; UseConnections((redis, connection) => { var t = new Thread(() => Job = connection.FetchNextJob(new string[] { "1" }, cancel.Token)); t.IsBackground = true; t.Start(); Thread.Sleep(10); //Enough time for Redis to respond that there are no jobs in queue, and the thread to start waiting redis.ListRightPush(Prefix + "queue:1", "job2"); redis.Publish(Prefix + "announce", "1"); //Pub to wake up thread t.Join(); Assert.Equal("job2", Job.JobId); }); }