protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var pollDelay = TimeSpan.FromSeconds(1); var inactivityCounter = 0; while (!stoppingToken.IsCancellationRequested) { var referenceTime = DateTime.UtcNow; var messageCounter = 0; using (var serviceScope = _services.CreateScope()) { var queueSource = serviceScope.ServiceProvider.GetRequiredService <IQueueSource>(); using (var batch = (await queueSource.GetMessagesAsync(referenceTime))) { foreach (var message in batch.Messages) { messageCounter += 1; try { // Lookup the handler based on the type of the message var handler = _handlerFactory.GetHandlerForMessage(message.MessageType, serviceScope.ServiceProvider); if (handler == null) { _logger.LogWarning("Failed to find a handler for {MessageType}", message.MessageType); } // handle the message await handler.ExecuteAsync(message.MessageId, message.Data, stoppingToken); // and mark it for removal queueSource.RemoveMessage(message); } catch (Exception e) { _logger.LogError(e, "An error occurred while handling a message"); // Something went wrong and we couldn't handle the message so track the error and requeue the message message.ErrorCount += 1; message.ErrorMessage = e.Message; message.LastRunTime = referenceTime; message.NextRunTime = message.ErrorCount > 10 // If the message has failed too many times then dead-letter it ? DateTime.MaxValue // Otherwise push it down the queue so it can be retried later : DateTime.UtcNow.AddMinutes(message.ErrorCount); // requeue the message queueSource.UpdateMessage(message); } } } } if (messageCounter == 0) { inactivityCounter += 1; } else { inactivityCounter = 0; } // The longer we don't have any work to do, the longer we wait until the next check. // The benefit is that during periods of inactivity we aren't spamming the db too often // The downside is that when work does appear, the delay will impact the soonest time that it will be noticed. // We also cap the delay, so that the wait doesn't get too long. pollDelay = pollDelay.Add(TimeSpan.FromSeconds(Math.Min(Math.Floor(inactivityCounter / 120.0), 360))); _logger.LogInformation("Delay until next execution is {Delay}", pollDelay); await Task.Delay(pollDelay, stoppingToken); } }