async Task InnerProcessMessages(BackoffStrategy backoffStrategy)
{
while (!cancellationTokenSource.IsCancellationRequested)
{
try
{
var retrieved = await messageReceiver.Receive(backoffStrategy, cancellationTokenSource.Token).ConfigureAwait(false);
circuitBreaker.Success();
foreach (var message in retrieved)
{
await concurrencyLimiter.WaitAsync(cancellationToken).ConfigureAwait(false);
if (cancellationTokenSource.IsCancellationRequested)
{
return;
}
InnerReceive(message).Ignore();
}
}
catch (OperationCanceledException)
{
// For graceful shutdown purposes
return;
}
catch (Exception ex)
{
Logger.Warn("Receiving from the queue failed", ex);
await circuitBreaker.Failure(ex).ConfigureAwait(false);
}
}
}
public Task StartReceive(CancellationToken cancellationToken = default)
{
maximumConcurrency = limitations.MaxConcurrency;
concurrencyLimiter = new SemaphoreSlim(maximumConcurrency, maximumConcurrency);
messagePumpCancellationTokenSource = new CancellationTokenSource();
messageProcessingCancellationTokenSource = new CancellationTokenSource();
circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker("AzureStorageQueue-MessagePump", TimeToWaitBeforeTriggering, ex => criticalErrorAction("Failed to receive message from Azure Storage Queue.", ex, messageProcessingCancellationTokenSource.Token));
Logger.DebugFormat($"Starting MessageReceiver {Id} with max concurrency: {0}", maximumConcurrency);
var receiverConfigurations = MessagePumpHelpers.DetermineReceiverConfiguration(receiveBatchSize, degreeOfReceiveParallelism, maximumConcurrency);
messagePumpTasks = new Task[receiverConfigurations.Count];
for (var i = 0; i < receiverConfigurations.Count; i++)
{
var backoffStrategy = new BackoffStrategy(peekInterval, maximumWaitTime);
var batchSizeForReceive = receiverConfigurations[i].BatchSize;
// Task.Run() so the call returns immediately instead of waiting for the first await or return down the call stack
messagePumpTasks[i] = Task.Run(() => PumpMessagesAndSwallowExceptions(batchSizeForReceive, backoffStrategy, messagePumpCancellationTokenSource.Token), CancellationToken.None);
}
return(Task.CompletedTask);
}
public DelayedMessagesPoller(CloudTable delayedDeliveryTable, string connectionString, string errorQueue, bool isAtMostOnce, Dispatcher dispatcher, BackoffStrategy backoffStrategy)
{
this.errorQueue = errorQueue;
this.isAtMostOnce = isAtMostOnce;
this.delayedDeliveryTable = delayedDeliveryTable;
this.connectionString = connectionString;
this.dispatcher = dispatcher;
this.backoffStrategy = backoffStrategy;
}
async Task ProcessMessages(int batchSizeForReceive, BackoffStrategy backoffStrategy)
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
await InnerProcessMessages(batchSizeForReceive, backoffStrategy).ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.Error("Polling Dequeue Strategy failed", ex);
}
}
}
public NativeDelayedDeliveryProcessor(
Dispatcher dispatcher,
CloudTable delayedMessageStorageTable,
BlobServiceClient blobServiceClient,
string errorQueueAddress,
TransportTransactionMode transportTransactionMode,
BackoffStrategy backoffStrategy)
{
enabled = true;
this.dispatcher = dispatcher;
this.delayedMessageStorageTable = delayedMessageStorageTable;
this.blobServiceClient = blobServiceClient;
this.errorQueueAddress = errorQueueAddress;
this.transportTransactionMode = transportTransactionMode;
this.backoffStrategy = backoffStrategy;
}
async Task PumpMessages(int batchSizeForReceive, BackoffStrategy backoffStrategy, CancellationToken pumpCancellationToken)
{
var receivedMessages = new List <MessageRetrieved>(batchSizeForReceive);
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Fetching {0} messages", batchSizeForReceive);
}
while (true)
{
pumpCancellationToken.ThrowIfCancellationRequested();
#pragma warning disable PS0021 // Highlight when a try block passes multiple cancellation tokens - justification:
// The message processing cancellation token is used for processing,
// since we only want that to be canceled when the public token passed to Stop() is canceled.
// The message pump token is being used elsewhere, because we want those operations to be canceled as soon as Stop() is called.
// The catch clause is correctly filtered on the message pump cancellation token.
try
#pragma warning restore PS0021 // Highlight when a try block passes multiple cancellation tokens
{
await azureMessageQueueReceiver.Receive(batchSizeForReceive, receivedMessages, backoffStrategy, pumpCancellationToken).ConfigureAwait(false);
circuitBreaker.Success();
foreach (var message in receivedMessages)
{
await concurrencyLimiter.WaitAsync(pumpCancellationToken).ConfigureAwait(false);
// no Task.Run() here to avoid a closure
_ = ProcessMessageSwallowExceptionsAndReleaseConcurrencyLimiter(message, messageProcessingCancellationTokenSource.Token);
}
}
catch (Exception ex) when(!ex.IsCausedBy(pumpCancellationToken))
{
Logger.Warn("Receiving from the queue failed", ex);
await circuitBreaker.Failure(ex, pumpCancellationToken).ConfigureAwait(false);
}
finally
{
receivedMessages.Clear();
}
}
}
public void Start(PushRuntimeSettings limitations)
{
maximumConcurrency = limitations.MaxConcurrency;
concurrencyLimiter = new SemaphoreSlim(maximumConcurrency, maximumConcurrency);
cancellationTokenSource = new CancellationTokenSource();
var receiverConfigurations = MessagePumpHelpers.DetermineReceiverConfiguration(receiveBatchSize, degreeOfReceiveParallelism, maximumConcurrency);
messagePumpTasks = new Task[receiverConfigurations.Count];
cancellationToken = cancellationTokenSource.Token;
for (var i = 0; i < receiverConfigurations.Count; i++)
{
var backoffStrategy = new BackoffStrategy(peekInterval, maximumWaitTime);
var batchSizeForReceive = receiverConfigurations[i].BatchSize;
messagePumpTasks[i] = Task.Run(() => ProcessMessages(batchSizeForReceive, backoffStrategy), CancellationToken.None);
}
}
public void Start(PushRuntimeSettings limitations)
{
maximumConcurrency = limitations.MaxConcurrency;
concurrencyLimiter = new SemaphoreSlim(maximumConcurrency, maximumConcurrency);
cancellationTokenSource = new CancellationTokenSource();
if (!degreeOfReceiveParallelism.HasValue)
{
degreeOfReceiveParallelism = EstimateDegreeOfReceiveParallelism(maximumConcurrency);
}
messagePumpTasks = new Task[degreeOfReceiveParallelism.Value];
cancellationToken = cancellationTokenSource.Token;
for (var i = 0; i < degreeOfReceiveParallelism; i++)
{
var backoffStrategy = new BackoffStrategy(peekInterval, maximumWaitTime);
messagePumpTasks[i] = Task.Run(() => ProcessMessages(backoffStrategy), CancellationToken.None);
}
}
internal async Task <List <MessageRetrieved> > Receive(BackoffStrategy backoffStrategy, CancellationToken token)
{
var rawMessages = await inputQueue.GetMessagesAsync(BatchSize, MessageInvisibleTime, null, null, token).ConfigureAwait(false);
var messageFound = false;
List <MessageRetrieved> messages = null;
foreach (var rawMessage in rawMessages)
{
if (!messageFound)
{
messages = new List <MessageRetrieved>(BatchSize);
messageFound = true;
}
messages.Add(new MessageRetrieved(unwrapper, rawMessage, inputQueue, errorQueue));
}
await backoffStrategy.OnBatch(messageFound?messages.Count : 0, token).ConfigureAwait(false);
return(messageFound ? messages : noMessagesFound);
}
async Task PumpMessagesAndSwallowExceptions(int batchSizeForReceive, BackoffStrategy backoffStrategy, CancellationToken pumpCancellationToken)
{
Logger.Debug("Pumping messages");
while (!pumpCancellationToken.IsCancellationRequested)
{
try
{
await PumpMessages(batchSizeForReceive, backoffStrategy, pumpCancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when(ex.IsCausedBy(pumpCancellationToken))
{
// private token, receiver is being canceled, log exception in case stack trace is ever needed for debugging
Logger.Debug("Operation canceled while stopping message receiver.", ex);
break;
}
catch (Exception ex)
{
Logger.Error("Polling Dequeue Strategy failed", ex);
}
}
}
public DelayedMessagesPoller(CloudTable delayedDeliveryTable, BlobServiceClient blobServiceClient, string errorQueueAddress, bool isAtMostOnce, Dispatcher dispatcher, BackoffStrategy backoffStrategy)
{
this.errorQueueAddress = errorQueueAddress;
this.isAtMostOnce = isAtMostOnce;
this.delayedDeliveryTable = delayedDeliveryTable;
this.blobServiceClient = blobServiceClient;
this.dispatcher = dispatcher;
this.backoffStrategy = backoffStrategy;
}
internal async Task Receive(int batchSize, List <MessageRetrieved> receivedMessages, BackoffStrategy backoffStrategy, CancellationToken token)
{
var rawMessages = await inputQueue.GetMessagesAsync(batchSize, MessageInvisibleTime, null, null, token).ConfigureAwait(false);
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var rawMessage in rawMessages)
{
receivedMessages.Add(new MessageRetrieved(unwrapper, rawMessage, inputQueue, errorQueue));
}
await backoffStrategy.OnBatch(receivedMessages.Count, token).ConfigureAwait(false);
}
