private async Task ProcessMessages(TimeSpan?visibilityTimeout = null, CancellationToken cancellationToken = default(CancellationToken))
{
var runningTasks = new ConcurrentDictionary <Task, Task>();
var semaphore = new SemaphoreSlim(_concurrentTasks, _concurrentTasks);
var queuedMessages = new ConcurrentQueue <CloudMessage>();
// Define the task that fetches messages from the Azure queue
RecurrentCancellableTask.StartNew(
async() =>
{
// Fetch messages from the Azure queue when the number of items in the concurrent queue falls below an "acceptable" level.
if (!cancellationToken.IsCancellationRequested && queuedMessages.Count <= _concurrentTasks / 2)
{
using (_metrics.Measure.Timer.Time(Metrics.MessageFetchingTimer))
{
IEnumerable <CloudMessage> messages = null;
try
{
messages = await _queueManager.GetMessagesAsync(_concurrentTasks, visibilityTimeout, null, null, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
// The message pump is shutting down.
// This exception can be safely ignored.
}
catch (Exception e)
{
_logger.InfoException("An error occured while fetching messages from the Azure queue. The error was caught and ignored.", e.GetBaseException());
}
if (messages == null)
{
return;
}
if (messages.Any())
{
_logger.Trace($"Fetched {messages.Count()} message(s) from the queue.");
foreach (var message in messages)
{
queuedMessages.Enqueue(message);
}
}
else
{
_logger.Trace("The queue is empty, no messages fetched.");
try
{
// The queue is empty
OnQueueEmpty?.Invoke(cancellationToken);
_metrics.Measure.Counter.Increment(Metrics.QueueEmptyCounter);
}
catch (Exception e)
{
_logger.InfoException("An error occured when handling an empty queue. The error was caught and ignored.", e.GetBaseException());
}
}
}
}
},
TimeSpan.FromMilliseconds(500),
cancellationToken,
TaskCreationOptions.LongRunning);
// Define the task that checks how many messages are queued
RecurrentCancellableTask.StartNew(
async() =>
{
try
{
var count = await _queueManager.GetApproximateMessageCountAsync(cancellationToken).ConfigureAwait(false);
count += queuedMessages.Count;
_metrics.Measure.Gauge.SetValue(Metrics.QueuedMessagesGauge, count);
}
catch (TaskCanceledException)
{
// The message pump is shutting down.
// This exception can be safely ignored.
}
catch (Exception e)
{
_logger.InfoException("An error occured while checking how many message are waiting in the queue. The error was caught and ignored.", e.GetBaseException());
}
},
TimeSpan.FromMilliseconds(5000),
cancellationToken,
TaskCreationOptions.LongRunning);
// Define the task pump
var pumpTask = Task.Run(async() =>
{
while (!cancellationToken.IsCancellationRequested)
{
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
// Retrieved the next message from the queue and process it
var runningTask = Task.Run(
async() =>
{
var messageProcessed = false;
if (cancellationToken.IsCancellationRequested)
{
return(messageProcessed);
}
using (_metrics.Measure.Timer.Time(Metrics.MessageProcessingTimer))
{
queuedMessages.TryDequeue(out CloudMessage message);
if (message != null)
{
try
{
// Process the message
OnMessage?.Invoke(message, cancellationToken);
// Delete the processed message from the queue
// PLEASE NOTE: we use "CancellationToken.None" to ensure a processed message is deleted from the queue even when the message pump is shutting down
await _queueManager.DeleteMessageAsync(message, null, null, CancellationToken.None).ConfigureAwait(false);
}
catch (Exception ex)
{
var isPoison = message.DequeueCount > _maxDequeueCount;
OnError?.Invoke(message, ex, isPoison);
if (isPoison)
{
// PLEASE NOTE: we use "CancellationToken.None" to ensure a processed message is deleted from the queue even when the message pump is shutting down
await _queueManager.DeleteMessageAsync(message, null, null, CancellationToken.None).ConfigureAwait(false);
}
}
messageProcessed = true;
}
}
// Increment the counter if we processed a message
if (messageProcessed)
{
_metrics.Measure.Counter.Increment(Metrics.MessagesProcessedCounter);
}
// Return a value indicating whether we processed a message or not
return(messageProcessed);
},
CancellationToken.None);
// Add the task to the dictionary of tasks (allows us to keep track of the running tasks)
runningTasks.TryAdd(runningTask, runningTask);
// Complete the task
runningTask.ContinueWith(
t =>
{
semaphore.Release();
runningTasks.TryRemove(t, out Task taskToBeRemoved);
}, TaskContinuationOptions.ExecuteSynchronously)
.IgnoreAwait();
}
});
// Run the task pump until canceled
await pumpTask.UntilCancelled().ConfigureAwait(false);
// Task pump has been canceled, wait for the currently running tasks to complete
await Task.WhenAll(runningTasks.Values).UntilCancelled().ConfigureAwait(false);
}
