/// <summary>
/// Handles poison <paramref name="messages" /> by either delegating it to a handler or deleting them if no handler is provided.
/// </summary>
/// <param name="messages">The list of messages to operate on.</param>
/// <param name="messageOptions">Initialisation options for this method.</param>
/// <param name="asyncLock">An object that's responsible for synchronising access to shared resources in an asynchronous manner.</param>
/// <returns>
/// Returns a list of <paramref name="messages" /> that were not poison <paramref name="messages" /> and should be further processed.
/// </returns>
private async Task <IList <QueueMessageWrapper> > WerePoisonMessagesAndRemovedBatch(
[NotNull] HandleMessagesBatchOptions messageOptions,
[NotNull] IList <QueueMessageWrapper> messages,
[NotNull] AsyncLock asyncLock)
{
Guard.NotNull(messageOptions, "messageOptions");
Guard.NotNull(messages, "messages");
Guard.NotNull(asyncLock, "asyncLock");
var poisonMessages = messages.Where(p => p.ActualMessage.DequeueCount > messageOptions.PoisonMessageThreshold).ToList();
if (poisonMessages.Count == 0)
{
return(messages);
}
var handledMessages = messageOptions.PoisonHandler != null ? await messageOptions.PoisonHandler(poisonMessages).ConfigureAwait(false) : new List <QueueMessageWrapper>(poisonMessages);
foreach (var message in handledMessages)
{
await this.Top.SyncDeleteMessage(asyncLock, message, null).ConfigureAwait(false);
}
this.Statistics.IncreasePoisonMessages(poisonMessages.Count);
return(messages.Except(handledMessages).ToList());
}
/// <summary>
/// Processes <paramref name="messages" /> in batch.
/// </summary>
/// <param name="messages">The messages to be processed.</param>
/// <param name="asyncLock">An object that's responsible for synchronising access to shared resources in an asynchronous manner.</param>
/// <param name="batchCancellationToken">A cancellation token that's responsible for all tasks used in keep-alive.</param>
/// <param name="messageOptions">Initialisation options for the method that handles the messages.</param>
private async Task <Task> ProcessMessageInternalBatch(
[NotNull] IList <QueueMessageWrapper> messages,
[NotNull] CancellationTokenSource batchCancellationToken,
[NotNull] HandleMessagesBatchOptions messageOptions)
{
Guard.NotNull(messages, "messages");
Guard.NotNull(messageOptions, "messageOptions");
var asynclock = new AsyncLock();
var oldMessages = messageOptions.TimeWindow.TotalSeconds <= 0
? new List <QueueMessageWrapper>()
: messages.Where(m => !m.ActualMessage.InsertionTime.HasValue || m.ActualMessage.InsertionTime.Value.UtcDateTime.Add(messageOptions.TimeWindow) < DateTime.UtcNow).ToList();
var timeValidMessages = messages.Except(oldMessages).ToList();
// Very old messages; delete them and move to the next one
if (oldMessages.Count > 0)
{
foreach (var message in messages)
{
await this.Top.SyncDeleteMessage(asynclock, message, null).ConfigureAwait(false);
}
}
// Handles poison messages by either delegating it to a handler or deleting it if no handler is provided.
var toBeProcessedMessages = await this.Top.WerePoisonMessagesAndRemovedBatch(messageOptions, timeValidMessages, asynclock).ConfigureAwait(false);
if (toBeProcessedMessages.Count == 0)
{
return(Task.FromResult <Task>(null));
}
var generalCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(batchCancellationToken.Token, messageOptions.CancelToken).Token;
var keepAliveTask = this.Top.KeepMessageAlive(messages, messageOptions.MessageLeaseTime, generalCancellationToken, asynclock);
var handledMessages = await messageOptions.MessageHandler(messages).ConfigureAwait(false);
this.Statistics.IncreaseSuccessfulMessages(handledMessages.Count);
batchCancellationToken.Cancel();
// Execute the provided action and if successful, delete the message.
foreach (var message in handledMessages)
{
await this.Top.SyncDeleteMessage(asynclock, message, null).ConfigureAwait(false);
}
var nonHandledMessages = messages.Except(handledMessages).Count();
this.Statistics.IncreaseReenqueuesCount(nonHandledMessages);
batchCancellationToken.Cancel();
return(keepAliveTask);
}
/// <summary>
/// The finally handler in the try/catch/finally statement of HandleMessagesInBatchAsync.
/// </summary>
/// <param name="messageOptions">The message options object.</param>
/// <param name="keepAliveTask">The <see cref="Task"/> that keeps the message "alive".</param>
/// <param name="batchCancelToken">The cancellation token for the batch.</param>
private void BatchFinallyHandler(
[CanBeNull] HandleMessagesBatchOptions messageOptions,
[CanBeNull] Task keepAliveTask,
[NotNull] CancellationTokenSource batchCancelToken)
{
if (Guard.IsAnyNull(messageOptions))
{
return;
}
Guard.NotNull(batchCancelToken, "generalCancelToken");
// Cancel any outstanding jobs. Since we don't have separate threads per message as in the serial processing, this indicates no faulted processing.
if (keepAliveTask != null && !keepAliveTask.IsCompleted)
{
//messageOptions.QuickLogDebug("HandleBatchMessages", "Queue's '{0}' batch messages' processing faulted; cancelling related jobs", queue.Name);
batchCancelToken.Cancel();
}
}
public async Task HandleMessagesInBatchAsync([NotNull] HandleMessagesBatchOptions messageOptions)
{
Guard.NotNull(messageOptions, "messageOptions");
this.Statistics.IncreaseListeners();
this.Statistics.IncreaseAllMessageSlots(messageOptions.MaximumCurrentMessages);
while (true)
{
if (messageOptions.CancelToken.IsCancellationRequested)
{
this.Statistics.DecreaseListeners();
this.Statistics.DecreaseAllMessageSlots(messageOptions.MaximumCurrentMessages);
return;
}
// When set to true, the queue won't wait before it requests another message
var shouldDelayNextRequest = false;
// Used to prevent a message operation from running on a specific message
var rawMessages = new List <IQueueMessage>();
var convertedMessages = new List <QueueMessageWrapper>();
var batchCancellationToken = new CancellationTokenSource();
Task keepAliveTask = null;
while (true)
{
try
{
this.Top.LogAction(LogSeverity.Debug, "Attempting to retrieve new messages from a queue", "Queue: {0}", this.Name);
var howManyMoreFit = messageOptions.MaximumCurrentMessages - rawMessages.Count;
IList <IQueueMessage> retrievedMessages;
if (howManyMoreFit > 0)
{
retrievedMessages = (await this.Top.GetMessagesAsync(
Math.Min(this.MaximumMessagesProvider.MaximumMessagesPerRequest, howManyMoreFit),
messageOptions.MessageLeaseTime,
messageOptions.CancelToken).ConfigureAwait(false)).ToList();
}
else
{
retrievedMessages = new List <IQueueMessage>();
}
if (retrievedMessages.Count == 0 && rawMessages.Count == 0)
{
shouldDelayNextRequest = true;
}
else
{
// Keep trying to retrieve messages until there are no more or the quota is reached
if (retrievedMessages.Count > 0)
{
rawMessages.AddRange(retrievedMessages);
if (rawMessages.Count < messageOptions.MaximumCurrentMessages)
{
continue;
}
}
// Have buffered messages and optionally some were retrieved from the cache. Proceed normally.
convertedMessages.AddRange(rawMessages.Select(m => new QueueMessageWrapper(this.Top, m)));
//messageOptions.QuickLogDebug("HandleBatchMessages", "Started processing queue's '{0}' {1} messages", queue.Name, rawMessages.Count);
this.Statistics.IncreaseBusyMessageSlots(convertedMessages.Count);
keepAliveTask = await this.ProcessMessageInternalBatch(convertedMessages, batchCancellationToken, messageOptions).ConfigureAwait(false);
break;
}
}
catch (OperationCanceledException)
{
if (messageOptions.CancelToken.IsCancellationRequested)
{
this.Statistics.DecreaseListeners();
this.Statistics.DecreaseAllMessageSlots(messageOptions.MaximumCurrentMessages);
this.Statistics.DecreaseBusyMessageSlots(convertedMessages.Count);
return;
}
else if (batchCancellationToken.IsCancellationRequested)
{
break;
}
}
catch (CloudToolsStorageException ex)
{
this.Top.HandleStorageExceptions(messageOptions, ex);
}
catch (Exception ex)
{
this.Top.HandleGeneralExceptions(messageOptions, ex);
}
finally
{
this.Top.BatchFinallyHandler(messageOptions, keepAliveTask, batchCancellationToken);
}
// Delay the next polling attempt for a new message, since no messages were received last time.
if (shouldDelayNextRequest)
{
await Task.Delay(messageOptions.PollFrequency, messageOptions.CancelToken).ConfigureAwait(false);
}
}
this.Statistics.DecreaseBusyMessageSlots(convertedMessages.Count);
}
}
public void TestSerial_BatchProcessingDelayed()
{
// Arrange
const int runCount = 30;
var client = new CloudEnvironment();
var queue = client.QueueClient.GetQueueReference("test12");
var overflow = client.BlobClient.GetContainerReference("overflownqueues-12");
var locking = new object();
var result = string.Empty;
var expected = string.Empty;
var sw = new Stopwatch();
var factory = new AzureExtendedQueueFactory(new AzureBlobContainer(overflow), new ConsoleLogService());
var equeue = factory.Create(new AzureQueue(queue));
var lck = new AsyncLock();
for (var i = 1; i < runCount + 1; i++)
{
expected += ((char)(i)).ToString(CultureInfo.InvariantCulture);
}
using (var mre = new ManualResetEvent(false))
{
var options = new HandleMessagesBatchOptions(
TimeSpan.FromSeconds(0),
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30),
5,
10,
new CancellationToken(),
async messages =>
{
using (await lck.LockAsync())
{
foreach (var message in messages)
{
var character = message.GetMessageContents <string>();
result += character;
var innersw = new Stopwatch();
innersw.Start();
// Intentional spinning
while (true)
{
if (innersw.Elapsed > TimeSpan.FromSeconds(5))
{
break;
}
}
}
}
if (result.Length == runCount)
{
mre.Set();
}
return(messages);
},
null,
ex => { throw ex; });
// Act
sw.Start();
queue.CreateIfNotExists();
overflow.CreateIfNotExists();
queue.Clear();
for (var i = 1; i < runCount + 1; i++)
{
equeue.AddMessageEntity(((char)(i)).ToString(CultureInfo.InvariantCulture));
}
equeue.HandleMessagesInBatchAsync(options);
// Assert
mre.WaitOne();
sw.Stop();
Trace.WriteLine("Total execution time (in seconds): " + sw.Elapsed.TotalSeconds.ToString(CultureInfo.InvariantCulture));
Assert.IsTrue(expected.All(c => result.Contains(c)));
}
}
