/// <summary> /// Handles poison <paramref name="messages" /> by either delegating it to a handler or deleting them if no handler is provided. /// </summary> /// <param name="messages">The list of messages to operate on.</param> /// <param name="messageOptions">Initialisation options for this method.</param> /// <param name="asyncLock">An object that's responsible for synchronising access to shared resources in an asynchronous manner.</param> /// <returns> /// Returns a list of <paramref name="messages" /> that were not poison <paramref name="messages" /> and should be further processed. /// </returns> private async Task <IList <QueueMessageWrapper> > WerePoisonMessagesAndRemovedBatch( [NotNull] HandleMessagesBatchOptions messageOptions, [NotNull] IList <QueueMessageWrapper> messages, [NotNull] AsyncLock asyncLock) { Guard.NotNull(messageOptions, "messageOptions"); Guard.NotNull(messages, "messages"); Guard.NotNull(asyncLock, "asyncLock"); var poisonMessages = messages.Where(p => p.ActualMessage.DequeueCount > messageOptions.PoisonMessageThreshold).ToList(); if (poisonMessages.Count == 0) { return(messages); } var handledMessages = messageOptions.PoisonHandler != null ? await messageOptions.PoisonHandler(poisonMessages).ConfigureAwait(false) : new List <QueueMessageWrapper>(poisonMessages); foreach (var message in handledMessages) { await this.Top.SyncDeleteMessage(asyncLock, message, null).ConfigureAwait(false); } this.Statistics.IncreasePoisonMessages(poisonMessages.Count); return(messages.Except(handledMessages).ToList()); }
/// <summary> /// Processes <paramref name="messages" /> in batch. /// </summary> /// <param name="messages">The messages to be processed.</param> /// <param name="asyncLock">An object that's responsible for synchronising access to shared resources in an asynchronous manner.</param> /// <param name="batchCancellationToken">A cancellation token that's responsible for all tasks used in keep-alive.</param> /// <param name="messageOptions">Initialisation options for the method that handles the messages.</param> private async Task <Task> ProcessMessageInternalBatch( [NotNull] IList <QueueMessageWrapper> messages, [NotNull] CancellationTokenSource batchCancellationToken, [NotNull] HandleMessagesBatchOptions messageOptions) { Guard.NotNull(messages, "messages"); Guard.NotNull(messageOptions, "messageOptions"); var asynclock = new AsyncLock(); var oldMessages = messageOptions.TimeWindow.TotalSeconds <= 0 ? new List <QueueMessageWrapper>() : messages.Where(m => !m.ActualMessage.InsertionTime.HasValue || m.ActualMessage.InsertionTime.Value.UtcDateTime.Add(messageOptions.TimeWindow) < DateTime.UtcNow).ToList(); var timeValidMessages = messages.Except(oldMessages).ToList(); // Very old messages; delete them and move to the next one if (oldMessages.Count > 0) { foreach (var message in messages) { await this.Top.SyncDeleteMessage(asynclock, message, null).ConfigureAwait(false); } } // Handles poison messages by either delegating it to a handler or deleting it if no handler is provided. var toBeProcessedMessages = await this.Top.WerePoisonMessagesAndRemovedBatch(messageOptions, timeValidMessages, asynclock).ConfigureAwait(false); if (toBeProcessedMessages.Count == 0) { return(Task.FromResult <Task>(null)); } var generalCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(batchCancellationToken.Token, messageOptions.CancelToken).Token; var keepAliveTask = this.Top.KeepMessageAlive(messages, messageOptions.MessageLeaseTime, generalCancellationToken, asynclock); var handledMessages = await messageOptions.MessageHandler(messages).ConfigureAwait(false); this.Statistics.IncreaseSuccessfulMessages(handledMessages.Count); batchCancellationToken.Cancel(); // Execute the provided action and if successful, delete the message. foreach (var message in handledMessages) { await this.Top.SyncDeleteMessage(asynclock, message, null).ConfigureAwait(false); } var nonHandledMessages = messages.Except(handledMessages).Count(); this.Statistics.IncreaseReenqueuesCount(nonHandledMessages); batchCancellationToken.Cancel(); return(keepAliveTask); }
/// <summary> /// The finally handler in the try/catch/finally statement of HandleMessagesInBatchAsync. /// </summary> /// <param name="messageOptions">The message options object.</param> /// <param name="keepAliveTask">The <see cref="Task"/> that keeps the message "alive".</param> /// <param name="batchCancelToken">The cancellation token for the batch.</param> private void BatchFinallyHandler( [CanBeNull] HandleMessagesBatchOptions messageOptions, [CanBeNull] Task keepAliveTask, [NotNull] CancellationTokenSource batchCancelToken) { if (Guard.IsAnyNull(messageOptions)) { return; } Guard.NotNull(batchCancelToken, "generalCancelToken"); // Cancel any outstanding jobs. Since we don't have separate threads per message as in the serial processing, this indicates no faulted processing. if (keepAliveTask != null && !keepAliveTask.IsCompleted) { //messageOptions.QuickLogDebug("HandleBatchMessages", "Queue's '{0}' batch messages' processing faulted; cancelling related jobs", queue.Name); batchCancelToken.Cancel(); } }
public async Task HandleMessagesInBatchAsync([NotNull] HandleMessagesBatchOptions messageOptions) { Guard.NotNull(messageOptions, "messageOptions"); this.Statistics.IncreaseListeners(); this.Statistics.IncreaseAllMessageSlots(messageOptions.MaximumCurrentMessages); while (true) { if (messageOptions.CancelToken.IsCancellationRequested) { this.Statistics.DecreaseListeners(); this.Statistics.DecreaseAllMessageSlots(messageOptions.MaximumCurrentMessages); return; } // When set to true, the queue won't wait before it requests another message var shouldDelayNextRequest = false; // Used to prevent a message operation from running on a specific message var rawMessages = new List <IQueueMessage>(); var convertedMessages = new List <QueueMessageWrapper>(); var batchCancellationToken = new CancellationTokenSource(); Task keepAliveTask = null; while (true) { try { this.Top.LogAction(LogSeverity.Debug, "Attempting to retrieve new messages from a queue", "Queue: {0}", this.Name); var howManyMoreFit = messageOptions.MaximumCurrentMessages - rawMessages.Count; IList <IQueueMessage> retrievedMessages; if (howManyMoreFit > 0) { retrievedMessages = (await this.Top.GetMessagesAsync( Math.Min(this.MaximumMessagesProvider.MaximumMessagesPerRequest, howManyMoreFit), messageOptions.MessageLeaseTime, messageOptions.CancelToken).ConfigureAwait(false)).ToList(); } else { retrievedMessages = new List <IQueueMessage>(); } if (retrievedMessages.Count == 0 && rawMessages.Count == 0) { shouldDelayNextRequest = true; } else { // Keep trying to retrieve messages until there are no more or the quota is reached if (retrievedMessages.Count > 0) { rawMessages.AddRange(retrievedMessages); if (rawMessages.Count < messageOptions.MaximumCurrentMessages) { continue; } } // Have buffered messages and optionally some were retrieved from the cache. Proceed normally. convertedMessages.AddRange(rawMessages.Select(m => new QueueMessageWrapper(this.Top, m))); //messageOptions.QuickLogDebug("HandleBatchMessages", "Started processing queue's '{0}' {1} messages", queue.Name, rawMessages.Count); this.Statistics.IncreaseBusyMessageSlots(convertedMessages.Count); keepAliveTask = await this.ProcessMessageInternalBatch(convertedMessages, batchCancellationToken, messageOptions).ConfigureAwait(false); break; } } catch (OperationCanceledException) { if (messageOptions.CancelToken.IsCancellationRequested) { this.Statistics.DecreaseListeners(); this.Statistics.DecreaseAllMessageSlots(messageOptions.MaximumCurrentMessages); this.Statistics.DecreaseBusyMessageSlots(convertedMessages.Count); return; } else if (batchCancellationToken.IsCancellationRequested) { break; } } catch (CloudToolsStorageException ex) { this.Top.HandleStorageExceptions(messageOptions, ex); } catch (Exception ex) { this.Top.HandleGeneralExceptions(messageOptions, ex); } finally { this.Top.BatchFinallyHandler(messageOptions, keepAliveTask, batchCancellationToken); } // Delay the next polling attempt for a new message, since no messages were received last time. if (shouldDelayNextRequest) { await Task.Delay(messageOptions.PollFrequency, messageOptions.CancelToken).ConfigureAwait(false); } } this.Statistics.DecreaseBusyMessageSlots(convertedMessages.Count); } }
public void TestSerial_BatchProcessingDelayed() { // Arrange const int runCount = 30; var client = new CloudEnvironment(); var queue = client.QueueClient.GetQueueReference("test12"); var overflow = client.BlobClient.GetContainerReference("overflownqueues-12"); var locking = new object(); var result = string.Empty; var expected = string.Empty; var sw = new Stopwatch(); var factory = new AzureExtendedQueueFactory(new AzureBlobContainer(overflow), new ConsoleLogService()); var equeue = factory.Create(new AzureQueue(queue)); var lck = new AsyncLock(); for (var i = 1; i < runCount + 1; i++) { expected += ((char)(i)).ToString(CultureInfo.InvariantCulture); } using (var mre = new ManualResetEvent(false)) { var options = new HandleMessagesBatchOptions( TimeSpan.FromSeconds(0), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30), 5, 10, new CancellationToken(), async messages => { using (await lck.LockAsync()) { foreach (var message in messages) { var character = message.GetMessageContents <string>(); result += character; var innersw = new Stopwatch(); innersw.Start(); // Intentional spinning while (true) { if (innersw.Elapsed > TimeSpan.FromSeconds(5)) { break; } } } } if (result.Length == runCount) { mre.Set(); } return(messages); }, null, ex => { throw ex; }); // Act sw.Start(); queue.CreateIfNotExists(); overflow.CreateIfNotExists(); queue.Clear(); for (var i = 1; i < runCount + 1; i++) { equeue.AddMessageEntity(((char)(i)).ToString(CultureInfo.InvariantCulture)); } equeue.HandleMessagesInBatchAsync(options); // Assert mre.WaitOne(); sw.Stop(); Trace.WriteLine("Total execution time (in seconds): " + sw.Elapsed.TotalSeconds.ToString(CultureInfo.InvariantCulture)); Assert.IsTrue(expected.All(c => result.Contains(c))); } }