async Task RouteBatchWithEnforcedBatchSizeAsync(IMessageSenderInternal messageSender, IEnumerable <BrokeredMessage> messagesToSend) { var chunk = new List <BrokeredMessage>(); long batchSize = 0; var chunkNumber = 1; foreach (var message in messagesToSend) { if (await GuardMessageSize(message).ConfigureAwait(false)) { return; } var messageSize = message.EstimatedSize(); if (batchSize + messageSize > maximuMessageSizeInKilobytes * 1024) { if (chunk.Any()) { logger.Debug($"Routing batched messages, chunk #{chunkNumber++}."); var currentChunk = chunk; await messageSender.RetryOnThrottleAsync(s => s.SendBatch(currentChunk), s => s.SendBatch(currentChunk.Select(x => x.Clone())), backOffTimeOnThrottle, maxRetryAttemptsOnThrottle).ConfigureAwait(false); } chunk = new List <BrokeredMessage> { message }; batchSize = messageSize; } else { chunk.Add(message); batchSize += messageSize; } } if (chunk.Any()) { logger.Debug($"Routing batched messages, chunk #{chunkNumber}."); await messageSender.RetryOnThrottleAsync(s => s.SendBatch(chunk), s => s.SendBatch(chunk.Select(x => x.Clone())), backOffTimeOnThrottle, maxRetryAttemptsOnThrottle).ConfigureAwait(false); } }
public static async Task RetryOnThrottleAsync(this IMessageSenderInternal sender, Func <IMessageSenderInternal, Task> action, Func <IMessageSenderInternal, Task> retryAction, TimeSpan delay, int maxRetryAttempts, int retryAttempts = 0) { try { // upon retries have to use new BrokeredMessage instances var actionToTake = retryAttempts == 0 ? action : retryAction; await actionToTake(sender).ConfigureAwait(false); } catch (ServerBusyException) { if (retryAttempts < maxRetryAttempts) { logger.Warn($"We are throttled, backing off for {delay.TotalSeconds} seconds (attempt {retryAttempts + 1}/{maxRetryAttempts})."); await Task.Delay(delay).ConfigureAwait(false); await sender.RetryOnThrottleAsync(action, retryAction, delay, maxRetryAttempts, ++retryAttempts).ConfigureAwait(false); } else { throw; } } }
async Task RouteOutBatchesWithFallbackAndLogExceptionsAsync(IMessageSenderInternal messageSender, IList <IMessageSenderInternal> fallbacks, IList <BrokeredMessage> messagesToSend, bool suppressTransaction) { try { var scope = suppressTransaction && Transaction.Current != null ? new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled) : null; if (scope != null) { using (scope) { await RouteBatchWithEnforcedBatchSizeAsync(messageSender, messagesToSend).ConfigureAwait(false); scope?.Complete(); } } else { await RouteBatchWithEnforcedBatchSizeAsync(messageSender, messagesToSend).ConfigureAwait(false); } } catch (Exception exception) { // ASB team promissed to fix the issue with MessagingEntityNotFoundException (missing entity path) - verify that var message = "Failed to dispatch a batch with the following message IDs: " + string.Join(", ", messagesToSend.Select(x => x.MessageId)); logger.Error(message, exception); // no need to try and send too large messages to another namespace, won't work if (exception is MessageTooLargeException) { throw; } if (exception is TransactionSizeExceededException) { throw new TransactionContainsTooManyMessages(exception); } var fallBackSucceeded = false; if (fallbacks.Any()) { foreach (var fallback in fallbacks) { var clones = messagesToSend.Select(x => x.Clone()).ToList(); try { using (var scope = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled)) { await RouteBatchWithEnforcedBatchSizeAsync(fallback, clones).ConfigureAwait(false); scope.Complete(); } logger.Info("Successfully dispatched a batch with the following message IDs: " + string.Join(", ", clones.Select(x => x.MessageId) + " to fallback namespace")); fallBackSucceeded = true; break; } catch (Exception ex) { logger.Error("Failed to dispatch batch to fallback namespace.", ex); } } } if (!fallBackSucceeded) { throw; } } }