async Task RouteBatchWithEnforcedBatchSizeAsync(IMessageSenderInternal messageSender, IEnumerable <BrokeredMessage> messagesToSend)
{
var chunk = new List <BrokeredMessage>();
long batchSize = 0;
var chunkNumber = 1;
foreach (var message in messagesToSend)
{
if (await GuardMessageSize(message).ConfigureAwait(false))
{
return;
}
var messageSize = message.EstimatedSize();
if (batchSize + messageSize > maximuMessageSizeInKilobytes * 1024)
{
if (chunk.Any())
{
logger.Debug($"Routing batched messages, chunk #{chunkNumber++}.");
var currentChunk = chunk;
await messageSender.RetryOnThrottleAsync(s => s.SendBatch(currentChunk), s => s.SendBatch(currentChunk.Select(x => x.Clone())), backOffTimeOnThrottle, maxRetryAttemptsOnThrottle).ConfigureAwait(false);
}
chunk = new List <BrokeredMessage>
{
message
};
batchSize = messageSize;
}
else
{
chunk.Add(message);
batchSize += messageSize;
}
}
if (chunk.Any())
{
logger.Debug($"Routing batched messages, chunk #{chunkNumber}.");
await messageSender.RetryOnThrottleAsync(s => s.SendBatch(chunk), s => s.SendBatch(chunk.Select(x => x.Clone())), backOffTimeOnThrottle, maxRetryAttemptsOnThrottle).ConfigureAwait(false);
}
}
public static async Task RetryOnThrottleAsync(this IMessageSenderInternal sender, Func <IMessageSenderInternal, Task> action, Func <IMessageSenderInternal, Task> retryAction, TimeSpan delay, int maxRetryAttempts, int retryAttempts = 0)
{
try
{
// upon retries have to use new BrokeredMessage instances
var actionToTake = retryAttempts == 0 ? action : retryAction;
await actionToTake(sender).ConfigureAwait(false);
}
catch (ServerBusyException)
{
if (retryAttempts < maxRetryAttempts)
{
logger.Warn($"We are throttled, backing off for {delay.TotalSeconds} seconds (attempt {retryAttempts + 1}/{maxRetryAttempts}).");
await Task.Delay(delay).ConfigureAwait(false);
await sender.RetryOnThrottleAsync(action, retryAction, delay, maxRetryAttempts, ++retryAttempts).ConfigureAwait(false);
}
else
{
throw;
}
}
}
async Task RouteOutBatchesWithFallbackAndLogExceptionsAsync(IMessageSenderInternal messageSender, IList <IMessageSenderInternal> fallbacks, IList <BrokeredMessage> messagesToSend, bool suppressTransaction)
{
try
{
var scope = suppressTransaction && Transaction.Current != null ? new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled) : null;
if (scope != null)
{
using (scope)
{
await RouteBatchWithEnforcedBatchSizeAsync(messageSender, messagesToSend).ConfigureAwait(false);
scope?.Complete();
}
}
else
{
await RouteBatchWithEnforcedBatchSizeAsync(messageSender, messagesToSend).ConfigureAwait(false);
}
}
catch (Exception exception)
{
// ASB team promissed to fix the issue with MessagingEntityNotFoundException (missing entity path) - verify that
var message = "Failed to dispatch a batch with the following message IDs: " + string.Join(", ", messagesToSend.Select(x => x.MessageId));
logger.Error(message, exception);
// no need to try and send too large messages to another namespace, won't work
if (exception is MessageTooLargeException)
{
throw;
}
if (exception is TransactionSizeExceededException)
{
throw new TransactionContainsTooManyMessages(exception);
}
var fallBackSucceeded = false;
if (fallbacks.Any())
{
foreach (var fallback in fallbacks)
{
var clones = messagesToSend.Select(x => x.Clone()).ToList();
try
{
using (var scope = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled))
{
await RouteBatchWithEnforcedBatchSizeAsync(fallback, clones).ConfigureAwait(false);
scope.Complete();
}
logger.Info("Successfully dispatched a batch with the following message IDs: " + string.Join(", ", clones.Select(x => x.MessageId) + " to fallback namespace"));
fallBackSucceeded = true;
break;
}
catch (Exception ex)
{
logger.Error("Failed to dispatch batch to fallback namespace.", ex);
}
}
}
if (!fallBackSucceeded)
{
throw;
}
}
}
