async Task ChangeVisibilityOfDelayedMessagesThatFailedBatchDeliveryInBatches(BatchEntry <SqsReceivedDelayedMessage> batch, SendMessageBatchResponse result, int batchNumber, int totalBatches)
{
try
{
List <ChangeMessageVisibilityBatchRequestEntry> changeVisibilityBatchRequestEntries = null;
foreach (var failed in result.Failed)
{
changeVisibilityBatchRequestEntries = changeVisibilityBatchRequestEntries ?? new List <ChangeMessageVisibilityBatchRequestEntry>(result.Failed.Count);
var preparedMessage = batch.PreparedMessagesBydId[failed.Id];
// need to reuse the previous batch entry ID so that we can map again in failure scenarios, this is fine given that IDs only need to be unique per request
changeVisibilityBatchRequestEntries.Add(new ChangeMessageVisibilityBatchRequestEntry(failed.Id, preparedMessage.ReceiptHandle)
{
VisibilityTimeout = 0
});
}
if (changeVisibilityBatchRequestEntries != null)
{
if (Logger.IsDebugEnabled)
{
var message = batch.PreparedMessagesBydId.Values.First();
Logger.Debug($"Changing delayed message visibility for batch '{batchNumber}/{totalBatches}' with message ids '{string.Join(", ", batch.PreparedMessagesBydId.Values.Select(v => v.MessageId))}' for destination {message.Destination}");
}
var changeVisibilityResult = await sqsClient.ChangeMessageVisibilityBatchAsync(new ChangeMessageVisibilityBatchRequest(delayedDeliveryQueueUrl, changeVisibilityBatchRequestEntries), CancellationToken.None)
.ConfigureAwait(false);
if (Logger.IsDebugEnabled)
{
var message = batch.PreparedMessagesBydId.Values.First();
Logger.Debug($"Changed delayed message visibility for batch '{batchNumber}/{totalBatches}' with message ids '{string.Join(", ", batch.PreparedMessagesBydId.Values.Select(v => v.MessageId))}' for destination {message.Destination}");
}
if (Logger.IsDebugEnabled && changeVisibilityResult.Failed.Count > 0)
{
var builder = new StringBuilder();
foreach (var failed in changeVisibilityResult.Failed)
{
builder.AppendLine($"{failed.Id}: {failed.Message} | {failed.Code} | {failed.SenderFault}");
}
Logger.Debug($"Changing visibility failed for {builder}");
}
}
}
catch (Exception e)
{
var builder = new StringBuilder();
foreach (var failed in result.Failed)
{
builder.AppendLine($"{failed.Id}: {failed.Message} | {failed.Code} | {failed.SenderFault}");
}
Logger.Error($"Changing visibility failed for {builder}", e);
}
}
async Task SendBatch(BatchEntry <SqsPreparedMessage> batch, int batchNumber, int totalBatches)
{
try
{
if (Logger.IsDebugEnabled)
{
var message = batch.PreparedMessagesBydId.Values.First();
Logger.Debug($"Sending batch '{batchNumber}/{totalBatches}' with message ids '{string.Join(", ", batch.PreparedMessagesBydId.Values.Select(v => v.MessageId))}' to destination {message.Destination}");
}
var result = await sqsClient.SendMessageBatchAsync(batch.BatchRequest).ConfigureAwait(false);
if (Logger.IsDebugEnabled)
{
var message = batch.PreparedMessagesBydId.Values.First();
Logger.Debug($"Sent batch '{batchNumber}/{totalBatches}' with message ids '{string.Join(", ", batch.PreparedMessagesBydId.Values.Select(v => v.MessageId))}' to destination {message.Destination}");
}
List <Task> redispatchTasks = null;
foreach (var errorEntry in result.Failed)
{
redispatchTasks = redispatchTasks ?? new List <Task>(result.Failed.Count);
var messageToRetry = batch.PreparedMessagesBydId[errorEntry.Id];
Logger.Info($"Retrying message with MessageId {messageToRetry.MessageId} that failed in batch '{batchNumber}/{totalBatches}' due to '{errorEntry.Message}'.");
redispatchTasks.Add(SendMessageForBatch(messageToRetry, batchNumber, totalBatches));
}
if (redispatchTasks != null)
{
await Task.WhenAll(redispatchTasks).ConfigureAwait(false);
}
}
catch (QueueDoesNotExistException e)
{
var message = batch.PreparedMessagesBydId.Values.First();
if (message.OriginalDestination != null)
{
throw new QueueDoesNotExistException($"Unable to send batch '{batchNumber}/{totalBatches}'. Destination '{message.OriginalDestination}' doesn't support delayed messages longer than {TimeSpan.FromSeconds(configuration.DelayedDeliveryQueueDelayTime)}. To enable support for longer delays, call '.UseTransport<SqsTransport>().UnrestrictedDurationDelayedDelivery()' on the '{message.OriginalDestination}' endpoint.", e);
}
Logger.Error($"Error while sending batch '{batchNumber}/{totalBatches}', with message ids '{string.Join(", ", batch.PreparedMessagesBydId.Values.Select(v => v.MessageId))}', to '{message.Destination}'. The destination does not exist.", e);
throw;
}
catch (Exception ex)
{
var message = batch.PreparedMessagesBydId.Values.First();
Logger.Error($"Error while sending batch '{batchNumber}/{totalBatches}', with message ids '{string.Join(", ", batch.PreparedMessagesBydId.Values.Select(v => v.MessageId))}', to '{message.Destination}'", ex);
throw;
}
}
async Task SendDelayedMessagesInBatches(BatchEntry <SqsReceivedDelayedMessage> batch, int batchNumber, int totalBatches)
{
if (Logger.IsDebugEnabled)
{
var message = batch.PreparedMessagesBydId.Values.First();
Logger.Debug($"Sending delayed message batch '{batchNumber}/{totalBatches}' with message ids '{string.Join(", ", batch.PreparedMessagesBydId.Values.Select(v => v.MessageId))}' to destination {message.Destination}");
}
var result = await sqsClient.SendMessageBatchAsync(batch.BatchRequest).ConfigureAwait(false);
if (Logger.IsDebugEnabled)
{
var message = batch.PreparedMessagesBydId.Values.First();
Logger.Debug($"Sent delayed message '{batchNumber}/{totalBatches}' with message ids '{string.Join(", ", batch.PreparedMessagesBydId.Values.Select(v => v.MessageId))}' to destination {message.Destination}");
}
var deletionTask = DeleteDelayedMessagesThatWereDeliveredSuccessfullyAsBatchesInBatches(batch, result, batchNumber, totalBatches);
// deliberately fire&forget because we treat this as a best effort
_ = ChangeVisibilityOfDelayedMessagesThatFailedBatchDeliveryInBatches(batch, result, batchNumber, totalBatches);
await deletionTask.ConfigureAwait(false);
}
async Task DeleteDelayedMessagesThatWereDeliveredSuccessfullyAsBatchesInBatches(BatchEntry <SqsReceivedDelayedMessage> batch, SendMessageBatchResponse result, int batchNumber, int totalBatches)
{
List <DeleteMessageBatchRequestEntry> deleteBatchRequestEntries = null;
foreach (var successful in result.Successful)
{
deleteBatchRequestEntries = deleteBatchRequestEntries ?? new List <DeleteMessageBatchRequestEntry>(result.Successful.Count);
var preparedMessage = batch.PreparedMessagesBydId[successful.Id];
// need to reuse the previous batch entry ID so that we can map again in failure scenarios, this is fine given that IDs only need to be unique per request
deleteBatchRequestEntries.Add(new DeleteMessageBatchRequestEntry(successful.Id, preparedMessage.ReceiptHandle));
}
if (deleteBatchRequestEntries != null)
{
if (Logger.IsDebugEnabled)
{
var message = batch.PreparedMessagesBydId.Values.First();
Logger.Debug($"Deleting delayed message for batch '{batchNumber}/{totalBatches}' with message ids '{string.Join(", ", batch.PreparedMessagesBydId.Values.Select(v => v.MessageId))}' for destination {message.Destination}");
}
var deleteResult = await sqsClient.DeleteMessageBatchAsync(new DeleteMessageBatchRequest(delayedDeliveryQueueUrl, deleteBatchRequestEntries), CancellationToken.None)
.ConfigureAwait(false);
if (Logger.IsDebugEnabled)
{
var message = batch.PreparedMessagesBydId.Values.First();
Logger.Debug($"Deleted delayed message for batch '{batchNumber}/{totalBatches}' with message ids '{string.Join(", ", batch.PreparedMessagesBydId.Values.Select(v => v.MessageId))}' for destination {message.Destination}");
}
List <Task> deleteTasks = null;
foreach (var errorEntry in deleteResult.Failed)
{
deleteTasks = deleteTasks ?? new List <Task>(deleteResult.Failed.Count);
var messageToDeleteWithAnotherAttempt = batch.PreparedMessagesBydId[errorEntry.Id];
Logger.Info($"Retrying message deletion with MessageId {messageToDeleteWithAnotherAttempt.MessageId} that failed in batch '{batchNumber}/{totalBatches}' due to '{errorEntry.Message}'.");
deleteTasks.Add(DeleteMessage(messageToDeleteWithAnotherAttempt));
}
if (deleteTasks != null)
{
await Task.WhenAll(deleteTasks).ConfigureAwait(false);
}
}
}
