public static SendMessageRequest ToRequest(this PreparedMessage message)
{
return(new SendMessageRequest(message.QueueUrl, message.Body)
{
MessageGroupId = message.MessageGroupId,
MessageDeduplicationId = message.MessageDeduplicationId,
MessageAttributes = message.MessageAttributes,
DelaySeconds = message.DelaySeconds
});
}
static SendMessageBatchRequestEntry ToBatchEntry(this PreparedMessage message, string batchEntryId)
{
return(new SendMessageBatchRequestEntry(batchEntryId, message.Body)
{
MessageAttributes = message.MessageAttributes,
MessageGroupId = message.MessageGroupId,
MessageDeduplicationId = message.MessageDeduplicationId,
DelaySeconds = message.DelaySeconds
});
}
public static IReadOnlyList <BatchEntry> Batch(IEnumerable <PreparedMessage> preparedMessages)
{
var allBatches = new List <BatchEntry>();
var currentDestinationBatches = new Dictionary <string, PreparedMessage>();
var groupByDestination = preparedMessages.GroupBy(m => m.QueueUrl, StringComparer.Ordinal);
foreach (var group in groupByDestination)
{
PreparedMessage firstMessage = null;
var payloadSize = 0L;
foreach (var message in group)
{
firstMessage = firstMessage ?? message;
// Assumes the size was already calculated by the dispatcher
var size = message.Size;
payloadSize += size;
if (payloadSize > TransportConfiguration.MaximumMessageSize)
{
allBatches.Add(message.ToBatchRequest(currentDestinationBatches));
currentDestinationBatches.Clear();
payloadSize = size;
}
// we don't have to recheck payload size here because the support layer checks that a request can always fit 256 KB size limit
// we can't take MessageId because batch request ID can only contain alphanumeric characters, hyphen and underscores, message id could be overloaded
currentDestinationBatches.Add(Guid.NewGuid().ToString(), message);
var currentCount = currentDestinationBatches.Count;
if (currentCount != 0 && currentCount % TransportConfiguration.MaximumItemsInBatch == 0)
{
allBatches.Add(message.ToBatchRequest(currentDestinationBatches));
currentDestinationBatches.Clear();
payloadSize = 0;
}
}
if (currentDestinationBatches.Count > 0)
{
allBatches.Add(firstMessage.ToBatchRequest(currentDestinationBatches));
currentDestinationBatches.Clear();
}
}
return(allBatches);
}
async Task SendMessage(PreparedMessage message)
{
try
{
await sqsClient.SendMessageAsync(message.ToRequest())
.ConfigureAwait(false);
}
catch (QueueDoesNotExistException e) when(message.OriginalDestination != null)
{
throw new QueueDoesNotExistException($"Destination '{message.OriginalDestination}' doesn't support delayed messages longer than {TimeSpan.FromSeconds(configuration.DelayedDeliveryQueueDelayTime)}. To enable support for longer delays, call '.UseTransport<SqsTransport>().UnrestrictedDelayedDelivery()' on the '{message.OriginalDestination}' endpoint.", e);
}
catch (Exception ex)
{
Logger.Error($"Error while sending message, with MessageId '{message.MessageId}', to '{message.Destination}'", ex);
throw;
}
}
public static BatchEntry ToBatchRequest(this PreparedMessage message, Dictionary <string, PreparedMessage> batchEntries)
{
var preparedMessagesBydId = batchEntries.ToDictionary(x => x.Key, x => x.Value);
var batchRequestEntries = new List <SendMessageBatchRequestEntry>();
foreach (var kvp in preparedMessagesBydId)
{
batchRequestEntries.Add(kvp.Value.ToBatchEntry(kvp.Key));
}
return(new BatchEntry
{
BatchRequest = new SendMessageBatchRequest(message.QueueUrl, batchRequestEntries),
PreparedMessagesBydId = preparedMessagesBydId
});
}
async Task <PreparedMessage> PrepareMessage(UnicastTransportOperation transportOperation)
{
var delayDeliveryWith = transportOperation.DeliveryConstraints.OfType <DelayDeliveryWith>().SingleOrDefault();
var doNotDeliverBefore = transportOperation.DeliveryConstraints.OfType <DoNotDeliverBefore>().SingleOrDefault();
long delaySeconds = 0;
if (delayDeliveryWith != null)
{
delaySeconds = Convert.ToInt64(Math.Ceiling(delayDeliveryWith.Delay.TotalSeconds));
}
else if (doNotDeliverBefore != null)
{
delaySeconds = Convert.ToInt64(Math.Ceiling((doNotDeliverBefore.At - DateTime.UtcNow).TotalSeconds));
}
if (!configuration.IsDelayedDeliveryEnabled && delaySeconds > TransportConfiguration.AwsMaximumQueueDelayTime)
{
throw new NotSupportedException($"To send messages with a delay time greater than '{TimeSpan.FromSeconds(TransportConfiguration.AwsMaximumQueueDelayTime)}', call '.UseTransport<SqsTransport>().UnrestrictedDelayedDelivery()'.");
}
var sqsTransportMessage = new TransportMessage(transportOperation.Message, transportOperation.DeliveryConstraints);
var serializedMessage = SimpleJson.SerializeObject(sqsTransportMessage, serializerStrategy);
var messageId = transportOperation.Message.MessageId;
if (serializedMessage.Length > TransportConfiguration.MaximumMessageSize)
{
if (string.IsNullOrEmpty(configuration.S3BucketForLargeMessages))
{
throw new Exception("Cannot send large message because no S3 bucket was configured. Add an S3 bucket name to your configuration.");
}
var key = $"{configuration.S3KeyPrefix}/{messageId}";
using (var bodyStream = new MemoryStream(transportOperation.Message.Body))
{
var request = new PutObjectRequest
{
BucketName = configuration.S3BucketForLargeMessages,
InputStream = bodyStream,
Key = key
};
if (configuration.S3EncryptionMethod != null)
{
request.ServerSideEncryptionMethod = configuration.S3EncryptionMethod;
}
await s3Client.PutObjectAsync(request).ConfigureAwait(false);
}
sqsTransportMessage.S3BodyKey = key;
sqsTransportMessage.Body = string.Empty;
serializedMessage = SimpleJson.SerializeObject(sqsTransportMessage, serializerStrategy);
}
var preparedMessage = new PreparedMessage();
var delayLongerThanConfiguredDelayedDeliveryQueueDelayTime = configuration.IsDelayedDeliveryEnabled && delaySeconds > configuration.DelayedDeliveryQueueDelayTime;
if (delayLongerThanConfiguredDelayedDeliveryQueueDelayTime)
{
preparedMessage.OriginalDestination = transportOperation.Destination;
preparedMessage.Destination = $"{transportOperation.Destination}{TransportConfiguration.DelayedDeliveryQueueSuffix}";
preparedMessage.QueueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(preparedMessage.Destination, configuration))
.ConfigureAwait(false);
preparedMessage.MessageDeduplicationId = messageId;
preparedMessage.MessageGroupId = messageId;
preparedMessage.MessageAttributes[TransportHeaders.DelaySeconds] = new MessageAttributeValue
{
StringValue = delaySeconds.ToString(),
DataType = "String"
};
}
else
{
preparedMessage.Destination = transportOperation.Destination;
preparedMessage.QueueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(preparedMessage.Destination, configuration))
.ConfigureAwait(false);
preparedMessage.MessageAttributes[Headers.MessageId] = new MessageAttributeValue
{
StringValue = messageId,
DataType = "String"
};
if (delaySeconds > 0)
{
preparedMessage.DelaySeconds = Convert.ToInt32(delaySeconds);
}
}
preparedMessage.Body = serializedMessage;
preparedMessage.MessageId = messageId;
return(preparedMessage);
}
async Task SendMessageForBatch(PreparedMessage message, int batchNumber, int totalBatches)
{
await SendMessage(message).ConfigureAwait(false);
Logger.Info($"Retried message with MessageId {message.MessageId} that failed in batch '{batchNumber}/{totalBatches}'.");
}