public static SendMessageRequest ToRequest(this PreparedMessage message) { return(new SendMessageRequest(message.QueueUrl, message.Body) { MessageGroupId = message.MessageGroupId, MessageDeduplicationId = message.MessageDeduplicationId, MessageAttributes = message.MessageAttributes, DelaySeconds = message.DelaySeconds }); }
static SendMessageBatchRequestEntry ToBatchEntry(this PreparedMessage message, string batchEntryId) { return(new SendMessageBatchRequestEntry(batchEntryId, message.Body) { MessageAttributes = message.MessageAttributes, MessageGroupId = message.MessageGroupId, MessageDeduplicationId = message.MessageDeduplicationId, DelaySeconds = message.DelaySeconds }); }
public static IReadOnlyList <BatchEntry> Batch(IEnumerable <PreparedMessage> preparedMessages) { var allBatches = new List <BatchEntry>(); var currentDestinationBatches = new Dictionary <string, PreparedMessage>(); var groupByDestination = preparedMessages.GroupBy(m => m.QueueUrl, StringComparer.Ordinal); foreach (var group in groupByDestination) { PreparedMessage firstMessage = null; var payloadSize = 0L; foreach (var message in group) { firstMessage = firstMessage ?? message; // Assumes the size was already calculated by the dispatcher var size = message.Size; payloadSize += size; if (payloadSize > TransportConfiguration.MaximumMessageSize) { allBatches.Add(message.ToBatchRequest(currentDestinationBatches)); currentDestinationBatches.Clear(); payloadSize = size; } // we don't have to recheck payload size here because the support layer checks that a request can always fit 256 KB size limit // we can't take MessageId because batch request ID can only contain alphanumeric characters, hyphen and underscores, message id could be overloaded currentDestinationBatches.Add(Guid.NewGuid().ToString(), message); var currentCount = currentDestinationBatches.Count; if (currentCount != 0 && currentCount % TransportConfiguration.MaximumItemsInBatch == 0) { allBatches.Add(message.ToBatchRequest(currentDestinationBatches)); currentDestinationBatches.Clear(); payloadSize = 0; } } if (currentDestinationBatches.Count > 0) { allBatches.Add(firstMessage.ToBatchRequest(currentDestinationBatches)); currentDestinationBatches.Clear(); } } return(allBatches); }
async Task SendMessage(PreparedMessage message) { try { await sqsClient.SendMessageAsync(message.ToRequest()) .ConfigureAwait(false); } catch (QueueDoesNotExistException e) when(message.OriginalDestination != null) { throw new QueueDoesNotExistException($"Destination '{message.OriginalDestination}' doesn't support delayed messages longer than {TimeSpan.FromSeconds(configuration.DelayedDeliveryQueueDelayTime)}. To enable support for longer delays, call '.UseTransport<SqsTransport>().UnrestrictedDelayedDelivery()' on the '{message.OriginalDestination}' endpoint.", e); } catch (Exception ex) { Logger.Error($"Error while sending message, with MessageId '{message.MessageId}', to '{message.Destination}'", ex); throw; } }
public static BatchEntry ToBatchRequest(this PreparedMessage message, Dictionary <string, PreparedMessage> batchEntries) { var preparedMessagesBydId = batchEntries.ToDictionary(x => x.Key, x => x.Value); var batchRequestEntries = new List <SendMessageBatchRequestEntry>(); foreach (var kvp in preparedMessagesBydId) { batchRequestEntries.Add(kvp.Value.ToBatchEntry(kvp.Key)); } return(new BatchEntry { BatchRequest = new SendMessageBatchRequest(message.QueueUrl, batchRequestEntries), PreparedMessagesBydId = preparedMessagesBydId }); }
async Task <PreparedMessage> PrepareMessage(UnicastTransportOperation transportOperation) { var delayDeliveryWith = transportOperation.DeliveryConstraints.OfType <DelayDeliveryWith>().SingleOrDefault(); var doNotDeliverBefore = transportOperation.DeliveryConstraints.OfType <DoNotDeliverBefore>().SingleOrDefault(); long delaySeconds = 0; if (delayDeliveryWith != null) { delaySeconds = Convert.ToInt64(Math.Ceiling(delayDeliveryWith.Delay.TotalSeconds)); } else if (doNotDeliverBefore != null) { delaySeconds = Convert.ToInt64(Math.Ceiling((doNotDeliverBefore.At - DateTime.UtcNow).TotalSeconds)); } if (!configuration.IsDelayedDeliveryEnabled && delaySeconds > TransportConfiguration.AwsMaximumQueueDelayTime) { throw new NotSupportedException($"To send messages with a delay time greater than '{TimeSpan.FromSeconds(TransportConfiguration.AwsMaximumQueueDelayTime)}', call '.UseTransport<SqsTransport>().UnrestrictedDelayedDelivery()'."); } var sqsTransportMessage = new TransportMessage(transportOperation.Message, transportOperation.DeliveryConstraints); var serializedMessage = SimpleJson.SerializeObject(sqsTransportMessage, serializerStrategy); var messageId = transportOperation.Message.MessageId; if (serializedMessage.Length > TransportConfiguration.MaximumMessageSize) { if (string.IsNullOrEmpty(configuration.S3BucketForLargeMessages)) { throw new Exception("Cannot send large message because no S3 bucket was configured. Add an S3 bucket name to your configuration."); } var key = $"{configuration.S3KeyPrefix}/{messageId}"; using (var bodyStream = new MemoryStream(transportOperation.Message.Body)) { var request = new PutObjectRequest { BucketName = configuration.S3BucketForLargeMessages, InputStream = bodyStream, Key = key }; if (configuration.S3EncryptionMethod != null) { request.ServerSideEncryptionMethod = configuration.S3EncryptionMethod; } await s3Client.PutObjectAsync(request).ConfigureAwait(false); } sqsTransportMessage.S3BodyKey = key; sqsTransportMessage.Body = string.Empty; serializedMessage = SimpleJson.SerializeObject(sqsTransportMessage, serializerStrategy); } var preparedMessage = new PreparedMessage(); var delayLongerThanConfiguredDelayedDeliveryQueueDelayTime = configuration.IsDelayedDeliveryEnabled && delaySeconds > configuration.DelayedDeliveryQueueDelayTime; if (delayLongerThanConfiguredDelayedDeliveryQueueDelayTime) { preparedMessage.OriginalDestination = transportOperation.Destination; preparedMessage.Destination = $"{transportOperation.Destination}{TransportConfiguration.DelayedDeliveryQueueSuffix}"; preparedMessage.QueueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(preparedMessage.Destination, configuration)) .ConfigureAwait(false); preparedMessage.MessageDeduplicationId = messageId; preparedMessage.MessageGroupId = messageId; preparedMessage.MessageAttributes[TransportHeaders.DelaySeconds] = new MessageAttributeValue { StringValue = delaySeconds.ToString(), DataType = "String" }; } else { preparedMessage.Destination = transportOperation.Destination; preparedMessage.QueueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(preparedMessage.Destination, configuration)) .ConfigureAwait(false); preparedMessage.MessageAttributes[Headers.MessageId] = new MessageAttributeValue { StringValue = messageId, DataType = "String" }; if (delaySeconds > 0) { preparedMessage.DelaySeconds = Convert.ToInt32(delaySeconds); } } preparedMessage.Body = serializedMessage; preparedMessage.MessageId = messageId; return(preparedMessage); }
async Task SendMessageForBatch(PreparedMessage message, int batchNumber, int totalBatches) { await SendMessage(message).ConfigureAwait(false); Logger.Info($"Retried message with MessageId {message.MessageId} that failed in batch '{batchNumber}/{totalBatches}'."); }