public static SendMessageRequest ToRequest(this PreparedMessage message)
 {
     return(new SendMessageRequest(message.QueueUrl, message.Body)
     {
         MessageGroupId = message.MessageGroupId,
         MessageDeduplicationId = message.MessageDeduplicationId,
         MessageAttributes = message.MessageAttributes,
         DelaySeconds = message.DelaySeconds
     });
 }
 static SendMessageBatchRequestEntry ToBatchEntry(this PreparedMessage message, string batchEntryId)
 {
     return(new SendMessageBatchRequestEntry(batchEntryId, message.Body)
     {
         MessageAttributes = message.MessageAttributes,
         MessageGroupId = message.MessageGroupId,
         MessageDeduplicationId = message.MessageDeduplicationId,
         DelaySeconds = message.DelaySeconds
     });
 }
        public static IReadOnlyList <BatchEntry> Batch(IEnumerable <PreparedMessage> preparedMessages)
        {
            var allBatches = new List <BatchEntry>();
            var currentDestinationBatches = new Dictionary <string, PreparedMessage>();

            var groupByDestination = preparedMessages.GroupBy(m => m.QueueUrl, StringComparer.Ordinal);

            foreach (var group in groupByDestination)
            {
                PreparedMessage firstMessage = null;
                var             payloadSize  = 0L;
                foreach (var message in group)
                {
                    firstMessage = firstMessage ?? message;

                    // Assumes the size was already calculated by the dispatcher
                    var size = message.Size;
                    payloadSize += size;

                    if (payloadSize > TransportConfiguration.MaximumMessageSize)
                    {
                        allBatches.Add(message.ToBatchRequest(currentDestinationBatches));
                        currentDestinationBatches.Clear();
                        payloadSize = size;
                    }

                    // we don't have to recheck payload size here because the support layer checks that a request can always fit 256 KB size limit
                    // we can't take MessageId because batch request ID can only contain alphanumeric characters, hyphen and underscores, message id could be overloaded
                    currentDestinationBatches.Add(Guid.NewGuid().ToString(), message);

                    var currentCount = currentDestinationBatches.Count;
                    if (currentCount != 0 && currentCount % TransportConfiguration.MaximumItemsInBatch == 0)
                    {
                        allBatches.Add(message.ToBatchRequest(currentDestinationBatches));
                        currentDestinationBatches.Clear();
                        payloadSize = 0;
                    }
                }

                if (currentDestinationBatches.Count > 0)
                {
                    allBatches.Add(firstMessage.ToBatchRequest(currentDestinationBatches));
                    currentDestinationBatches.Clear();
                }
            }

            return(allBatches);
        }
Example #4
0
 async Task SendMessage(PreparedMessage message)
 {
     try
     {
         await sqsClient.SendMessageAsync(message.ToRequest())
         .ConfigureAwait(false);
     }
     catch (QueueDoesNotExistException e) when(message.OriginalDestination != null)
     {
         throw new QueueDoesNotExistException($"Destination '{message.OriginalDestination}' doesn't support delayed messages longer than {TimeSpan.FromSeconds(configuration.DelayedDeliveryQueueDelayTime)}. To enable support for longer delays, call '.UseTransport<SqsTransport>().UnrestrictedDelayedDelivery()' on the '{message.OriginalDestination}' endpoint.", e);
     }
     catch (Exception ex)
     {
         Logger.Error($"Error while sending message, with MessageId '{message.MessageId}', to '{message.Destination}'", ex);
         throw;
     }
 }
        public static BatchEntry ToBatchRequest(this PreparedMessage message, Dictionary <string, PreparedMessage> batchEntries)
        {
            var preparedMessagesBydId = batchEntries.ToDictionary(x => x.Key, x => x.Value);

            var batchRequestEntries = new List <SendMessageBatchRequestEntry>();

            foreach (var kvp in preparedMessagesBydId)
            {
                batchRequestEntries.Add(kvp.Value.ToBatchEntry(kvp.Key));
            }

            return(new BatchEntry
            {
                BatchRequest = new SendMessageBatchRequest(message.QueueUrl, batchRequestEntries),
                PreparedMessagesBydId = preparedMessagesBydId
            });
        }
Example #6
0
        async Task <PreparedMessage> PrepareMessage(UnicastTransportOperation transportOperation)
        {
            var delayDeliveryWith  = transportOperation.DeliveryConstraints.OfType <DelayDeliveryWith>().SingleOrDefault();
            var doNotDeliverBefore = transportOperation.DeliveryConstraints.OfType <DoNotDeliverBefore>().SingleOrDefault();

            long delaySeconds = 0;

            if (delayDeliveryWith != null)
            {
                delaySeconds = Convert.ToInt64(Math.Ceiling(delayDeliveryWith.Delay.TotalSeconds));
            }
            else if (doNotDeliverBefore != null)
            {
                delaySeconds = Convert.ToInt64(Math.Ceiling((doNotDeliverBefore.At - DateTime.UtcNow).TotalSeconds));
            }

            if (!configuration.IsDelayedDeliveryEnabled && delaySeconds > TransportConfiguration.AwsMaximumQueueDelayTime)
            {
                throw new NotSupportedException($"To send messages with a delay time greater than '{TimeSpan.FromSeconds(TransportConfiguration.AwsMaximumQueueDelayTime)}', call '.UseTransport<SqsTransport>().UnrestrictedDelayedDelivery()'.");
            }

            var sqsTransportMessage = new TransportMessage(transportOperation.Message, transportOperation.DeliveryConstraints);

            var serializedMessage = SimpleJson.SerializeObject(sqsTransportMessage, serializerStrategy);

            var messageId = transportOperation.Message.MessageId;

            if (serializedMessage.Length > TransportConfiguration.MaximumMessageSize)
            {
                if (string.IsNullOrEmpty(configuration.S3BucketForLargeMessages))
                {
                    throw new Exception("Cannot send large message because no S3 bucket was configured. Add an S3 bucket name to your configuration.");
                }

                var key = $"{configuration.S3KeyPrefix}/{messageId}";

                using (var bodyStream = new MemoryStream(transportOperation.Message.Body))
                {
                    var request = new PutObjectRequest
                    {
                        BucketName  = configuration.S3BucketForLargeMessages,
                        InputStream = bodyStream,
                        Key         = key
                    };

                    if (configuration.S3EncryptionMethod != null)
                    {
                        request.ServerSideEncryptionMethod = configuration.S3EncryptionMethod;
                    }

                    await s3Client.PutObjectAsync(request).ConfigureAwait(false);
                }

                sqsTransportMessage.S3BodyKey = key;
                sqsTransportMessage.Body      = string.Empty;
                serializedMessage             = SimpleJson.SerializeObject(sqsTransportMessage, serializerStrategy);
            }

            var preparedMessage = new PreparedMessage();

            var delayLongerThanConfiguredDelayedDeliveryQueueDelayTime = configuration.IsDelayedDeliveryEnabled && delaySeconds > configuration.DelayedDeliveryQueueDelayTime;

            if (delayLongerThanConfiguredDelayedDeliveryQueueDelayTime)
            {
                preparedMessage.OriginalDestination = transportOperation.Destination;
                preparedMessage.Destination         = $"{transportOperation.Destination}{TransportConfiguration.DelayedDeliveryQueueSuffix}";
                preparedMessage.QueueUrl            = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(preparedMessage.Destination, configuration))
                                                      .ConfigureAwait(false);

                preparedMessage.MessageDeduplicationId = messageId;
                preparedMessage.MessageGroupId         = messageId;

                preparedMessage.MessageAttributes[TransportHeaders.DelaySeconds] = new MessageAttributeValue
                {
                    StringValue = delaySeconds.ToString(),
                    DataType    = "String"
                };
            }
            else
            {
                preparedMessage.Destination = transportOperation.Destination;
                preparedMessage.QueueUrl    = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(preparedMessage.Destination, configuration))
                                              .ConfigureAwait(false);

                preparedMessage.MessageAttributes[Headers.MessageId] = new MessageAttributeValue
                {
                    StringValue = messageId,
                    DataType    = "String"
                };

                if (delaySeconds > 0)
                {
                    preparedMessage.DelaySeconds = Convert.ToInt32(delaySeconds);
                }
            }

            preparedMessage.Body      = serializedMessage;
            preparedMessage.MessageId = messageId;

            return(preparedMessage);
        }
Example #7
0
        async Task SendMessageForBatch(PreparedMessage message, int batchNumber, int totalBatches)
        {
            await SendMessage(message).ConfigureAwait(false);

            Logger.Info($"Retried message with MessageId {message.MessageId} that failed in batch '{batchNumber}/{totalBatches}'.");
        }