async Task SendMessage(string message, string destination, List <DeliveryConstraint> constraints) { var delayWithConstraint = constraints.OfType <DelayDeliveryWith>().SingleOrDefault(); var deliverAtConstraint = constraints.OfType <DoNotDeliverBefore>().SingleOrDefault(); var delayDeliveryBy = TimeSpan.MaxValue; if (delayWithConstraint == null) { if (deliverAtConstraint != null) { delayDeliveryBy = deliverAtConstraint.At - DateTime.UtcNow; } } else { delayDeliveryBy = delayWithConstraint.Delay; } var queueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(destination, configuration)) .ConfigureAwait(false); var sendMessageRequest = new SendMessageRequest(queueUrl, message); // There should be no need to check if the delay time is greater than the maximum allowed // by SQS (15 minutes); the call to AWS will fail with an appropriate exception if the limit is exceeded. if (delayDeliveryBy != TimeSpan.MaxValue) { sendMessageRequest.DelaySeconds = Math.Max(0, (int)delayDeliveryBy.TotalSeconds); } await sqsClient.SendMessageAsync(sendMessageRequest) .ConfigureAwait(false); }
public static async Task CreateQueue(string queueName, TimeSpan?maxTimeToLive = null, string queueNamePrefix = null) { using (var client = ClientFactory.CreateSqsClient()) { var sqsRequest = new CreateQueueRequest { QueueName = QueueNameHelper.GetSqsQueueName(queueName, queueNamePrefix), }; sqsRequest.Attributes.Add(QueueAttributeName.MessageRetentionPeriod, Convert.ToInt32((maxTimeToLive ?? DefaultTimeToLive).TotalSeconds).ToString()); if (queueName.EndsWith(".fifo")) { sqsRequest.Attributes = new Dictionary <string, string> { { QueueAttributeName.FifoQueue, "true" }, { QueueAttributeName.DelaySeconds, "900" } }; } try { var createQueueResponse = await client.CreateQueueAsync(sqsRequest) .ConfigureAwait(false); } catch (QueueNameExistsException) { } } }
static string ReadFailedQueueHeader(Message message) { var headers = ExtractHeaders(message); var queueName = headers.Single(x => x.Key == "NServiceBus.FailedQ").Value; return(QueueNameHelper.GetSqsQueueName(queueName)); }
public async Task Init(Func <MessageContext, Task> onMessage, Func <ErrorContext, Task <ErrorHandleResult> > onError, CriticalError criticalError, PushSettings settings) { queueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(settings.InputQueue, configuration)) .ConfigureAwait(false); errorQueueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(settings.ErrorQueue, configuration)) .ConfigureAwait(false); if (settings.PurgeOnStartup) { // SQS only allows purging a queue once every 60 seconds or so. // If you try to purge a queue twice in relatively quick succession, // PurgeQueueInProgressException will be thrown. // This will happen if you are trying to start an endpoint twice or more // in that time. try { await sqsClient.PurgeQueueAsync(queueUrl, CancellationToken.None).ConfigureAwait(false); } catch (PurgeQueueInProgressException ex) { Logger.Warn("Multiple queue purges within 60 seconds are not permitted by SQS.", ex); } catch (Exception ex) { Logger.Error("Exception thrown from PurgeQueue.", ex); throw; } } this.onMessage = onMessage; this.onError = onError; }
public static async Task SendLargeMessage(IAmazonSQS sqsClient, IAmazonS3 s3Client, string queue, string s3Prefix, string bucketName, string messageBody, Dictionary <string, string> headers) { var bodyBytes = Encoding.UTF8.GetBytes(messageBody); var key = $"{s3Prefix}/{headers["NServiceBus.MessageId"]}"; using (var bodyStream = new MemoryStream(bodyBytes)) { await s3Client.PutObjectAsync(new PutObjectRequest { BucketName = bucketName, InputStream = bodyStream, Key = key }).ConfigureAwait(false); } var serializedMessage = Newtonsoft.Json.JsonConvert.SerializeObject(new { Headers = headers, Body = string.Empty, S3BodyKey = key }); var queueUrlResponse = await sqsClient.GetQueueUrlAsync(QueueNameHelper.GetSqsQueueName(queue)) .ConfigureAwait(false); await sqsClient.SendMessageAsync(queueUrlResponse.QueueUrl, serializedMessage) .ConfigureAwait(false); }
public async Task Init(Func <MessageContext, Task> onMessage, Func <ErrorContext, Task <ErrorHandleResult> > onError, CriticalError criticalError, PushSettings settings) { this.criticalError = criticalError; queueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(settings.InputQueue, configuration)) .ConfigureAwait(false); errorQueueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(settings.ErrorQueue, configuration)) .ConfigureAwait(false); if (configuration.IsDelayedDeliveryEnabled) { var delayedDeliveryQueueName = settings.InputQueue + TransportConfiguration.DelayedDeliveryQueueSuffix; delayedDeliveryQueueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(delayedDeliveryQueueName, configuration)) .ConfigureAwait(false); var queueAttributes = await GetQueueAttributesFromDelayedDeliveryQueueWithRetriesToWorkaroundSDKIssue() .ConfigureAwait(false); if (queueAttributes.DelaySeconds < configuration.DelayedDeliveryQueueDelayTime) { throw new Exception($"Delayed delivery queue '{delayedDeliveryQueueName}' should not have Delivery Delay less than {TimeSpan.FromSeconds(configuration.DelayedDeliveryQueueDelayTime)}."); } if (queueAttributes.MessageRetentionPeriod < (int)TransportConfiguration.DelayedDeliveryQueueMessageRetentionPeriod.TotalSeconds) { throw new Exception($"Delayed delivery queue '{delayedDeliveryQueueName}' should not have Message Retention Period less than {TransportConfiguration.DelayedDeliveryQueueMessageRetentionPeriod}."); } if (queueAttributes.Attributes.ContainsKey("RedrivePolicy")) { throw new Exception($"Delayed delivery queue '{delayedDeliveryQueueName}' should not have Redrive Policy enabled."); } } if (settings.PurgeOnStartup) { // SQS only allows purging a queue once every 60 seconds or so. // If you try to purge a queue twice in relatively quick succession, // PurgeQueueInProgressException will be thrown. // This will happen if you are trying to start an endpoint twice or more // in that time. try { await sqsClient.PurgeQueueAsync(queueUrl, CancellationToken.None).ConfigureAwait(false); } catch (PurgeQueueInProgressException ex) { Logger.Warn("Multiple queue purges within 60 seconds are not permitted by SQS.", ex); } catch (Exception ex) { Logger.Error("Exception thrown from PurgeQueue.", ex); throw; } } this.onMessage = onMessage; this.onError = onError; }
static async Task RetryCreateOnThrottle(IAmazonSQS client, string queueName, TimeSpan delay, int maxRetryAttempts, string queueNamePrefix = null, int retryAttempts = 0) { try { await client.CreateQueueAsync(QueueNameHelper.GetSqsQueueName(queueName, queueNamePrefix)).ConfigureAwait(false); } catch (AmazonServiceException ex) when(ex.ErrorCode == "RequestThrottled") { if (retryAttempts < maxRetryAttempts) { var attempts = TimeSpan.FromMilliseconds(Convert.ToInt32(delay.TotalMilliseconds * (retryAttempts + 1))); Console.WriteLine($"Retry {queueName} {retryAttempts}/{maxRetryAttempts} with delay {attempts}."); await Task.Delay(attempts).ConfigureAwait(false); await RetryCreateOnThrottle(client, queueName, delay, maxRetryAttempts, retryAttempts : ++retryAttempts).ConfigureAwait(false); } else { Console.WriteLine($"Unable to create queue. Max retry attempts reached on {queueName}."); } } catch (AmazonServiceException ex) { Console.WriteLine($"Unable to create {queueName} on {retryAttempts}/{maxRetryAttempts}. Reason: {ex.Message}"); } }
public void Preserves_FifoQueue(string destination, string expected) { var configuration = new TransportConfiguration(new SettingsHolder()); var result = QueueNameHelper.GetSqsQueueName(destination, configuration); Assert.AreEqual(expected, result); }
public WorkflowOperationAsyncResult(WorkflowOperationInvoker workflowOperationInvoker, WorkflowDurableInstance workflowDurableInstance, object[] inputs, AsyncCallback callback, object state, long time) : base(callback, state) { if (inputs == null) { throw DiagnosticUtility.ExceptionUtility.ThrowHelperArgumentNull("inputs"); } if (workflowDurableInstance == null) { throw DiagnosticUtility.ExceptionUtility.ThrowHelperArgumentNull("workflowDurableInstance"); } if (workflowOperationInvoker == null) { throw DiagnosticUtility.ExceptionUtility.ThrowHelperArgumentNull("workflowOperationInvoker"); } string queueName; WorkflowRequestContext workflowRequestContext = new WorkflowRequestContext( this, inputs, GetContextProperties()); queueName = workflowOperationInvoker.StaticQueueName; if (workflowRequestContext.ContextProperties.Count > 1) //DurableDispatchContextProperty. { queueName = QueueNameHelper.Create(workflowOperationInvoker.StaticQueueName, workflowRequestContext.ContextProperties); } WorkflowInstance workflowInstance = workflowDurableInstance.GetWorkflowInstance (workflowOperationInvoker.CanCreateInstance); AsyncCallbackState callbackState = new AsyncCallbackState(workflowRequestContext, workflowInstance, workflowOperationInvoker.DispatchRuntime.SynchronizationContext, workflowOperationInvoker.InstanceLifetimeManager, queueName); this.isOneway = workflowOperationInvoker.IsOneWay; this.instanceIdGuid = workflowInstance.InstanceId; this.time = time; ActionItem.Schedule(waitCallback, callbackState); if (DiagnosticUtility.ShouldTraceVerbose) { string traceText = SR2.GetString(SR2.WorkflowOperationInvokerItemQueued, this.InstanceId, queueName); TraceUtility.TraceEvent(TraceEventType.Verbose, TraceCode.WorkflowOperationInvokerItemQueued, SR.GetString(SR.TraceCodeWorkflowOperationInvokerItemQueued), new StringTraceRecord("ItemDetails", traceText), this, null); } }
public static async Task CreateQueuesForEndpoint(string endpointName, string templatePath, TimeSpan?maxTimeToLive = null, string queueNamePrefix = null, bool includeRetries = false, string delayedDeliveryMethod = "Native") { using (var client = ClientFactory.CreateCloudFormationClient()) { var endpointNameWithPrefix = QueueNameHelper.GetSqsQueueName(endpointName, queueNamePrefix); var request = new CreateStackRequest { StackName = endpointNameWithPrefix, Parameters = new List <Parameter> { new Parameter { ParameterKey = "EndpointName", ParameterValue = endpointNameWithPrefix }, new Parameter { ParameterKey = "MaxTimeToLive", ParameterValue = Convert.ToInt32((maxTimeToLive ?? QueueCreationUtils.DefaultTimeToLive).TotalSeconds).ToString() }, new Parameter { ParameterKey = "IncludeRetries", ParameterValue = includeRetries.ToString() }, new Parameter { ParameterKey = "DelayedDeliveryMethod", ParameterValue = delayedDeliveryMethod }, }, TemplateBody = CloudFormationHelper.ConvertToValidJson(templatePath) }; await client.CreateStackAsync(request) .ConfigureAwait(false); var describeRequest = new DescribeStacksRequest { StackName = endpointNameWithPrefix }; StackStatus currentStatus = string.Empty; while (currentStatus != StackStatus.CREATE_COMPLETE) { var response = await client.DescribeStacksAsync(describeRequest) .ConfigureAwait(false); var stack = response.Stacks.SingleOrDefault(); currentStatus = stack?.StackStatus; await Task.Delay(1000); } } }
async Task SendMessage(string message, string destination, long delaySeconds, string messageId) { try { SendMessageRequest sendMessageRequest; if (configuration.IsDelayedDeliveryEnabled && delaySeconds > configuration.DelayedDeliveryQueueDelayTime) { destination += TransportConfiguration.DelayedDeliveryQueueSuffix; var queueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(destination, configuration)) .ConfigureAwait(false); sendMessageRequest = new SendMessageRequest(queueUrl, message) { MessageAttributes = { [TransportHeaders.DelaySeconds] = new MessageAttributeValue { StringValue = delaySeconds.ToString(), DataType = "String" } }, MessageDeduplicationId = messageId, MessageGroupId = messageId }; } else { var queueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(destination, configuration)) .ConfigureAwait(false); sendMessageRequest = new SendMessageRequest(queueUrl, message); if (delaySeconds > 0) { sendMessageRequest.DelaySeconds = (int)delaySeconds; } } await sqsClient.SendMessageAsync(sendMessageRequest) .ConfigureAwait(false); } catch (QueueDoesNotExistException e) when(destination.EndsWith(TransportConfiguration.DelayedDeliveryQueueSuffix, StringComparison.OrdinalIgnoreCase)) { var queueName = destination.Substring(0, destination.Length - TransportConfiguration.DelayedDeliveryQueueSuffix.Length); throw new QueueDoesNotExistException($"Destination '{queueName}' doesn't support delayed messages longer than {TimeSpan.FromSeconds(configuration.DelayedDeliveryQueueDelayTime)}. To enable support for longer delays, call '.UseTransport<SqsTransport>().UnrestrictedDelayedDelivery()' on the '{queueName}' endpoint.", e); } catch (Exception ex) { Logger.Error($"Error while sending message, with MessageId '{messageId}', to '{destination}'", ex); throw; } }
async Task <string> GetQueueUrl(string queueName) { var sanitizedErrorQueueName = QueueNameHelper.GetSanitizedQueueName(queueName); try { return((await sqsClient.GetQueueUrlAsync(sanitizedErrorQueueName).ConfigureAwait(false)).QueueUrl); } catch (Exception e) { Logger.Error($"Failed to obtain the queue URL for queue {sanitizedErrorQueueName} (derived from configured name {queueName}).", e); throw; } }
public void Preserves_FifoQueue_WithPreTruncate(string destination, string expected) { var settings = new SettingsHolder(); settings.Set(SettingsKeys.PreTruncateQueueNames, true); settings.Set(SettingsKeys.QueueNamePrefix, string.Empty); var configuration = new TransportConfiguration(settings); var result = QueueNameHelper.GetSqsQueueName(destination, configuration); Console.WriteLine(result); Assert.AreEqual(expected, result); }
public static async Task ReturnMessageToSourceQueue(string errorQueueName, string messageId) { var path = QueueNameHelper.GetSqsQueueName(errorQueueName); using (var client = ClientFactory.CreateSqsClient()) { var queueUrlResponse = await client.GetQueueUrlAsync(path) .ConfigureAwait(false); var queueUrl = queueUrlResponse.QueueUrl; await InspectMessagesUntilFound(client, messageId, queueUrl) .ConfigureAwait(false); } }
public static async Task SendMessage(IAmazonSQS sqsClient, string queue, string messageBody, Dictionary <string, string> headers) { var bodyBytes = Encoding.UTF8.GetBytes(messageBody); var base64Body = Convert.ToBase64String(bodyBytes); var serializedMessage = Newtonsoft.Json.JsonConvert.SerializeObject(new { Headers = headers, Body = base64Body, }); var queueUrlResponse = await sqsClient.GetQueueUrlAsync(QueueNameHelper.GetSqsQueueName(queue)) .ConfigureAwait(false); await sqsClient.SendMessageAsync(queueUrlResponse.QueueUrl, serializedMessage) .ConfigureAwait(false); }
public override string ToTransportAddress(LogicalAddress logicalAddress) { var queueName = logicalAddress.EndpointInstance.Endpoint; var queue = new StringBuilder(queueName); if (logicalAddress.EndpointInstance.Discriminator != null) { queue.Append("-" + logicalAddress.EndpointInstance.Discriminator); } if (logicalAddress.Qualifier != null) { queue.Append("-" + logicalAddress.Qualifier); } return(QueueNameHelper.GetSanitizedQueueName(queue, queueName)); }
public static async Task DeleteQueue(string queueName, string queueNamePrefix = null) { try { using (var client = ClientFactory.CreateSqsClient()) { var sqsQueueName = QueueNameHelper.GetSqsQueueName(queueName, queueNamePrefix); var queueUrlResponse = await client.GetQueueUrlAsync(sqsQueueName) .ConfigureAwait(false); await client.DeleteQueueAsync(queueUrlResponse.QueueUrl) .ConfigureAwait(false); } } catch (QueueDoesNotExistException) { } }
public static async Task <bool> Exists(string queueName, string queueNamePrefix = null) { using (var client = ClientFactory.CreateSqsClient()) { try { var sqsQueueName = QueueNameHelper.GetSqsQueueName(queueName, queueNamePrefix); await client.GetQueueUrlAsync(sqsQueueName) .ConfigureAwait(false); return(true); } catch (QueueDoesNotExistException) { return(false); } } }
public static async Task CreateQueue(string queueName, string templatePath, TimeSpan?maxTimeToLive = null, string queueNamePrefix = null) { using (var client = ClientFactory.CreateCloudFormationClient()) { var sqsQueueName = QueueNameHelper.GetSqsQueueName(queueName, queueNamePrefix); var request = new CreateStackRequest { StackName = sqsQueueName, Parameters = new List <Parameter> { new Parameter { ParameterKey = "QueueName", ParameterValue = sqsQueueName }, new Parameter { ParameterKey = "MaxTimeToLive", ParameterValue = Convert.ToInt32((maxTimeToLive ?? DefaultTimeToLive).TotalSeconds).ToString() } }, TemplateBody = CloudFormationHelper.ConvertToValidJson(templatePath) }; await client.CreateStackAsync(request) .ConfigureAwait(false); var describeRequest = new DescribeStacksRequest { StackName = sqsQueueName }; StackStatus currentStatus = string.Empty; while (currentStatus != StackStatus.CREATE_COMPLETE) { var response = await client.DescribeStacksAsync(describeRequest) .ConfigureAwait(false); var stack = response.Stacks.SingleOrDefault(); currentStatus = stack?.StackStatus; await Task.Delay(1000); } } }
public static async Task <GetQueueAttributesResponse> Exists(string queueName, string queueNamePrefix = null, List <string> attributeNames = null) { using (var client = ClientFactory.CreateSqsClient()) { try { var sqsQueueName = QueueNameHelper.GetSqsQueueName(queueName, queueNamePrefix); var response = await client.GetQueueUrlAsync(sqsQueueName).ConfigureAwait(false); var attributesResponse = await client.GetQueueAttributesAsync(response.QueueUrl, attributeNames ?? new List <string>()) .ConfigureAwait(false); return(attributesResponse); } catch (QueueDoesNotExistException) { return(null); } } }
public static async Task CreateQueue(string queueName, TimeSpan?maxTimeToLive = null, string queueNamePrefix = null) { using (var client = ClientFactory.CreateSqsClient()) { var sqsRequest = new CreateQueueRequest { QueueName = QueueNameHelper.GetSqsQueueName(queueName, queueNamePrefix) }; var value = Convert.ToInt32((maxTimeToLive ?? DefaultTimeToLive).TotalSeconds).ToString(); sqsRequest.Attributes.Add(QueueAttributeName.MessageRetentionPeriod, value); try { await client.CreateQueueAsync(sqsRequest) .ConfigureAwait(false); } catch (QueueNameExistsException) { } } }
async Task SendPoisonMessage(string inputQueueName) { var transport = new TransportExtensions <SqsTransport>(new SettingsHolder()); transport = transport.ConfigureSqsTransport(SetupFixture.SqsQueueNamePrefix); var transportConfiguration = new TransportConfiguration(transport.GetSettings()); using (var sqsClient = SqsTransportExtensions.CreateSQSClient()) { var getQueueUrlResponse = await sqsClient.GetQueueUrlAsync(new GetQueueUrlRequest { QueueName = QueueNameHelper.GetSqsQueueName(inputQueueName, transportConfiguration) }).ConfigureAwait(false); await sqsClient.SendMessageAsync(new SendMessageRequest { QueueUrl = getQueueUrlResponse.QueueUrl, MessageBody = PoisonMessageBody }).ConfigureAwait(false); } }
public static async Task DeleteQueuesForEndpoint(string endpointName, string queueNamePrefix = null) { using (var client = ClientFactory.CreateCloudFormationClient()) { var endpointNameWithPrefix = QueueNameHelper.GetSqsQueueName(endpointName, queueNamePrefix); var request = new DeleteStackRequest { StackName = endpointNameWithPrefix, }; await client.DeleteStackAsync(request) .ConfigureAwait(false); var describeRequest = new DescribeStacksRequest { StackName = endpointNameWithPrefix }; StackStatus currentStatus = string.Empty; while (currentStatus != StackStatus.DELETE_IN_PROGRESS) // in progress is enough, no need to wait for completion { try { var response = await client.DescribeStacksAsync(describeRequest) .ConfigureAwait(false); var stack = response.Stacks.SingleOrDefault(); currentStatus = stack?.StackStatus; await Task.Delay(1000); } catch (AmazonCloudFormationException) { Console.WriteLine("Stack does not exist"); return; } } } }
static async Task <string> GetMessageId(string errorQueueName) { var path = QueueNameHelper.GetSqsQueueName($"{errorQueueName}"); using (var client = ClientFactory.CreateSqsClient()) { var getQueueUrlResponse = await client.GetQueueUrlAsync(path) .ConfigureAwait(false); var messages = await client.ReceiveMessageAsync( new ReceiveMessageRequest(getQueueUrlResponse.QueueUrl) { MaxNumberOfMessages = 1, WaitTimeSeconds = 20, VisibilityTimeout = 0, // message needs to be immediately visible again to be seen by the ReturnToSourceQueue algorithm }) .ConfigureAwait(false); var message = messages.Messages.Single(); return(message.MessageId); } }
public static async Task DeleteQueue(string queueName, string queueNamePrefix = null) { using (var client = ClientFactory.CreateCloudFormationClient()) { var sqsQueueName = QueueNameHelper.GetSqsQueueName(queueName, queueNamePrefix); var request = new DeleteStackRequest { StackName = sqsQueueName, }; await client.DeleteStackAsync(request) .ConfigureAwait(false); var describeRequest = new DescribeStacksRequest { StackName = sqsQueueName }; StackStatus currentStatus = string.Empty; while (currentStatus != StackStatus.DELETE_COMPLETE) { try { var response = await client.DescribeStacksAsync(describeRequest) .ConfigureAwait(false); var stack = response.Stacks.SingleOrDefault(); currentStatus = stack?.StackStatus; } catch (AmazonCloudFormationException) { Console.WriteLine("Stack does not exist"); return; } } } }
public WorkflowOperationInvoker(OperationDescription operationDescription, WorkflowOperationBehavior workflowOperationBehavior, WorkflowRuntime workflowRuntime, DispatchRuntime dispatchRuntime) { if (operationDescription == null) { throw DiagnosticUtility.ExceptionUtility.ThrowHelperArgumentNull("operationDescription"); } if (workflowRuntime == null) { throw DiagnosticUtility.ExceptionUtility.ThrowHelperArgumentNull("workflowRuntime"); } if (workflowOperationBehavior == null) { throw DiagnosticUtility.ExceptionUtility.ThrowHelperArgumentNull("workflowOperationBehavior"); } this.isOneWay = operationDescription.IsOneWay; if (operationDescription.BeginMethod != null) { this.syncMethod = operationDescription.BeginMethod; inParameterCount = GetFlowedInParameterCount(operationDescription.BeginMethod.GetParameters()) - 2; } else { this.syncMethod = operationDescription.SyncMethod; inParameterCount = GetFlowedInParameterCount(operationDescription.SyncMethod.GetParameters()); } this.operationDescription = operationDescription; this.workflowRuntime = workflowRuntime; this.canCreateInstance = workflowOperationBehavior.CanCreateInstance; this.serviceAuthorizationManager = workflowOperationBehavior.ServiceAuthorizationManager; this.dispatchRuntime = dispatchRuntime; staticQueueName = QueueNameHelper.Create(this.syncMethod.DeclaringType, operationDescription.Name); }
async Task CheckErrorQueue(string errorQueueName, CancellationToken cancellationToken) { var transport = new TransportExtensions <SqsTransport>(new SettingsHolder()); transport = transport.ConfigureSqsTransport(SetupFixture.SqsQueueNamePrefix); var transportConfiguration = new TransportConfiguration(transport.GetSettings()); using (var sqsClient = SqsTransportExtensions.CreateSQSClient()) { var getQueueUrlResponse = await sqsClient.GetQueueUrlAsync(new GetQueueUrlRequest { QueueName = QueueNameHelper.GetSqsQueueName(errorQueueName, transportConfiguration) }, cancellationToken).ConfigureAwait(false); var messageReceived = false; ReceiveMessageResponse receiveMessageResponse = null; while (messageReceived == false && !cancellationToken.IsCancellationRequested) { receiveMessageResponse = await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = getQueueUrlResponse.QueueUrl, WaitTimeSeconds = 20 }, cancellationToken).ConfigureAwait(false); if (receiveMessageResponse.Messages.Any()) { messageReceived = true; } } Assert.NotNull(receiveMessageResponse); Assert.AreEqual(1, receiveMessageResponse.Messages.Count); Assert.AreEqual(PoisonMessageBody, receiveMessageResponse.Messages.Single().Body); } }
async Task <PreparedMessage> PrepareMessage(UnicastTransportOperation transportOperation) { var delayDeliveryWith = transportOperation.DeliveryConstraints.OfType <DelayDeliveryWith>().SingleOrDefault(); var doNotDeliverBefore = transportOperation.DeliveryConstraints.OfType <DoNotDeliverBefore>().SingleOrDefault(); long delaySeconds = 0; if (delayDeliveryWith != null) { delaySeconds = Convert.ToInt64(Math.Ceiling(delayDeliveryWith.Delay.TotalSeconds)); } else if (doNotDeliverBefore != null) { delaySeconds = Convert.ToInt64(Math.Ceiling((doNotDeliverBefore.At - DateTime.UtcNow).TotalSeconds)); } if (!configuration.IsDelayedDeliveryEnabled && delaySeconds > TransportConfiguration.AwsMaximumQueueDelayTime) { throw new NotSupportedException($"To send messages with a delay time greater than '{TimeSpan.FromSeconds(TransportConfiguration.AwsMaximumQueueDelayTime)}', call '.UseTransport<SqsTransport>().UnrestrictedDelayedDelivery()'."); } var sqsTransportMessage = new TransportMessage(transportOperation.Message, transportOperation.DeliveryConstraints); var serializedMessage = SimpleJson.SerializeObject(sqsTransportMessage, serializerStrategy); var messageId = transportOperation.Message.MessageId; if (serializedMessage.Length > TransportConfiguration.MaximumMessageSize) { if (string.IsNullOrEmpty(configuration.S3BucketForLargeMessages)) { throw new Exception("Cannot send large message because no S3 bucket was configured. Add an S3 bucket name to your configuration."); } var key = $"{configuration.S3KeyPrefix}/{messageId}"; using (var bodyStream = new MemoryStream(transportOperation.Message.Body)) { var request = new PutObjectRequest { BucketName = configuration.S3BucketForLargeMessages, InputStream = bodyStream, Key = key }; if (configuration.S3EncryptionMethod != null) { request.ServerSideEncryptionMethod = configuration.S3EncryptionMethod; } await s3Client.PutObjectAsync(request).ConfigureAwait(false); } sqsTransportMessage.S3BodyKey = key; sqsTransportMessage.Body = string.Empty; serializedMessage = SimpleJson.SerializeObject(sqsTransportMessage, serializerStrategy); } var preparedMessage = new PreparedMessage(); var delayLongerThanConfiguredDelayedDeliveryQueueDelayTime = configuration.IsDelayedDeliveryEnabled && delaySeconds > configuration.DelayedDeliveryQueueDelayTime; if (delayLongerThanConfiguredDelayedDeliveryQueueDelayTime) { preparedMessage.OriginalDestination = transportOperation.Destination; preparedMessage.Destination = $"{transportOperation.Destination}{TransportConfiguration.DelayedDeliveryQueueSuffix}"; preparedMessage.QueueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(preparedMessage.Destination, configuration)) .ConfigureAwait(false); preparedMessage.MessageDeduplicationId = messageId; preparedMessage.MessageGroupId = messageId; preparedMessage.MessageAttributes[TransportHeaders.DelaySeconds] = new MessageAttributeValue { StringValue = delaySeconds.ToString(), DataType = "String" }; } else { preparedMessage.Destination = transportOperation.Destination; preparedMessage.QueueUrl = await queueUrlCache.GetQueueUrl(QueueNameHelper.GetSqsQueueName(preparedMessage.Destination, configuration)) .ConfigureAwait(false); preparedMessage.MessageAttributes[Headers.MessageId] = new MessageAttributeValue { StringValue = messageId, DataType = "String" }; if (delaySeconds > 0) { preparedMessage.DelaySeconds = Convert.ToInt32(delaySeconds); } } preparedMessage.Body = serializedMessage; preparedMessage.MessageId = messageId; return(preparedMessage); }
public void Setup() { DeleteEndpointQueues.DeleteQueuesForEndpoint(QueueNameHelper.GetSqsQueueName(endpointName)).GetAwaiter().GetResult(); QueueDeletionUtils.DeleteQueue(QueueNameHelper.GetSqsQueueName(errorQueueName)).GetAwaiter().GetResult(); }
async Task CreateQueueIfNecessary(string address, bool createDelayedDeliveryQueue) { try { var queueName = QueueNameHelper.GetSqsQueueName(address, configuration); var sqsRequest = new CreateQueueRequest { QueueName = queueName }; Logger.Info($"Creating SQS Queue with name '{sqsRequest.QueueName}' for address '{address}'."); var createQueueResponse = await sqsClient.CreateQueueAsync(sqsRequest).ConfigureAwait(false); queueUrlCache.SetQueueUrl(queueName, createQueueResponse.QueueUrl); // Set the queue attributes in a separate call. // If you call CreateQueue with a queue name that already exists, and with a different // value for MessageRetentionPeriod, the service throws. This will happen if you // change the MaxTTLDays configuration property. var sqsAttributesRequest = new SetQueueAttributesRequest { QueueUrl = createQueueResponse.QueueUrl }; sqsAttributesRequest.Attributes.Add(QueueAttributeName.MessageRetentionPeriod, configuration.MaxTimeToLive.TotalSeconds.ToString(CultureInfo.InvariantCulture)); await sqsClient.SetQueueAttributesAsync(sqsAttributesRequest).ConfigureAwait(false); if (createDelayedDeliveryQueue) { queueName = QueueNameHelper.GetSqsQueueName(address + TransportConfiguration.DelayedDeliveryQueueSuffix, configuration); sqsRequest = new CreateQueueRequest { QueueName = queueName, Attributes = new Dictionary <string, string> { { "FifoQueue", "true" } } }; Logger.Info($"Creating SQS delayed delivery queue with name '{sqsRequest.QueueName}' for address '{address}'."); createQueueResponse = await sqsClient.CreateQueueAsync(sqsRequest).ConfigureAwait(false); queueUrlCache.SetQueueUrl(queueName, createQueueResponse.QueueUrl); sqsAttributesRequest = new SetQueueAttributesRequest { QueueUrl = createQueueResponse.QueueUrl }; sqsAttributesRequest.Attributes.Add(QueueAttributeName.MessageRetentionPeriod, TransportConfiguration.DelayedDeliveryQueueMessageRetentionPeriod.TotalSeconds.ToString(CultureInfo.InvariantCulture)); sqsAttributesRequest.Attributes.Add(QueueAttributeName.DelaySeconds, configuration.DelayedDeliveryQueueDelayTime.ToString(CultureInfo.InvariantCulture)); await sqsClient.SetQueueAttributesAsync(sqsAttributesRequest).ConfigureAwait(false); } if (!string.IsNullOrEmpty(configuration.S3BucketForLargeMessages)) { // determine if the configured bucket exists; create it if it doesn't var listBucketsResponse = await s3Client.ListBucketsAsync(new ListBucketsRequest()).ConfigureAwait(false); var bucketExists = listBucketsResponse.Buckets.Any(x => string.Equals(x.BucketName, configuration.S3BucketForLargeMessages, StringComparison.InvariantCultureIgnoreCase)); if (!bucketExists) { await s3Client.RetryConflictsAsync(async() => await s3Client.PutBucketAsync(new PutBucketRequest { BucketName = configuration.S3BucketForLargeMessages }).ConfigureAwait(false), onRetry : x => { Logger.Warn($"Conflict when creating S3 bucket, retrying after {x}ms."); }).ConfigureAwait(false); } var lifecycleConfig = await s3Client.GetLifecycleConfigurationAsync(configuration.S3BucketForLargeMessages).ConfigureAwait(false); var setLifecycleConfig = lifecycleConfig.Configuration.Rules.All(x => x.Id != "NServiceBus.SQS.DeleteMessageBodies"); if (setLifecycleConfig) { await s3Client.RetryConflictsAsync(async() => await s3Client.PutLifecycleConfigurationAsync(new PutLifecycleConfigurationRequest { BucketName = configuration.S3BucketForLargeMessages, Configuration = new LifecycleConfiguration { Rules = new List <LifecycleRule> { new LifecycleRule { Id = "NServiceBus.SQS.DeleteMessageBodies", Filter = new LifecycleFilter { LifecycleFilterPredicate = new LifecyclePrefixPredicate { Prefix = configuration.S3KeyPrefix } }, Status = LifecycleRuleStatus.Enabled, Expiration = new LifecycleRuleExpiration { Days = (int)Math.Ceiling(configuration.MaxTimeToLive.TotalDays) } } } } }).ConfigureAwait(false), onRetry : x => { Logger.Warn($"Conflict when setting S3 lifecycle configuration, retrying after {x}ms."); }).ConfigureAwait(false); } } } catch (Exception e) { Logger.Error("Exception from CreateQueueIfNecessary.", e); throw; } }