private List <SendMessageBatchRequestEntry> CreateMessageBatchRequestEntries <T>(IEnumerable <T> entries) { List <SendMessageBatchRequestEntry> sendMesssageReqs = new List <SendMessageBatchRequestEntry>(); int id = 0; foreach (var payload in entries) { SendMessageBatchRequestEntry batchReqEntry = new SendMessageBatchRequestEntry(id.ToString(), JsonConvert.SerializeObject(payload)); //batchReqEntry.DelaySeconds = DelaySeconds; if (string.IsNullOrEmpty(batchReqEntry.MessageBody)) { continue; } // TODO: this can be redone to use attributes passed directly to this method. AppendCustomAttributes(MessageAttributes, batchReqEntry.MessageAttributes); AppendMessageTypeAttribute((typeof(T)).FullName, batchReqEntry.MessageAttributes); sendMesssageReqs.Add(batchReqEntry); id++; //increment message id; } return(sendMesssageReqs); }
SendMessageBatchRequestEntry GetBatchRequestEntry(OutgoingMessage message) { var transportMessage = message.TransportMessage; var headers = transportMessage.Headers; var messageId = headers[Headers.MessageId]; var sqsMessage = new AmazonSQSTransportMessage(transportMessage.Headers, GetBody(transportMessage.Body)); var entry = new SendMessageBatchRequestEntry(messageId, _serializer.Serialize(sqsMessage)); var delaySeconds = GetDelaySeconds(headers); if (delaySeconds != null) { entry.DelaySeconds = delaySeconds.Value; } if (message.DestinationAddress.EndsWith(".fifo")) { if (headers.ContainsKey(MessageGroupIdHeader)) { entry.MessageGroupId = headers[MessageGroupIdHeader]; } if (headers.ContainsKey(MessageDeduplicationIdHeader)) { entry.MessageDeduplicationId = headers[MessageDeduplicationIdHeader]; } } return(entry); }
private async Task <SendMessageBatchRequestEntry> StoreMessageAsync(SendMessageBatchRequestEntry sendMessageRequest, CancellationToken cancellationToken = default(CancellationToken)) { if (_storageClient == null) { return(sendMessageRequest); } CheckMessageAttributes(sendMessageRequest.MessageAttributes); var messageContentStr = sendMessageRequest.MessageBody; var messageContentSize = Encoding.UTF8.GetBytes(messageContentStr).Length; // LongLength; var messageAttributeValue = new MessageAttributeValue { DataType = "Number", StringValue = messageContentSize.ToString() }; // !!! sendMessageRequest.MessageAttributes.Add(SQSConstants.RESERVED_ATTRIBUTE_NAME, messageAttributeValue); var storeReference = await _storageClient.PutDataAsync(sendMessageRequest.MessageBody, null, cancellationToken).ConfigureAwait(false); sendMessageRequest.MessageBody = JsonConvert.SerializeObject(storeReference); return(sendMessageRequest); }
private bool IsLarge(SendMessageBatchRequestEntry batchEntry) { var contentSize = Encoding.UTF8.GetBytes(batchEntry.MessageBody).LongCount(); var attributesSize = GetAttributesSize(batchEntry.MessageAttributes); return(contentSize + attributesSize > clientConfiguration.MessageSizeThreshold); }
protected override void Append(LoggingEvent loggingEvent) { LogLog.Debug(_declaringType, "Appending"); if (!EventRateLimiter.Request(loggingEvent.TimeStamp)) { LogLog.Debug(_declaringType, "Appending denied due to event limiter saturated."); return; } var sqsDatum = _eventProcessor.ProcessEvent(loggingEvent, RenderLoggingEvent(loggingEvent)).Single(); var sendMessageBatchRequestEntry = new SendMessageBatchRequestEntry { MessageBody = sqsDatum.Message, Id = sqsDatum.ID, }; if (sqsDatum.DelaySeconds.HasValue) { sendMessageBatchRequestEntry.DelaySeconds = sqsDatum.DelaySeconds.Value; } _client.AddSendMessageRequest(new SendMessageBatchRequestWrapper { QueueName = sqsDatum.QueueName, Entries = new[] { sendMessageBatchRequestEntry }.ToList() } ); }
private static SendMessageBatchRequestEntry ToSQSMessage(QueueMessage message) { var r = new SendMessageBatchRequestEntry(Guid.NewGuid().ToString(), message.StringContent); //todo: message attributes return(r); }
public override async Task DispatchAsync(Message <T> message, CancellationToken cancellationToken) { if (message.Value is null) { Throw.Exception("Message value cannot be null"); } var json = _serializer.AsString(message.Value); if (string.IsNullOrWhiteSpace(json)) { Throw.Exception("Message could not be serialized"); } var options = _options.CurrentValue; if (options is null) { Throw.Exception("Options cannot be null"); } LogDispatch(message); var request = new SendMessageBatchRequestEntry { Id = Guid.NewGuid().ToString("N"), MessageAttributes = GetMessageProperties(message), MessageBody = json }; var delay = DelaySeconds(message); if (delay.HasValue) { request.DelaySeconds = delay.Value; } var msg = new SendSqsMessageCommand { Message = request, QueueUrl = options.QueueUrl, ServiceUrl = options.ServiceURL, RegionEndpoint = options.RegionEndpoint }; await _messageWriter.WriteAsync(msg, cancellationToken); var taskCancellation = cancellationToken.Register(() => msg.Cancel(cancellationToken)); try { await msg.WaitForCompletion(); } finally { taskCancellation.Dispose(); } }
/// <summary> /// добавить сообщение в запрос на отправку /// </summary> /// <param name="entry"></param> /// <returns></returns> public SendMessageBatchRequest SetSendMessageBatchRequestEntry(SendMessageBatchRequestEntry entry) { if (entry == null) { throw new ArgumentNullException(nameof(entry), "Entry cannot was null"); } SendMessageBatchRequestEntry.Add(entry); return(this); }
async Task SendOutgoingMessages(ConcurrentQueue <OutgoingMessage> outgoingMessages, ITransactionContext context) { if (!outgoingMessages.Any()) { return; } var client = GetClientFromTransactionContext(context); var messagesByDestination = outgoingMessages .GroupBy(m => m.DestinationAddress) .ToList(); await Task.WhenAll( messagesByDestination .Select(async batch => { var entries = batch .Select(message => { var transportMessage = message.TransportMessage; var headers = transportMessage.Headers; var messageId = headers[Headers.MessageId]; var sqsMessage = new AmazonSQSTransportMessage(transportMessage.Headers, GetBody(transportMessage.Body)); var entry = new SendMessageBatchRequestEntry(messageId, _serializer.Serialize(sqsMessage)); var delaySeconds = GetDelaySeconds(headers); if (delaySeconds != null) { entry.DelaySeconds = delaySeconds.Value; } return(entry); }) .ToList(); var destinationUrl = GetDestinationQueueUrlByName(batch.Key, context); foreach (var batchToSend in entries.Batch(10)) { var request = new SendMessageBatchRequest(destinationUrl, batchToSend); var response = await client.SendMessageBatchAsync(request); if (response.Failed.Any()) { var failed = response.Failed .Select(f => new AmazonSQSException($"Failed {f.Message} with Id={f.Id}, Code={f.Code}, SenderFault={f.SenderFault}")); throw new AggregateException(failed); } } }) ); }
/// <summary> /// PutOnQueue: Sends in batch all jobs, returns list of all job.id confirmed /// </summary> /// <param name="jobs"></param> /// <returns></returns> public List <Guid> PutOnQueue(SubmitJobMessage[] jobs) { int i = 0; var entries = new List <SendMessageBatchRequestEntry>(); var d = new Dictionary <String, SubmitJobMessage>(); for (i = 0; i < jobs.Count(); i++) { var job = jobs[i]; var entry = new SendMessageBatchRequestEntry() .WithId(Guid.NewGuid().ToString()) .WithMessageBody(JsonConvert.SerializeObject(job)); d.Add(entry.Id, job); } var l = new List <Guid>(); if (entries.Count == 0) { return(l); } var sendResponse = queue.SendMessageBatch(new SendMessageBatchRequest() .WithEntries(entries.ToArray <SendMessageBatchRequestEntry>()) .WithQueueUrl(submitQueueUrl)); if (sendResponse.IsSetSendMessageBatchResult() == false) { Debug.WriteLine("SendResponse missing result", GetType().Name + ".PutOnQueue"); throw new Exception("SendResponse missing result"); } if (sendResponse.SendMessageBatchResult.IsSetSendMessageBatchResultEntry()) { foreach (var e in sendResponse.SendMessageBatchResult.BatchResultErrorEntry) { var job = d[e.Id]; l.Add(job.Id); } } if (sendResponse.SendMessageBatchResult.IsSetBatchResultErrorEntry()) { Debug.WriteLine("SendResponse received error entry", GetType().Name + ".PutOnQueue"); foreach (var b in sendResponse.SendMessageBatchResult.BatchResultErrorEntry) { var job = d[b.Id]; string s = String.Format("BatchResultErrorEntry (Job ID {0}): {1} {2} {3} {4}", job.Id, b.Id, b.Code, b.SenderFault, b.Message); Debug.WriteLine(s, GetType().Name + ".PutOnQueue"); } } return(l); }
private static List <SendMessageBatchRequestEntry> generateMessages() { var messages = new List <SendMessageBatchRequestEntry>(); for (var idx = 1; idx <= 10; idx++) { var uuid = System.Guid.NewGuid().ToString(); //Console.WriteLine(uuid); var message = new SendMessageBatchRequestEntry(uuid, "Message"); messages.Add(message); } return(messages); }
public static SendMessageBatchRequestEntry ApplyFifo(this SendMessageBatchRequestEntry request, string url) { if (!url.EndsWith(".fifo", StringComparison.InvariantCulture)) { return(request); } var id = Guid.NewGuid().ToString("D"); request.MessageGroupId = id; request.MessageDeduplicationId = id; return(request); }
private static IEnumerable <SendMessageBatchRequestWrapper> Assemble(IEnumerable <SQSDatum> data) { if (data.Any(x => System.Text.UTF8Encoding.UTF8.GetByteCount(x.Message) > 256 * 1024)) { throw new MessageTooLargeException(); } var requests = new List <SendMessageBatchRequestWrapper>(); foreach (var grouping in data.GroupBy(r => r.QueueName)) { var skip = 0; while (grouping.Skip(skip).Any()) { var size = 0; var taken = grouping .Skip(skip) .TakeWhile(x => (size += System.Text.UTF8Encoding.UTF8.GetByteCount(x.Message)) < 256 * 1024) .Take(10); requests.Add(new SendMessageBatchRequestWrapper { QueueName = grouping.Key, Entries = taken .Select( sqsDatum => { var t = new SendMessageBatchRequestEntry { MessageBody = sqsDatum.Message, Id = sqsDatum.ID }; if (sqsDatum.DelaySeconds.HasValue) { t.DelaySeconds = sqsDatum.DelaySeconds.Value; } return(t); } ).ToList() }); skip += taken.Count(); } } return(requests); }
private SendMessageBatchRequest BuildSendMessageBatchRequest(IEnumerable <object> payloads) { var jsonPayloads = payloads.Select(x => x.ToJson()).ToList(); var sendMessageBatchRequest = new SendMessageBatchRequest { QueueUrl = _queueUrl, Entries = new List <SendMessageBatchRequestEntry>() }; foreach (var message in jsonPayloads) { var entry = new SendMessageBatchRequestEntry(Guid.NewGuid().ToString(), message); sendMessageBatchRequest.Entries.Add(entry); } return(sendMessageBatchRequest); }
public Task Send(string destinationAddress, TransportMessage message, ITransactionContext context) { if (destinationAddress == null) { throw new ArgumentNullException("destinationAddress"); } if (message == null) { throw new ArgumentNullException("message"); } if (context == null) { throw new ArgumentNullException("context"); } var outputQueue = context.GetOrAdd(OutgoingQueueContextKey, () => new InMemOutputQueue()); var contextActionsSet = context.GetOrAdd(OutgoingQueueContextActionIsSetKey, () => false); if (!contextActionsSet) { context.OnCommitted(async() => { var client = GetClientFromTransactionContext(context); var messageSendRequests = outputQueue.GetMessages(); var tasks = messageSendRequests.Select(r => client.SendMessageBatchAsync(new SendMessageBatchRequest(r.DestinationAddressUrl, r.Messages.ToList()))); var response = await Task.WhenAll(tasks); if (response.Any(r => r.Failed.Any())) { GenerateErrorsAndThrow(response); } }); context.OnAborted(outputQueue.Clear); context.Items[OutgoingQueueContextActionIsSetKey] = true; } var sendMessageRequest = new SendMessageBatchRequestEntry() { MessageAttributes = CreateAttributesFromHeaders(message.Headers), MessageBody = GetBody(message.Body), Id = message.Headers.GetValueOrNull(Headers.MessageId) ?? Guid.NewGuid().ToString(), }; outputQueue.AddMessage(GetDestinationQueueUrlByName(destinationAddress, context), sendMessageRequest); return(_emptyTask); }
protected override void Append(LoggingEvent loggingEvent) { LogLog.Debug(_declaringType, "Appending"); if (!EventRateLimiter.Request(loggingEvent.TimeStamp)) { LogLog.Debug(_declaringType, "Appending denied due to event limiter saturated."); return; } var sqsDatum = _eventProcessor.ProcessEvent(loggingEvent, RenderLoggingEvent(loggingEvent)).Single(); if (System.Text.UTF8Encoding.UTF8.GetByteCount(sqsDatum.Message) > 256 * 1024) { throw new MessageTooLargeException(sqsDatum.Message); } if (sqsDatum.QueueName != null && !_queueNameRegex.IsMatch(sqsDatum.QueueName)) { throw new MessageTooLargeException(sqsDatum.Message); } var sendMessageBatchRequestEntry = new SendMessageBatchRequestEntry { MessageBody = sqsDatum.Message, Id = sqsDatum.ID, }; if (sqsDatum.DelaySeconds.HasValue) { sendMessageBatchRequestEntry.DelaySeconds = sqsDatum.DelaySeconds.Value; } _client.AddSendMessageRequest(new SendMessageBatchRequestWrapper { QueueName = sqsDatum.QueueName ?? _fallbackQueueName, Entries = new[] { sendMessageBatchRequestEntry }.ToList() } ); }
public SendMessageResponseModel <T> Save(IEnumerable <T> messages) { List <SendMessageBatchRequestEntry> sendMessageBatchRequestEntries = new List <SendMessageBatchRequestEntry>(); int i = 0; foreach (T message in messages) { i++; SendMessageBatchRequestEntry sendMessageBatchRequestEntry = new SendMessageBatchRequestEntry(i.ToString(), JsonConvert.SerializeObject(message)); sendMessageBatchRequestEntries.Add(sendMessageBatchRequestEntry); } SendMessageBatchRequest sendMessageBatchRequest = new SendMessageBatchRequest(this.queueUrl, sendMessageBatchRequestEntries); SendMessageBatchResponse sendMessageBatchResponse = this.sqsClient.SendMessageBatch(sendMessageBatchRequest); SendMessageResponseModel <T> sendMessageBatchResponseModel = ConstructSendResponseModel <T>(sendMessageBatchRequestEntries, sendMessageBatchResponse); return(sendMessageBatchResponseModel); }
protected async Task Move(ReceiveContext context, Action <SendMessageBatchRequestEntry, IDictionary <string, MessageAttributeValue> > preSend) { if (!context.TryGetPayload(out ClientContext clientContext)) { throw new ArgumentException("The ReceiveContext must contain a ClientContext (from Amazon SQS)", nameof(context)); } await _topologyFilter.Send(clientContext, Pipe.Empty <ClientContext>()).ConfigureAwait(false); var message = new SendMessageBatchRequestEntry("", Encoding.UTF8.GetString(context.GetBody())); CopyReceivedMessageHeaders(context, message.MessageAttributes); preSend(message, message.MessageAttributes); var task = clientContext.SendMessage(_destination, message, context.CancellationToken); context.AddReceiveTask(task); }
private async Task <SendMessageBatchRequestEntry> StoreMessageInS3Async(SendMessageBatchRequestEntry batchEntry, CancellationToken cancellationToken = default(CancellationToken)) { CheckMessageAttributes(batchEntry.MessageAttributes); var s3Key = clientConfiguration.S3KeyPovider.GenerateName(); var messageContentStr = batchEntry.MessageBody; var messageContentSize = Encoding.UTF8.GetBytes(messageContentStr).LongCount(); var messageAttributeValue = new MessageAttributeValue { DataType = "Number", StringValue = messageContentSize.ToString() }; batchEntry.MessageAttributes.Add(SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME, messageAttributeValue); var s3Pointer = new MessageS3Pointer(clientConfiguration.S3BucketName, s3Key); await StoreTextInS3Async(s3Key, messageContentStr, cancellationToken).ConfigureAwait(false); batchEntry.MessageBody = GetJsonFromS3Pointer(s3Pointer); return(batchEntry); }
private SendMessageBatchRequestEntry StoreMessageInS3(SendMessageBatchRequestEntry batchEntry) { CheckMessageAttributes(batchEntry.MessageAttributes); var s3Key = clientConfiguration.Is3KeyProvider.GenerateName(); var messageContentStr = batchEntry.MessageBody; var messageContentSize = Encoding.UTF8.GetBytes(messageContentStr).LongCount(); var messageAttributeValue = new MessageAttributeValue { DataType = "Number", StringValue = messageContentSize.ToString() }; batchEntry.MessageAttributes.Add(SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME, messageAttributeValue); var s3Pointer = new MessageS3Pointer(clientConfiguration.S3BucketName, s3Key); StoreTextInS3(s3Key, messageContentStr); batchEntry.MessageBody = GetJsonFromS3Pointer(s3Pointer); return(batchEntry); }
private SendMessageBatchRequest CreateBatchRequest(Dictionary <Guid, T> map) { var entries = new List <SendMessageBatchRequestEntry>(); foreach (var item in map) { var serialized = SerializeEvent(item.Value); if (serialized == null) { continue; } var entry = new SendMessageBatchRequestEntry { Id = item.Key.ToString(), MessageBody = serialized }; entries.Add(entry); } var request = new SendMessageBatchRequest { QueueUrl = QueueUrl, Entries = entries }; return(request); }
public static SendMessageBatchRequestEntry ToSQSMessage(QueueMessage message) { if (message == null) { throw new ArgumentNullException(nameof(message)); } var r = new SendMessageBatchRequestEntry(Guid.NewGuid().ToString(), message.StringContent); if (message.Properties != null) { foreach (KeyValuePair <string, string> prop in message.Properties) { r.MessageAttributes[prop.Key] = new MessageAttributeValue { DataType = "String", StringValue = prop.Value }; } } return(r); }
protected async Task Move(ReceiveContext context, Action <SendMessageBatchRequestEntry, IDictionary <string, MessageAttributeValue> > preSend) { if (!context.TryGetPayload(out ClientContext clientContext)) { throw new ArgumentException("The ReceiveContext must contain a ClientContext (from Amazon SQS)", nameof(context)); } await _topologyFilter.Send(clientContext, Pipe.Empty <ClientContext>()).ConfigureAwait(false); var message = new SendMessageBatchRequestEntry("", Encoding.UTF8.GetString(context.GetBody())); if (context.TryGetPayload(out AmazonSqsMessageContext receiveContext)) { if (_isFifo) { if (receiveContext.TransportMessage.Attributes.TryGetValue(MessageSystemAttributeName.MessageGroupId, out var messageGroupId) && !string.IsNullOrWhiteSpace(messageGroupId)) { message.MessageGroupId = messageGroupId; } if (receiveContext.TransportMessage.Attributes.TryGetValue(MessageSystemAttributeName.MessageDeduplicationId, out var messageDeduplicationId) && !string.IsNullOrWhiteSpace(messageDeduplicationId)) { message.MessageDeduplicationId = messageDeduplicationId; } } CopyReceivedMessageHeaders(receiveContext, message.MessageAttributes); } preSend(message, message.MessageAttributes); var task = clientContext.SendMessage(_destination, message, context.CancellationToken); context.AddReceiveTask(task); }
private SendMessageBatchRequestEntry StoreMessage(SendMessageBatchRequestEntry sendMessageRequest) { if (_storageClient == null) { return(sendMessageRequest); } CheckMessageAttributes(sendMessageRequest.MessageAttributes); var messageContentStr = sendMessageRequest.MessageBody; var messageContentSize = Encoding.UTF8.GetBytes(messageContentStr).Length; // LongLength; var messageAttributeValue = new MessageAttributeValue { DataType = "Number", StringValue = messageContentSize.ToString() }; sendMessageRequest.MessageAttributes.Add(SQSConstants.RESERVED_ATTRIBUTE_NAME, messageAttributeValue); sendMessageRequest.MessageBody = JsonConvert.SerializeObject(_storageClient.PutData(sendMessageRequest.MessageBody)); return(sendMessageRequest); }
/// <summary> /// /// </summary> /// <param name="messages"></param> /// <param name="token"></param> /// <returns></returns> public async Task SendBatchAsync(string[] messages, CancellationToken token) { Console.WriteLine(string.Format("Sending {0} messages: ", messages.Length)); using (var client = new Amazon.SQS.AmazonSQSClient(accessKey, secretKey, region)) { var queueUrlResult = await client.GetQueueUrlAsync(queueName, token); var queueUrl = queueUrlResult.QueueUrl; var entries = new List <SendMessageBatchRequestEntry>(); foreach (var message in messages) { var req = new SendMessageBatchRequestEntry(Guid.NewGuid().ToString(), message); if (isFIFO) { // required req.MessageGroupId = "group"; // fifo atua sobre o grupo req.MessageDeduplicationId = Guid.NewGuid().ToString(); } entries.Add(req); } var batch = new SendMessageBatchRequest(queueUrl, entries); var r = await client.SendMessageBatchAsync(batch); if (r.Failed != null && r.Failed.Count > 0) { foreach (var f in r.Failed) { Console.WriteLine("Failed : ", f.Message); } } } }
async Task ClientContext.SendMessage(string queueName, SendMessageBatchRequestEntry request, CancellationToken cancellationToken) { var queueInfo = await _queueCache.GetByName(queueName).ConfigureAwait(false); await queueInfo.Send(request, cancellationToken).ConfigureAwait(false); }
public async Task HandlingManyMessages(int throttleMessageCount) { // Arrange var fixture = new JustSayingFixture(OutputHelper); var client = fixture.CreateSqsClient(); var queue = new SqsQueueByName(fixture.Region, fixture.UniqueName, client, 1, fixture.LoggerFactory); if (!await queue.ExistsAsync()) { await queue.CreateAsync(new SqsBasicConfiguration()); if (!fixture.IsSimulator) { // Wait for up to 60 secs for queue creation to be guaranteed completed by AWS using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1))) { while (!cts.IsCancellationRequested) { await Task.Delay(TimeSpan.FromSeconds(2)); if (await queue.ExistsAsync()) { break; } } } } } Assert.True(await queue.ExistsAsync(), "The queue was not created."); OutputHelper.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue."); var entriesAdded = 0; // Add some messages do { var entries = new List <SendMessageBatchRequestEntry>(); for (var j = 0; j < 10; j++) { var batchEntry = new SendMessageBatchRequestEntry { MessageBody = $"{{\"Subject\":\"GenericMessage\", \"Message\": \"{entriesAdded}\"}}", Id = Guid.NewGuid().ToString() }; entries.Add(batchEntry); entriesAdded++; } await client.SendMessageBatchAsync(queue.Url, entries); }while (entriesAdded < throttleMessageCount); OutputHelper.WriteLine($"{DateTime.Now} - Done adding messages."); var handleCount = 0; var serialisations = Substitute.For <IMessageSerialisationRegister>(); var monitor = Substitute.For <IMessageMonitor>(); var handler = Substitute.For <IHandlerAsync <SimpleMessage> >(); handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(_ => Interlocked.Increment(ref handleCount)); serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new SimpleMessage()); var listener = new SqsNotificationListener(queue, serialisations, monitor, fixture.LoggerFactory); listener.AddMessageHandler(() => handler); // Act var stopwatch = Stopwatch.StartNew(); listener.Listen(); using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5))) { do { if (!fixture.IsSimulator) { await Task.Delay(TimeSpan.FromSeconds(5)); } OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount} messages. Waiting for completion."); }while (handleCount < throttleMessageCount && !cts.IsCancellationRequested); } listener.StopListening(); stopwatch.Stop(); OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount:N0} messages."); OutputHelper.WriteLine($"{DateTime.Now} - Took {stopwatch.ElapsedMilliseconds} ms"); OutputHelper.WriteLine($"{DateTime.Now} - Throughput {(float)handleCount / stopwatch.ElapsedMilliseconds * 1000} messages/second"); // Assert Assert.Equal(throttleMessageCount, handleCount); }
public async Task Messages_Are_Throttled_But_Still_Delivered(int throttleMessageCount) { // Arrange ILoggerFactory loggerFactory = OutputHelper.ToLoggerFactory(); IAwsClientFactory clientFactory = CreateClientFactory(); int retryCountBeforeSendingToErrorQueue = 1; var client = clientFactory.GetSqsClient(Region); var queue = new SqsQueueByName( Region, UniqueName, client, retryCountBeforeSendingToErrorQueue, loggerFactory); if (!await queue.ExistsAsync(CancellationToken.None)) { await queue.CreateAsync(new SqsBasicConfiguration()); if (!IsSimulator) { // Wait for up to 60 secs for queue creation to be guaranteed completed by AWS using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1)); while (!cts.IsCancellationRequested) { await Task.Delay(TimeSpan.FromSeconds(2)); if (await queue.ExistsAsync(CancellationToken.None)) { break; } } } } Assert.True(await queue.ExistsAsync(CancellationToken.None), "The queue was not created."); OutputHelper.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue."); var entriesAdded = 0; // Add some messages do { var entries = new List <SendMessageBatchRequestEntry>(); for (int j = 0; j < 10; j++) { var batchEntry = new SendMessageBatchRequestEntry { MessageBody = $"{{\"Subject\":\"SimpleMessage\", \"Message\": {{ \"Content\": \"{entriesAdded}\"}}}}", Id = Guid.NewGuid().ToString() }; entries.Add(batchEntry); entriesAdded++; } await client.SendMessageBatchAsync(queue.Uri.AbsoluteUri, entries); }while (entriesAdded < throttleMessageCount); OutputHelper.WriteLine($"{DateTime.Now} - Done adding messages."); int count = 0; var handler = Substitute.For <IHandlerAsync <SimpleMessage> >(); handler.Handle(Arg.Any <SimpleMessage>()) .ReturnsForAnyArgs(true) .AndDoes((_) => Interlocked.Increment(ref count)); IServiceCollection services = GivenJustSaying(LogLevel.Warning) .ConfigureJustSaying((builder) => builder.WithLoopbackQueue <SimpleMessage>(UniqueName)) .AddSingleton(handler); TimeSpan timeToProcess = TimeSpan.Zero; // Act await WhenAsync( services, async (publisher, listener, cancellationToken) => { var stopwatch = Stopwatch.StartNew(); var delay = IsSimulator ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromSeconds(5); await listener.StartAsync(cancellationToken); do { await Task.Delay(delay, cancellationToken); OutputHelper.WriteLine($"{DateTime.Now} - Handled {count} messages. Waiting for completion."); }while (count < throttleMessageCount && !cancellationToken.IsCancellationRequested); stopwatch.Stop(); timeToProcess = stopwatch.Elapsed; }); // Assert OutputHelper.WriteLine($"{DateTime.Now} - Handled {count:N0} messages."); OutputHelper.WriteLine($"{DateTime.Now} - Took {timeToProcess.TotalMilliseconds} ms"); OutputHelper.WriteLine($"{DateTime.Now} - Throughput {(float)count / timeToProcess.TotalMilliseconds * 1000} messages/second"); Assert.Equal(throttleMessageCount, count); }
public async Task Send(ClientContext context) { LogContext.SetCurrentIfNull(_context.LogContext); await _context.ConfigureTopologyPipe.Send(context).ConfigureAwait(false); var sendContext = new TransportAmazonSqsSendContext <T>(_message, _cancellationToken); await _pipe.Send(sendContext).ConfigureAwait(false); StartedActivity?activity = LogContext.IfEnabled(OperationName.Transport.Send)?.StartSendActivity(sendContext); try { if (_context.SendObservers.Count > 0) { await _context.SendObservers.PreSend(sendContext).ConfigureAwait(false); } var message = new SendMessageBatchRequestEntry("", Encoding.UTF8.GetString(sendContext.Body)); _context.SqsSetHeaderAdapter.Set(message.MessageAttributes, sendContext.Headers); _context.SqsSetHeaderAdapter.Set(message.MessageAttributes, "Content-Type", sendContext.ContentType.MediaType); _context.SqsSetHeaderAdapter.Set(message.MessageAttributes, nameof(sendContext.CorrelationId), sendContext.CorrelationId); if (!string.IsNullOrEmpty(sendContext.DeduplicationId)) { message.MessageDeduplicationId = sendContext.DeduplicationId; } if (!string.IsNullOrEmpty(sendContext.GroupId)) { message.MessageGroupId = sendContext.GroupId; } if (sendContext.DelaySeconds.HasValue) { message.DelaySeconds = sendContext.DelaySeconds.Value; } await context.SendMessage(_context.EntityName, message, sendContext.CancellationToken).ConfigureAwait(false); sendContext.LogSent(); activity.AddSendContextHeadersPostSend(sendContext); if (_context.SendObservers.Count > 0) { await _context.SendObservers.PostSend(sendContext).ConfigureAwait(false); } } catch (Exception ex) { sendContext.LogFaulted(ex); if (_context.SendObservers.Count > 0) { await _context.SendObservers.SendFault(sendContext, ex).ConfigureAwait(false); } throw; } finally { activity?.Stop(); } }
Task ClientContext.SendMessage(string queueName, SendMessageBatchRequestEntry request, CancellationToken cancellationToken) { return(_context.SendMessage(queueName, request, cancellationToken)); }