private List <SendMessageBatchRequestEntry> CreateMessageBatchRequestEntries <T>(IEnumerable <T> entries)
{
List <SendMessageBatchRequestEntry> sendMesssageReqs = new List <SendMessageBatchRequestEntry>();
int id = 0;
foreach (var payload in entries)
{
SendMessageBatchRequestEntry batchReqEntry = new SendMessageBatchRequestEntry(id.ToString(), JsonConvert.SerializeObject(payload));
//batchReqEntry.DelaySeconds = DelaySeconds;
if (string.IsNullOrEmpty(batchReqEntry.MessageBody))
{
continue;
}
// TODO: this can be redone to use attributes passed directly to this method.
AppendCustomAttributes(MessageAttributes, batchReqEntry.MessageAttributes);
AppendMessageTypeAttribute((typeof(T)).FullName, batchReqEntry.MessageAttributes);
sendMesssageReqs.Add(batchReqEntry);
id++; //increment message id;
}
return(sendMesssageReqs);
}
SendMessageBatchRequestEntry GetBatchRequestEntry(OutgoingMessage message)
{
var transportMessage = message.TransportMessage;
var headers = transportMessage.Headers;
var messageId = headers[Headers.MessageId];
var sqsMessage = new AmazonSQSTransportMessage(transportMessage.Headers, GetBody(transportMessage.Body));
var entry = new SendMessageBatchRequestEntry(messageId, _serializer.Serialize(sqsMessage));
var delaySeconds = GetDelaySeconds(headers);
if (delaySeconds != null)
{
entry.DelaySeconds = delaySeconds.Value;
}
if (message.DestinationAddress.EndsWith(".fifo"))
{
if (headers.ContainsKey(MessageGroupIdHeader))
{
entry.MessageGroupId = headers[MessageGroupIdHeader];
}
if (headers.ContainsKey(MessageDeduplicationIdHeader))
{
entry.MessageDeduplicationId = headers[MessageDeduplicationIdHeader];
}
}
return(entry);
}
private async Task <SendMessageBatchRequestEntry> StoreMessageAsync(SendMessageBatchRequestEntry sendMessageRequest, CancellationToken cancellationToken = default(CancellationToken))
{
if (_storageClient == null)
{
return(sendMessageRequest);
}
CheckMessageAttributes(sendMessageRequest.MessageAttributes);
var messageContentStr = sendMessageRequest.MessageBody;
var messageContentSize = Encoding.UTF8.GetBytes(messageContentStr).Length; // LongLength;
var messageAttributeValue = new MessageAttributeValue
{
DataType = "Number",
StringValue = messageContentSize.ToString()
};
// !!!
sendMessageRequest.MessageAttributes.Add(SQSConstants.RESERVED_ATTRIBUTE_NAME, messageAttributeValue);
var storeReference = await _storageClient.PutDataAsync(sendMessageRequest.MessageBody, null, cancellationToken).ConfigureAwait(false);
sendMessageRequest.MessageBody = JsonConvert.SerializeObject(storeReference);
return(sendMessageRequest);
}
private bool IsLarge(SendMessageBatchRequestEntry batchEntry)
{
var contentSize = Encoding.UTF8.GetBytes(batchEntry.MessageBody).LongCount();
var attributesSize = GetAttributesSize(batchEntry.MessageAttributes);
return(contentSize + attributesSize > clientConfiguration.MessageSizeThreshold);
}
protected override void Append(LoggingEvent loggingEvent)
{
LogLog.Debug(_declaringType, "Appending");
if (!EventRateLimiter.Request(loggingEvent.TimeStamp))
{
LogLog.Debug(_declaringType, "Appending denied due to event limiter saturated.");
return;
}
var sqsDatum = _eventProcessor.ProcessEvent(loggingEvent, RenderLoggingEvent(loggingEvent)).Single();
var sendMessageBatchRequestEntry = new SendMessageBatchRequestEntry
{
MessageBody = sqsDatum.Message,
Id = sqsDatum.ID,
};
if (sqsDatum.DelaySeconds.HasValue)
{
sendMessageBatchRequestEntry.DelaySeconds = sqsDatum.DelaySeconds.Value;
}
_client.AddSendMessageRequest(new SendMessageBatchRequestWrapper
{
QueueName = sqsDatum.QueueName,
Entries = new[]
{
sendMessageBatchRequestEntry
}.ToList()
}
);
}
private static SendMessageBatchRequestEntry ToSQSMessage(QueueMessage message)
{
var r = new SendMessageBatchRequestEntry(Guid.NewGuid().ToString(), message.StringContent);
//todo: message attributes
return(r);
}
public override async Task DispatchAsync(Message <T> message, CancellationToken cancellationToken)
{
if (message.Value is null)
{
Throw.Exception("Message value cannot be null");
}
var json = _serializer.AsString(message.Value);
if (string.IsNullOrWhiteSpace(json))
{
Throw.Exception("Message could not be serialized");
}
var options = _options.CurrentValue;
if (options is null)
{
Throw.Exception("Options cannot be null");
}
LogDispatch(message);
var request = new SendMessageBatchRequestEntry
{
Id = Guid.NewGuid().ToString("N"),
MessageAttributes = GetMessageProperties(message),
MessageBody = json
};
var delay = DelaySeconds(message);
if (delay.HasValue)
{
request.DelaySeconds = delay.Value;
}
var msg = new SendSqsMessageCommand
{
Message = request,
QueueUrl = options.QueueUrl,
ServiceUrl = options.ServiceURL,
RegionEndpoint = options.RegionEndpoint
};
await _messageWriter.WriteAsync(msg, cancellationToken);
var taskCancellation = cancellationToken.Register(() => msg.Cancel(cancellationToken));
try
{
await msg.WaitForCompletion();
}
finally
{
taskCancellation.Dispose();
}
}
/// <summary>
/// добавить сообщение в запрос на отправку
/// </summary>
/// <param name="entry"></param>
/// <returns></returns>
public SendMessageBatchRequest SetSendMessageBatchRequestEntry(SendMessageBatchRequestEntry entry)
{
if (entry == null)
{
throw new ArgumentNullException(nameof(entry), "Entry cannot was null");
}
SendMessageBatchRequestEntry.Add(entry);
return(this);
}
async Task SendOutgoingMessages(ConcurrentQueue <OutgoingMessage> outgoingMessages, ITransactionContext context)
{
if (!outgoingMessages.Any())
{
return;
}
var client = GetClientFromTransactionContext(context);
var messagesByDestination = outgoingMessages
.GroupBy(m => m.DestinationAddress)
.ToList();
await Task.WhenAll(
messagesByDestination
.Select(async batch =>
{
var entries = batch
.Select(message =>
{
var transportMessage = message.TransportMessage;
var headers = transportMessage.Headers;
var messageId = headers[Headers.MessageId];
var sqsMessage = new AmazonSQSTransportMessage(transportMessage.Headers, GetBody(transportMessage.Body));
var entry = new SendMessageBatchRequestEntry(messageId, _serializer.Serialize(sqsMessage));
var delaySeconds = GetDelaySeconds(headers);
if (delaySeconds != null)
{
entry.DelaySeconds = delaySeconds.Value;
}
return(entry);
})
.ToList();
var destinationUrl = GetDestinationQueueUrlByName(batch.Key, context);
foreach (var batchToSend in entries.Batch(10))
{
var request = new SendMessageBatchRequest(destinationUrl, batchToSend);
var response = await client.SendMessageBatchAsync(request);
if (response.Failed.Any())
{
var failed = response.Failed
.Select(f => new AmazonSQSException($"Failed {f.Message} with Id={f.Id}, Code={f.Code}, SenderFault={f.SenderFault}"));
throw new AggregateException(failed);
}
}
})
);
}
/// <summary>
/// PutOnQueue: Sends in batch all jobs, returns list of all job.id confirmed
/// </summary>
/// <param name="jobs"></param>
/// <returns></returns>
public List <Guid> PutOnQueue(SubmitJobMessage[] jobs)
{
int i = 0;
var entries = new List <SendMessageBatchRequestEntry>();
var d = new Dictionary <String, SubmitJobMessage>();
for (i = 0; i < jobs.Count(); i++)
{
var job = jobs[i];
var entry = new SendMessageBatchRequestEntry()
.WithId(Guid.NewGuid().ToString())
.WithMessageBody(JsonConvert.SerializeObject(job));
d.Add(entry.Id, job);
}
var l = new List <Guid>();
if (entries.Count == 0)
{
return(l);
}
var sendResponse = queue.SendMessageBatch(new SendMessageBatchRequest()
.WithEntries(entries.ToArray <SendMessageBatchRequestEntry>())
.WithQueueUrl(submitQueueUrl));
if (sendResponse.IsSetSendMessageBatchResult() == false)
{
Debug.WriteLine("SendResponse missing result", GetType().Name + ".PutOnQueue");
throw new Exception("SendResponse missing result");
}
if (sendResponse.SendMessageBatchResult.IsSetSendMessageBatchResultEntry())
{
foreach (var e in sendResponse.SendMessageBatchResult.BatchResultErrorEntry)
{
var job = d[e.Id];
l.Add(job.Id);
}
}
if (sendResponse.SendMessageBatchResult.IsSetBatchResultErrorEntry())
{
Debug.WriteLine("SendResponse received error entry", GetType().Name + ".PutOnQueue");
foreach (var b in sendResponse.SendMessageBatchResult.BatchResultErrorEntry)
{
var job = d[b.Id];
string s = String.Format("BatchResultErrorEntry (Job ID {0}): {1} {2} {3} {4}",
job.Id, b.Id, b.Code, b.SenderFault, b.Message);
Debug.WriteLine(s, GetType().Name + ".PutOnQueue");
}
}
return(l);
}
private static List <SendMessageBatchRequestEntry> generateMessages()
{
var messages = new List <SendMessageBatchRequestEntry>();
for (var idx = 1; idx <= 10; idx++)
{
var uuid = System.Guid.NewGuid().ToString();
//Console.WriteLine(uuid);
var message = new SendMessageBatchRequestEntry(uuid, "Message");
messages.Add(message);
}
return(messages);
}
public static SendMessageBatchRequestEntry ApplyFifo(this SendMessageBatchRequestEntry request, string url)
{
if (!url.EndsWith(".fifo", StringComparison.InvariantCulture))
{
return(request);
}
var id = Guid.NewGuid().ToString("D");
request.MessageGroupId = id;
request.MessageDeduplicationId = id;
return(request);
}
private static IEnumerable <SendMessageBatchRequestWrapper> Assemble(IEnumerable <SQSDatum> data)
{
if (data.Any(x => System.Text.UTF8Encoding.UTF8.GetByteCount(x.Message) > 256 * 1024))
{
throw new MessageTooLargeException();
}
var requests = new List <SendMessageBatchRequestWrapper>();
foreach (var grouping in data.GroupBy(r => r.QueueName))
{
var skip = 0;
while (grouping.Skip(skip).Any())
{
var size = 0;
var taken = grouping
.Skip(skip)
.TakeWhile(x => (size += System.Text.UTF8Encoding.UTF8.GetByteCount(x.Message)) < 256 * 1024)
.Take(10);
requests.Add(new SendMessageBatchRequestWrapper
{
QueueName = grouping.Key,
Entries =
taken
.Select(
sqsDatum =>
{
var t = new SendMessageBatchRequestEntry
{
MessageBody = sqsDatum.Message,
Id = sqsDatum.ID
};
if (sqsDatum.DelaySeconds.HasValue)
{
t.DelaySeconds = sqsDatum.DelaySeconds.Value;
}
return(t);
}
).ToList()
});
skip += taken.Count();
}
}
return(requests);
}
private SendMessageBatchRequest BuildSendMessageBatchRequest(IEnumerable <object> payloads)
{
var jsonPayloads = payloads.Select(x => x.ToJson()).ToList();
var sendMessageBatchRequest = new SendMessageBatchRequest {
QueueUrl = _queueUrl, Entries = new List <SendMessageBatchRequestEntry>()
};
foreach (var message in jsonPayloads)
{
var entry = new SendMessageBatchRequestEntry(Guid.NewGuid().ToString(), message);
sendMessageBatchRequest.Entries.Add(entry);
}
return(sendMessageBatchRequest);
}
public Task Send(string destinationAddress, TransportMessage message, ITransactionContext context)
{
if (destinationAddress == null)
{
throw new ArgumentNullException("destinationAddress");
}
if (message == null)
{
throw new ArgumentNullException("message");
}
if (context == null)
{
throw new ArgumentNullException("context");
}
var outputQueue = context.GetOrAdd(OutgoingQueueContextKey, () => new InMemOutputQueue());
var contextActionsSet = context.GetOrAdd(OutgoingQueueContextActionIsSetKey, () => false);
if (!contextActionsSet)
{
context.OnCommitted(async() =>
{
var client = GetClientFromTransactionContext(context);
var messageSendRequests = outputQueue.GetMessages();
var tasks = messageSendRequests.Select(r => client.SendMessageBatchAsync(new SendMessageBatchRequest(r.DestinationAddressUrl, r.Messages.ToList())));
var response = await Task.WhenAll(tasks);
if (response.Any(r => r.Failed.Any()))
{
GenerateErrorsAndThrow(response);
}
});
context.OnAborted(outputQueue.Clear);
context.Items[OutgoingQueueContextActionIsSetKey] = true;
}
var sendMessageRequest = new SendMessageBatchRequestEntry()
{
MessageAttributes = CreateAttributesFromHeaders(message.Headers),
MessageBody = GetBody(message.Body),
Id = message.Headers.GetValueOrNull(Headers.MessageId) ?? Guid.NewGuid().ToString(),
};
outputQueue.AddMessage(GetDestinationQueueUrlByName(destinationAddress, context), sendMessageRequest);
return(_emptyTask);
}
protected override void Append(LoggingEvent loggingEvent)
{
LogLog.Debug(_declaringType, "Appending");
if (!EventRateLimiter.Request(loggingEvent.TimeStamp))
{
LogLog.Debug(_declaringType, "Appending denied due to event limiter saturated.");
return;
}
var sqsDatum = _eventProcessor.ProcessEvent(loggingEvent, RenderLoggingEvent(loggingEvent)).Single();
if (System.Text.UTF8Encoding.UTF8.GetByteCount(sqsDatum.Message) > 256 * 1024)
{
throw new MessageTooLargeException(sqsDatum.Message);
}
if (sqsDatum.QueueName != null && !_queueNameRegex.IsMatch(sqsDatum.QueueName))
{
throw new MessageTooLargeException(sqsDatum.Message);
}
var sendMessageBatchRequestEntry = new SendMessageBatchRequestEntry
{
MessageBody = sqsDatum.Message,
Id = sqsDatum.ID,
};
if (sqsDatum.DelaySeconds.HasValue)
{
sendMessageBatchRequestEntry.DelaySeconds = sqsDatum.DelaySeconds.Value;
}
_client.AddSendMessageRequest(new SendMessageBatchRequestWrapper
{
QueueName = sqsDatum.QueueName ?? _fallbackQueueName,
Entries = new[]
{
sendMessageBatchRequestEntry
}.ToList()
}
);
}
public SendMessageResponseModel <T> Save(IEnumerable <T> messages)
{
List <SendMessageBatchRequestEntry> sendMessageBatchRequestEntries = new List <SendMessageBatchRequestEntry>();
int i = 0;
foreach (T message in messages)
{
i++;
SendMessageBatchRequestEntry sendMessageBatchRequestEntry = new SendMessageBatchRequestEntry(i.ToString(), JsonConvert.SerializeObject(message));
sendMessageBatchRequestEntries.Add(sendMessageBatchRequestEntry);
}
SendMessageBatchRequest sendMessageBatchRequest = new SendMessageBatchRequest(this.queueUrl, sendMessageBatchRequestEntries);
SendMessageBatchResponse sendMessageBatchResponse = this.sqsClient.SendMessageBatch(sendMessageBatchRequest);
SendMessageResponseModel <T> sendMessageBatchResponseModel = ConstructSendResponseModel <T>(sendMessageBatchRequestEntries, sendMessageBatchResponse);
return(sendMessageBatchResponseModel);
}
protected async Task Move(ReceiveContext context, Action <SendMessageBatchRequestEntry, IDictionary <string, MessageAttributeValue> > preSend)
{
if (!context.TryGetPayload(out ClientContext clientContext))
{
throw new ArgumentException("The ReceiveContext must contain a ClientContext (from Amazon SQS)", nameof(context));
}
await _topologyFilter.Send(clientContext, Pipe.Empty <ClientContext>()).ConfigureAwait(false);
var message = new SendMessageBatchRequestEntry("", Encoding.UTF8.GetString(context.GetBody()));
CopyReceivedMessageHeaders(context, message.MessageAttributes);
preSend(message, message.MessageAttributes);
var task = clientContext.SendMessage(_destination, message, context.CancellationToken);
context.AddReceiveTask(task);
}
private async Task <SendMessageBatchRequestEntry> StoreMessageInS3Async(SendMessageBatchRequestEntry batchEntry, CancellationToken cancellationToken = default(CancellationToken))
{
CheckMessageAttributes(batchEntry.MessageAttributes);
var s3Key = clientConfiguration.S3KeyPovider.GenerateName();
var messageContentStr = batchEntry.MessageBody;
var messageContentSize = Encoding.UTF8.GetBytes(messageContentStr).LongCount();
var messageAttributeValue = new MessageAttributeValue {
DataType = "Number", StringValue = messageContentSize.ToString()
};
batchEntry.MessageAttributes.Add(SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME, messageAttributeValue);
var s3Pointer = new MessageS3Pointer(clientConfiguration.S3BucketName, s3Key);
await StoreTextInS3Async(s3Key, messageContentStr, cancellationToken).ConfigureAwait(false);
batchEntry.MessageBody = GetJsonFromS3Pointer(s3Pointer);
return(batchEntry);
}
private SendMessageBatchRequestEntry StoreMessageInS3(SendMessageBatchRequestEntry batchEntry)
{
CheckMessageAttributes(batchEntry.MessageAttributes);
var s3Key = clientConfiguration.Is3KeyProvider.GenerateName();
var messageContentStr = batchEntry.MessageBody;
var messageContentSize = Encoding.UTF8.GetBytes(messageContentStr).LongCount();
var messageAttributeValue = new MessageAttributeValue {
DataType = "Number", StringValue = messageContentSize.ToString()
};
batchEntry.MessageAttributes.Add(SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME, messageAttributeValue);
var s3Pointer = new MessageS3Pointer(clientConfiguration.S3BucketName, s3Key);
StoreTextInS3(s3Key, messageContentStr);
batchEntry.MessageBody = GetJsonFromS3Pointer(s3Pointer);
return(batchEntry);
}
private SendMessageBatchRequest CreateBatchRequest(Dictionary <Guid, T> map)
{
var entries = new List <SendMessageBatchRequestEntry>();
foreach (var item in map)
{
var serialized = SerializeEvent(item.Value);
if (serialized == null)
{
continue;
}
var entry = new SendMessageBatchRequestEntry {
Id = item.Key.ToString(), MessageBody = serialized
};
entries.Add(entry);
}
var request = new SendMessageBatchRequest {
QueueUrl = QueueUrl, Entries = entries
};
return(request);
}
public static SendMessageBatchRequestEntry ToSQSMessage(QueueMessage message)
{
if (message == null)
{
throw new ArgumentNullException(nameof(message));
}
var r = new SendMessageBatchRequestEntry(Guid.NewGuid().ToString(), message.StringContent);
if (message.Properties != null)
{
foreach (KeyValuePair <string, string> prop in message.Properties)
{
r.MessageAttributes[prop.Key] = new MessageAttributeValue
{
DataType = "String",
StringValue = prop.Value
};
}
}
return(r);
}
protected async Task Move(ReceiveContext context, Action <SendMessageBatchRequestEntry, IDictionary <string, MessageAttributeValue> > preSend)
{
if (!context.TryGetPayload(out ClientContext clientContext))
{
throw new ArgumentException("The ReceiveContext must contain a ClientContext (from Amazon SQS)", nameof(context));
}
await _topologyFilter.Send(clientContext, Pipe.Empty <ClientContext>()).ConfigureAwait(false);
var message = new SendMessageBatchRequestEntry("", Encoding.UTF8.GetString(context.GetBody()));
if (context.TryGetPayload(out AmazonSqsMessageContext receiveContext))
{
if (_isFifo)
{
if (receiveContext.TransportMessage.Attributes.TryGetValue(MessageSystemAttributeName.MessageGroupId, out var messageGroupId) &&
!string.IsNullOrWhiteSpace(messageGroupId))
{
message.MessageGroupId = messageGroupId;
}
if (receiveContext.TransportMessage.Attributes.TryGetValue(MessageSystemAttributeName.MessageDeduplicationId,
out var messageDeduplicationId) &&
!string.IsNullOrWhiteSpace(messageDeduplicationId))
{
message.MessageDeduplicationId = messageDeduplicationId;
}
}
CopyReceivedMessageHeaders(receiveContext, message.MessageAttributes);
}
preSend(message, message.MessageAttributes);
var task = clientContext.SendMessage(_destination, message, context.CancellationToken);
context.AddReceiveTask(task);
}
private SendMessageBatchRequestEntry StoreMessage(SendMessageBatchRequestEntry sendMessageRequest)
{
if (_storageClient == null)
{
return(sendMessageRequest);
}
CheckMessageAttributes(sendMessageRequest.MessageAttributes);
var messageContentStr = sendMessageRequest.MessageBody;
var messageContentSize = Encoding.UTF8.GetBytes(messageContentStr).Length; // LongLength;
var messageAttributeValue = new MessageAttributeValue
{
DataType = "Number",
StringValue = messageContentSize.ToString()
};
sendMessageRequest.MessageAttributes.Add(SQSConstants.RESERVED_ATTRIBUTE_NAME, messageAttributeValue);
sendMessageRequest.MessageBody = JsonConvert.SerializeObject(_storageClient.PutData(sendMessageRequest.MessageBody));
return(sendMessageRequest);
}
/// <summary>
///
/// </summary>
/// <param name="messages"></param>
/// <param name="token"></param>
/// <returns></returns>
public async Task SendBatchAsync(string[] messages, CancellationToken token)
{
Console.WriteLine(string.Format("Sending {0} messages: ", messages.Length));
using (var client = new Amazon.SQS.AmazonSQSClient(accessKey, secretKey, region))
{
var queueUrlResult = await client.GetQueueUrlAsync(queueName, token);
var queueUrl = queueUrlResult.QueueUrl;
var entries = new List <SendMessageBatchRequestEntry>();
foreach (var message in messages)
{
var req = new SendMessageBatchRequestEntry(Guid.NewGuid().ToString(), message);
if (isFIFO)
{
// required
req.MessageGroupId = "group"; // fifo atua sobre o grupo
req.MessageDeduplicationId = Guid.NewGuid().ToString();
}
entries.Add(req);
}
var batch = new SendMessageBatchRequest(queueUrl, entries);
var r = await client.SendMessageBatchAsync(batch);
if (r.Failed != null && r.Failed.Count > 0)
{
foreach (var f in r.Failed)
{
Console.WriteLine("Failed : ", f.Message);
}
}
}
}
async Task ClientContext.SendMessage(string queueName, SendMessageBatchRequestEntry request, CancellationToken cancellationToken)
{
var queueInfo = await _queueCache.GetByName(queueName).ConfigureAwait(false);
await queueInfo.Send(request, cancellationToken).ConfigureAwait(false);
}
public async Task HandlingManyMessages(int throttleMessageCount)
{
// Arrange
var fixture = new JustSayingFixture(OutputHelper);
var client = fixture.CreateSqsClient();
var queue = new SqsQueueByName(fixture.Region, fixture.UniqueName, client, 1, fixture.LoggerFactory);
if (!await queue.ExistsAsync())
{
await queue.CreateAsync(new SqsBasicConfiguration());
if (!fixture.IsSimulator)
{
// Wait for up to 60 secs for queue creation to be guaranteed completed by AWS
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1)))
{
while (!cts.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(2));
if (await queue.ExistsAsync())
{
break;
}
}
}
}
}
Assert.True(await queue.ExistsAsync(), "The queue was not created.");
OutputHelper.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue.");
var entriesAdded = 0;
// Add some messages
do
{
var entries = new List <SendMessageBatchRequestEntry>();
for (var j = 0; j < 10; j++)
{
var batchEntry = new SendMessageBatchRequestEntry
{
MessageBody = $"{{\"Subject\":\"GenericMessage\", \"Message\": \"{entriesAdded}\"}}",
Id = Guid.NewGuid().ToString()
};
entries.Add(batchEntry);
entriesAdded++;
}
await client.SendMessageBatchAsync(queue.Url, entries);
}while (entriesAdded < throttleMessageCount);
OutputHelper.WriteLine($"{DateTime.Now} - Done adding messages.");
var handleCount = 0;
var serialisations = Substitute.For <IMessageSerialisationRegister>();
var monitor = Substitute.For <IMessageMonitor>();
var handler = Substitute.For <IHandlerAsync <SimpleMessage> >();
handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(_ => Interlocked.Increment(ref handleCount));
serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new SimpleMessage());
var listener = new SqsNotificationListener(queue, serialisations, monitor, fixture.LoggerFactory);
listener.AddMessageHandler(() => handler);
// Act
var stopwatch = Stopwatch.StartNew();
listener.Listen();
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)))
{
do
{
if (!fixture.IsSimulator)
{
await Task.Delay(TimeSpan.FromSeconds(5));
}
OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount} messages. Waiting for completion.");
}while (handleCount < throttleMessageCount && !cts.IsCancellationRequested);
}
listener.StopListening();
stopwatch.Stop();
OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount:N0} messages.");
OutputHelper.WriteLine($"{DateTime.Now} - Took {stopwatch.ElapsedMilliseconds} ms");
OutputHelper.WriteLine($"{DateTime.Now} - Throughput {(float)handleCount / stopwatch.ElapsedMilliseconds * 1000} messages/second");
// Assert
Assert.Equal(throttleMessageCount, handleCount);
}
public async Task Messages_Are_Throttled_But_Still_Delivered(int throttleMessageCount)
{
// Arrange
ILoggerFactory loggerFactory = OutputHelper.ToLoggerFactory();
IAwsClientFactory clientFactory = CreateClientFactory();
int retryCountBeforeSendingToErrorQueue = 1;
var client = clientFactory.GetSqsClient(Region);
var queue = new SqsQueueByName(
Region,
UniqueName,
client,
retryCountBeforeSendingToErrorQueue,
loggerFactory);
if (!await queue.ExistsAsync(CancellationToken.None))
{
await queue.CreateAsync(new SqsBasicConfiguration());
if (!IsSimulator)
{
// Wait for up to 60 secs for queue creation to be guaranteed completed by AWS
using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1));
while (!cts.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(2));
if (await queue.ExistsAsync(CancellationToken.None))
{
break;
}
}
}
}
Assert.True(await queue.ExistsAsync(CancellationToken.None), "The queue was not created.");
OutputHelper.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue.");
var entriesAdded = 0;
// Add some messages
do
{
var entries = new List <SendMessageBatchRequestEntry>();
for (int j = 0; j < 10; j++)
{
var batchEntry = new SendMessageBatchRequestEntry
{
MessageBody = $"{{\"Subject\":\"SimpleMessage\", \"Message\": {{ \"Content\": \"{entriesAdded}\"}}}}",
Id = Guid.NewGuid().ToString()
};
entries.Add(batchEntry);
entriesAdded++;
}
await client.SendMessageBatchAsync(queue.Uri.AbsoluteUri, entries);
}while (entriesAdded < throttleMessageCount);
OutputHelper.WriteLine($"{DateTime.Now} - Done adding messages.");
int count = 0;
var handler = Substitute.For <IHandlerAsync <SimpleMessage> >();
handler.Handle(Arg.Any <SimpleMessage>())
.ReturnsForAnyArgs(true)
.AndDoes((_) => Interlocked.Increment(ref count));
IServiceCollection services = GivenJustSaying(LogLevel.Warning)
.ConfigureJustSaying((builder) => builder.WithLoopbackQueue <SimpleMessage>(UniqueName))
.AddSingleton(handler);
TimeSpan timeToProcess = TimeSpan.Zero;
// Act
await WhenAsync(
services,
async (publisher, listener, cancellationToken) =>
{
var stopwatch = Stopwatch.StartNew();
var delay = IsSimulator ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromSeconds(5);
await listener.StartAsync(cancellationToken);
do
{
await Task.Delay(delay, cancellationToken);
OutputHelper.WriteLine($"{DateTime.Now} - Handled {count} messages. Waiting for completion.");
}while (count < throttleMessageCount && !cancellationToken.IsCancellationRequested);
stopwatch.Stop();
timeToProcess = stopwatch.Elapsed;
});
// Assert
OutputHelper.WriteLine($"{DateTime.Now} - Handled {count:N0} messages.");
OutputHelper.WriteLine($"{DateTime.Now} - Took {timeToProcess.TotalMilliseconds} ms");
OutputHelper.WriteLine($"{DateTime.Now} - Throughput {(float)count / timeToProcess.TotalMilliseconds * 1000} messages/second");
Assert.Equal(throttleMessageCount, count);
}
public async Task Send(ClientContext context)
{
LogContext.SetCurrentIfNull(_context.LogContext);
await _context.ConfigureTopologyPipe.Send(context).ConfigureAwait(false);
var sendContext = new TransportAmazonSqsSendContext <T>(_message, _cancellationToken);
await _pipe.Send(sendContext).ConfigureAwait(false);
StartedActivity?activity = LogContext.IfEnabled(OperationName.Transport.Send)?.StartSendActivity(sendContext);
try
{
if (_context.SendObservers.Count > 0)
{
await _context.SendObservers.PreSend(sendContext).ConfigureAwait(false);
}
var message = new SendMessageBatchRequestEntry("", Encoding.UTF8.GetString(sendContext.Body));
_context.SqsSetHeaderAdapter.Set(message.MessageAttributes, sendContext.Headers);
_context.SqsSetHeaderAdapter.Set(message.MessageAttributes, "Content-Type", sendContext.ContentType.MediaType);
_context.SqsSetHeaderAdapter.Set(message.MessageAttributes, nameof(sendContext.CorrelationId), sendContext.CorrelationId);
if (!string.IsNullOrEmpty(sendContext.DeduplicationId))
{
message.MessageDeduplicationId = sendContext.DeduplicationId;
}
if (!string.IsNullOrEmpty(sendContext.GroupId))
{
message.MessageGroupId = sendContext.GroupId;
}
if (sendContext.DelaySeconds.HasValue)
{
message.DelaySeconds = sendContext.DelaySeconds.Value;
}
await context.SendMessage(_context.EntityName, message, sendContext.CancellationToken).ConfigureAwait(false);
sendContext.LogSent();
activity.AddSendContextHeadersPostSend(sendContext);
if (_context.SendObservers.Count > 0)
{
await _context.SendObservers.PostSend(sendContext).ConfigureAwait(false);
}
}
catch (Exception ex)
{
sendContext.LogFaulted(ex);
if (_context.SendObservers.Count > 0)
{
await _context.SendObservers.SendFault(sendContext, ex).ConfigureAwait(false);
}
throw;
}
finally
{
activity?.Stop();
}
}
Task ClientContext.SendMessage(string queueName, SendMessageBatchRequestEntry request, CancellationToken cancellationToken)
{
return(_context.SendMessage(queueName, request, cancellationToken));
}