public static void ValidateSendMessageBatch(SendMessageBatchRequest request, SendMessageBatchResponse response) { if (response != null && response.SendMessageBatchResult != null && response.SendMessageBatchResult.SendMessageBatchResultEntry != null && response.SendMessageBatchResult.SendMessageBatchResultEntry.Count > 0) { Dictionary <string, string> requestMessages = request.Entries.ToDictionary(entry => entry.Id, entry => entry.MessageBody, StringComparer.Ordinal); List <SendMessageBatchResultEntry> resultEntries = response.SendMessageBatchResult.SendMessageBatchResultEntry; foreach (SendMessageBatchResultEntry entry in resultEntries) { string body = requestMessages[entry.Id]; string md5 = entry.MD5OfMessageBody; string id = entry.MessageId; AmazonSQSUtil.ValidateMD5(body, id, md5); } } }
public async Task <bool> SendBatchMessagesAsync(List <object> items) { //only 10 messages can be sent in a batch for (int i = 0; i < items.Count; i += 10) { List <SendMessageBatchRequestEntry> requests = new List <SendMessageBatchRequestEntry>(10); List <object> ten = items.Skip(i).Take(10).ToList(); foreach (object item in ten) { requests.Add(new SendMessageBatchRequestEntry(queueUrl, JsonConvert.SerializeObject(item))); } SendMessageBatchRequest request = new SendMessageBatchRequest(); request.Entries = requests; var response = await sqsClient.SendMessageBatchAsync(request); } return(true); }
public async Task Short_Message_Async_It_Is_Stored_In_S3_If_Exceeds_Threshold() { var extendedClient = new AmazonSQSExtendedClient( sqsMock.Object, new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(s3Mock.Object, S3_BUCKET_NAME).WithMessageSizeThreshold(100)); var batchRequest = new SendMessageBatchRequest(SQS_QUEUE_NAME, new List <SendMessageBatchRequestEntry>()); var body = GenerateLongString(101); for (var i = 0; i < 3; i++) { batchRequest.Entries.Add(new SendMessageBatchRequestEntry(Guid.NewGuid().ToString("N"), body)); } await extendedClient.SendMessageBatchAsync(batchRequest); s3Mock.Verify(s => s.PutObjectAsync(It.IsAny <PutObjectRequest>(), It.IsAny <CancellationToken>()), Times.Exactly(3)); sqsMock.Verify(s => s.SendMessageBatchAsync(It.Is <SendMessageBatchRequest>(r => r.Entries.All(e => MessagePointerIsCorrect(e.MessageBody) && LargePayloadAttributeIsAdded(e.MessageAttributes))), default(CancellationToken))); }
public void Short_Message_It_Is_Stored_In_S3_If_AlwaysThroughS3_Configured() { var extendedClient = new AmazonSQSExtendedClient( sqsMock.Object, new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(s3Mock.Object, S3_BUCKET_NAME).WithAlwaysThroughS3(true)); var batchRequest = new SendMessageBatchRequest(SQS_QUEUE_NAME, new List <SendMessageBatchRequestEntry>()); var body = GenerateLongString(SQSExtendedClientConstants.DEFAULT_MESSAGE_SIZE_THRESHOLD); for (var i = 0; i < 3; i++) { batchRequest.Entries.Add(new SendMessageBatchRequestEntry(Guid.NewGuid().ToString("N"), body)); } extendedClient.SendMessageBatch(batchRequest); s3Mock.Verify(s => s.PutObject(It.IsAny <PutObjectRequest>()), Times.Exactly(3)); sqsMock.Verify( s => s.SendMessageBatch( It.Is <SendMessageBatchRequest>(r => r.Entries.All(e => MessagePointerIsCorrect(e.MessageBody) && LargePayloadAttributeIsAdded(e.MessageAttributes))))); }
public SendMessageResponseModel <T> Save(IEnumerable <T> messages) { List <SendMessageBatchRequestEntry> sendMessageBatchRequestEntries = new List <SendMessageBatchRequestEntry>(); int i = 0; foreach (T message in messages) { i++; SendMessageBatchRequestEntry sendMessageBatchRequestEntry = new SendMessageBatchRequestEntry(i.ToString(), JsonConvert.SerializeObject(message)); sendMessageBatchRequestEntries.Add(sendMessageBatchRequestEntry); } SendMessageBatchRequest sendMessageBatchRequest = new SendMessageBatchRequest(this.queueUrl, sendMessageBatchRequestEntries); SendMessageBatchResponse sendMessageBatchResponse = this.sqsClient.SendMessageBatch(sendMessageBatchRequest); SendMessageResponseModel <T> sendMessageBatchResponseModel = ConstructSendResponseModel <T>(sendMessageBatchRequestEntries, sendMessageBatchResponse); return(sendMessageBatchResponseModel); }
private static async Task SendBatch(SendMessageBatchRequest sendMessageBatchRequest) { SendMessageBatchResponse response = await client.SendMessageBatchAsync(sendMessageBatchRequest); Console.WriteLine("Messages successfully sent:"); foreach (var success in response.Successful) { Console.WriteLine(" Message id : {0}", success.MessageId); Console.WriteLine(" Message content MD5 : {0}", success.MD5OfMessageBody); } Console.WriteLine("Messages failed to send:"); foreach (var failed in response.Failed) { Console.WriteLine(" Message id : {0}", failed.Id); Console.WriteLine(" Message content : {0}", failed.Message); Console.WriteLine(" Sender's fault? : {0}", failed.SenderFault); } }
SendMessageBatchAsync_AnySendMessageBatchRequest_CallsSqsClientSendMessageBatchAsyncWithSendMessageBatchRequestAndCancellationTokenOnce() { var clientMock = new Mock <IAmazonSQS>(); var value = new SendMessageBatchRequest( "url", new List <SendMessageBatchRequestEntry> { new SendMessageBatchRequestEntry("id", new Sentence()) } ); var token = new CancellationToken(); var subject = new AmazonSQSCompressingClient(clientMock.Object); var result = subject.SendMessageBatchAsync(value, token); clientMock.Verify( x => x.SendMessageBatchAsync(value, token), Times.Once ); }
public async Task SQSWrapper_SendMessageBatchRequest_WrapRequestWithObject_OkResponse() { const string TestQueueName = "testQueueName"; const string ExpectedOperationName = "MessageBroker/SQS/Queue/Produce/Named/" + TestQueueName; const string QueueUrl = "https://sqs.us-west-2.amazonaws.com/123456789/" + TestQueueName; var batchEntries = new List <SendMessageBatchRequestEntry>(); batchEntries.Add(new SendMessageBatchRequestEntry("req1", "batch1message")); var sendMessageBatchRequest = new SendMessageBatchRequest(QueueUrl, batchEntries); var result = await SQSWrapper.WrapRequest(SendSQSBatchMessage, sendMessageBatchRequest); var span = _tracer.FinishedSpans()[0]; Assert.That(span.Tags["span.kind"], Is.EqualTo("client")); Assert.That(span.Tags["component"], Is.EqualTo("SQS")); Assert.That(span.Tags["aws.operation"], Is.EqualTo("Produce")); Assert.That(span.Tags["http.status_code"], Is.EqualTo((int)HttpStatusCode.OK)); Assert.That(span.OperationName, Is.EqualTo(ExpectedOperationName)); }
public IRequestContext Marshall(SendMessageBatchRequest input) { IRequestContext context = new RequestContext(); context.AddParametr("Action", input.ActionName); context.AddParametr("Version", YandexMqConfig.DEFAULT_SERVICE_VERSION); context.AddParametr("QueueUrl", input.QueueUrl); if (input.IsSetBatchEntry()) { var number = 1; foreach (var item in input.SendMessageBatchRequestEntry) { context.AddParametr($"SendMessageBatchRequestEntry.{number}.Id", item.Id); context.AddParametr($"SendMessageBatchRequestEntry.{number}.MessageBody", item.MessageBody); if (item.IsSetMessageAttribute()) { RequestAttributesBuilder.MessageAttributesBatchEntry(number, context, item.MessageAttribute); } if (item.DelaySeconds.HasValue) { context.AddParametr($"SendMessageBatchRequestEntry.{number}.DelaySeconds", item.DelaySeconds.ToString()); } if (!string.IsNullOrWhiteSpace(item.MessageDeduplicationId)) { context.AddParametr($"SendMessageBatchRequestEntry.{number}.MessageDeduplicationId", item.MessageDeduplicationId); } if (!string.IsNullOrWhiteSpace(item.MessageDeduplicationId)) { context.AddParametr($"SendMessageBatchRequestEntry.{number}.MessageGroupId", item.MessageGroupId); } number++; } } return(context); }
public override SendMessageBatchResponse SendMessageBatch(SendMessageBatchRequest sendMessageBatchRequest) { if (sendMessageBatchRequest == null) { throw new AmazonClientException("sendMessageBatch cannot be null"); } if (!clientConfiguration.IsLargePayloadSupportEnabled) { return(base.SendMessageBatch(sendMessageBatchRequest)); } for (var i = 0; i < sendMessageBatchRequest.Entries.Count; i++) { if (clientConfiguration.AlwaysThroughS3 || IsLarge(sendMessageBatchRequest.Entries[i])) { sendMessageBatchRequest.Entries[i] = StoreMessageInS3(sendMessageBatchRequest.Entries[i]); } } return(base.SendMessageBatch(sendMessageBatchRequest)); }
public override async Task <SendMessageBatchResponse> SendMessageBatchAsync(SendMessageBatchRequest sendMessageBatchRequest, CancellationToken cancellationToken = default(CancellationToken)) { if (sendMessageBatchRequest == null) { throw new AmazonClientException("sendMessageBatch cannot be null"); } if (!clientConfiguration.IsLargePayloadSupportEnabled) { return(await base.SendMessageBatchAsync(sendMessageBatchRequest, cancellationToken).ConfigureAwait(false)); } for (var i = 0; i < sendMessageBatchRequest.Entries.Count; i++) { if (clientConfiguration.AlwaysThroughS3 || IsLarge(sendMessageBatchRequest.Entries[i])) { sendMessageBatchRequest.Entries[i] = await StoreMessageInS3Async(sendMessageBatchRequest.Entries[i], cancellationToken); } } return(await base.SendMessageBatchAsync(sendMessageBatchRequest, cancellationToken)); }
private SendMessageBatchRequest CreateBatchRequest(Dictionary <Guid, T> map) { var entries = new List <SendMessageBatchRequestEntry>(); foreach (var item in map) { var serialized = SerializeEvent(item.Value); if (serialized == null) { continue; } var entry = new SendMessageBatchRequestEntry { Id = item.Key.ToString(), MessageBody = serialized }; entries.Add(entry); } var request = new SendMessageBatchRequest { QueueUrl = QueueUrl, Entries = entries }; return(request); }
public void TestSendMessageInBatch() { // Create the queue queueName = setQueueName(); queueURL = createQueueURLAsync(queueName); List <SendMessageBatchRequestEntry> batchMessage = new List <SendMessageBatchRequestEntry>(); //batchMessage.AddRange("one"); //batchMessage.AddRange("two"); //batchMessage.Add("three"); //batchMessage.Add("four"); //batchMessage.Add("four"); // Send message to the queue var request = new SendMessageBatchRequest(queueURL.Result.ToString(), batchMessage); // Get the response when sending the message request var response = sqsClient.SendMessageBatchAsync(request); // Verify the message body is correct between the request and the response ValidateMD5(request.Entries.ToString(), response.Result.ToString()); }
public bool SendMessages(List <Message> messages) { var queueUrl = GetQueueUrl(MailerQueueName); var entries = new List <SendMessageBatchRequestEntry>(); foreach (var message in messages) { entries.Add(new SendMessageBatchRequestEntry { Id = message.Id.ToString(), MessageBody = message.Body, MessageAttributes = new Dictionary <string, MessageAttributeValue> { { "MessageType", new MessageAttributeValue { StringValue = MessageType.Email.ToString() } } } } ); } var sendMessageBatchRequest = new SendMessageBatchRequest { Entries = entries, QueueUrl = queueUrl }; var sendMessageBatchResponse = _sqsClient.SendMessageBatch(sendMessageBatchRequest); // TODO: Handle failures. return(sendMessageBatchResponse.HttpStatusCode.Equals(HttpStatusCode.OK)); }
public int SendMessages(string queueUrl = null, int count = 1) { if (queueUrl == null) { queueUrl = defaultQueueUrl; } if (count <= 1) { var scalarResponse = client.SendMessage(new SendMessageRequest( queueUrl, FakeSqsModel.CreateJson())); return(string.IsNullOrEmpty(scalarResponse.MessageId) ? 0 : 1); } var request = new SendMessageBatchRequest { QueueUrl = queueUrl, Entries = new List <SendMessageBatchRequestEntry>(count) }; for (var x = 0; x < count; x++) { var model = new FakeSqsModel(); request.Entries.Add(new SendMessageBatchRequestEntry { Id = model.Id, MessageBody = model.ToJson(), }); } var response = client.SendMessageBatch(request); return(response.Successful.Count); }
private static async Task SendBatchMessage(IAmazonSQS amazonSqsClient, string queueUrl, int batchSize) { var request = new SendMessageBatchRequest { Entries = new List <SendMessageBatchRequestEntry>() { }, QueueUrl = queueUrl }; for (var i = 0; i < batchSize; i++) { request.Entries.Add(CreateNewEntry($"{i + _messageCount}")); } var response = await amazonSqsClient.SendMessageBatchAsync(request, CancellationToken.None); WriteToConsole(response.Successful); WriteToConsole(response.Failed); Console.ForegroundColor = ConsoleColor.White; }
public void RequestPictureTransformations(PicturesTransformationModel transformations) { using (var client = new AmazonSQSClient(_accessKeyId, _secretAccessKey, AmazonRegion.USWest2)) { var batchRequest = new SendMessageBatchRequest() { QueueUrl = _queueUrl, Entries = new List <SendMessageBatchRequestEntry>() }; foreach (var file in transformations.FileNames) { batchRequest.Entries.Add(new SendMessageBatchRequestEntry() { Id = Guid.NewGuid().ToString(), MessageBody = $"File '{file}' transformation [{transformations.Transformation.ToString()}] request", MessageAttributes = new Dictionary <string, MessageAttributeValue> { { "fileName", new MessageAttributeValue() { StringValue = file, DataType = "String" } }, { "transformation", new MessageAttributeValue() { StringValue = transformations.Transformation.ToString(), DataType = "String" } }, { "date", new MessageAttributeValue() { StringValue = DateTime.Now.ToString("HH:mm:ss dd.MM.yyyy"), DataType = "String" } } } }); } client.SendMessageBatchAsync(batchRequest).Wait(); } }
async Task SendOutgoingMessages(ConcurrentQueue <OutgoingMessage> outgoingMessages) { if (!outgoingMessages.Any()) { return; } var messagesByDestination = outgoingMessages .GroupBy(m => m.DestinationAddress) .ToList(); await Task.WhenAll( messagesByDestination .Select(async batch => { var entries = batch .Select(GetBatchRequestEntry) .ToList(); var destinationUrl = GetDestinationQueueUrlByName(batch.Key); foreach (var batchToSend in entries.Batch(10)) { var request = new SendMessageBatchRequest(destinationUrl, batchToSend); var response = await _client.SendMessageBatchAsync(request); if (response.Failed.Any()) { var failed = response.Failed .Select(f => new AmazonSQSException($"Failed {f.Message} with Id={f.Id}, Code={f.Code}, SenderFault={f.SenderFault}")); throw new AggregateException(failed); } } }) ); }
public async Task Long_Message_Async_S3KeyProvider_Is_Used_If_Configured() { var mockS3Provider = new Mock <IS3KeyProvider>(); mockS3Provider.Setup(m => m.GenerateName()).Returns(Constants.CustomPrefix + Guid.NewGuid().ToString("N")); var extendedClient = new AmazonSQSExtendedClient( sqsMock.Object, new ExtendedClientConfiguration() .WithLargePayloadSupportEnabled(s3Mock.Object, S3_BUCKET_NAME) .WithS3KeyProvider(mockS3Provider.Object)); var batchRequest = new SendMessageBatchRequest(SQS_QUEUE_NAME, new List <SendMessageBatchRequestEntry>()); for (var i = 0; i < 3; i++) { batchRequest.Entries.Add(new SendMessageBatchRequestEntry(Guid.NewGuid().ToString("N"), GenerateLongString(SQSExtendedClientConstants.DEFAULT_MESSAGE_SIZE_THRESHOLD + 1))); } await extendedClient.SendMessageBatchAsync(batchRequest); s3Mock.Verify(s => s.PutObjectAsync(It.Is <PutObjectRequest>(r => r.Key.StartsWith(Constants.CustomPrefix)), It.IsAny <CancellationToken>()), Times.Exactly(3)); sqsMock.Verify(s => s.SendMessageBatchAsync(It.Is <SendMessageBatchRequest>(r => r.Entries.All(e => MessagePointerIsCorrect(e.MessageBody, Constants.CustomPrefix) && LargePayloadAttributeIsAdded(e.MessageAttributes))), default(CancellationToken))); }
public async Task SQSWrapper_SendMessageBatchRequest_WrapRequestWithObject_GeneratesException() { const string TestQueueName = "testQueueName"; const string ExpectedOperationName = "MessageBroker/SQS/Queue/Produce/Named/" + TestQueueName; const string QueueUrl = "https://sqs.us-west-2.amazonaws.com/123456789/" + TestQueueName; const string ExpectedExceptionMessage = "whoopsie"; var batchEntries = new List <SendMessageBatchRequestEntry>(); batchEntries.Add(new SendMessageBatchRequestEntry("req1", ExpectedExceptionMessage)); var sendMessageBatchRequest = new SendMessageBatchRequest(QueueUrl, batchEntries); try { var result = await SQSWrapper.WrapRequest(SendSQSBatchMessageWithException, sendMessageBatchRequest); } catch (Exception) { var span = _tracer.FinishedSpans()[0]; Assert.That(span.Tags["span.kind"], Is.EqualTo("client")); Assert.That(span.Tags["component"], Is.EqualTo("SQS")); Assert.That(span.Tags["aws.operation"], Is.EqualTo("Produce")); Assert.That(span.Tags["error"], Is.EqualTo(true)); Assert.That(span.LogEntries.Count, Is.EqualTo(1)); Assert.That((string)span.LogEntries[0].Fields["event"], Is.EqualTo("error")); Assert.That(span.LogEntries[0].Fields["error.object"], Is.TypeOf(typeof(AmazonSQSException))); Assert.That((string)span.LogEntries[0].Fields["message"], Is.EqualTo(ExpectedExceptionMessage)); Assert.That((string)span.LogEntries[0].Fields["error.kind"], Is.EqualTo("Exception")); Assert.IsTrue(((string)span.LogEntries[0].Fields["stack"]).Contains("NewRelic.Tests.AwsLambda.AwsLambdaWrapperTests.LambdaWrapperTests.SendSQSBatchMessageWithException(SendMessageBatchRequest sendMessageBatchRequest, CancellationToken cancellationToken)")); Assert.That(span.OperationName, Is.EqualTo(ExpectedOperationName)); return; } Assert.Fail("Did not catch exception as expected."); }
public async Task <bool> Handle(SNSEvent @event, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); var entries = from record in @event.Records let mappedRequest = mapper.MapToRequest(record) let message = serializer.Serialize(mappedRequest) select new SendMessageBatchRequestEntry { Id = mappedRequest.Id.ToString(), MessageBody = message, }; var request = new SendMessageBatchRequest { QueueUrl = options.QueueUrl.ToString(), Entries = entries.ToList(), }; logger.LogInformation("Sending sqs:SendMessageBatch with request: {@request}", request); var response = await sqs.SendMessageBatchAsync(request, cancellationToken); logger.LogInformation("Received sqs:SendMessageBatch response: {@response}", response); return(true); }
/// <summary> /// /// </summary> /// <param name="messages"></param> /// <param name="token"></param> /// <returns></returns> public async Task SendBatchAsync(string[] messages, CancellationToken token) { Console.WriteLine(string.Format("Sending {0} messages: ", messages.Length)); using (var client = new Amazon.SQS.AmazonSQSClient(accessKey, secretKey, region)) { var queueUrlResult = await client.GetQueueUrlAsync(queueName, token); var queueUrl = queueUrlResult.QueueUrl; var entries = new List <SendMessageBatchRequestEntry>(); foreach (var message in messages) { var req = new SendMessageBatchRequestEntry(Guid.NewGuid().ToString(), message); if (isFIFO) { // required req.MessageGroupId = "group"; // fifo atua sobre o grupo req.MessageDeduplicationId = Guid.NewGuid().ToString(); } entries.Add(req); } var batch = new SendMessageBatchRequest(queueUrl, entries); var r = await client.SendMessageBatchAsync(batch); if (r.Failed != null && r.Failed.Count > 0) { foreach (var f in r.Failed) { Console.WriteLine("Failed : ", f.Message); } } } }
public void SendMessageBatchAsync_AnySendMessageBatchRequest_CallsMessageServiceToRequestBodyForEachEntry() { var clientMock = new Mock <IAmazonSQS>(); var messageServiceMock = new Mock <IMessageService>(); var values = new[] { (string)new Sentence(), (string)new Sentence() }; var entries = new List <SendMessageBatchRequestEntry> { new SendMessageBatchRequestEntry("id1", values[0]), new SendMessageBatchRequestEntry("id2", values[1]) }; var request = new SendMessageBatchRequest("url", entries); var token = new CancellationToken(); var subject = new AmazonSQSCompressingClientWithTestableMessageService( clientMock.Object, messageServiceMock.Object ); var result = subject.SendMessageBatchAsync(request, token); messageServiceMock.Verify( x => x.ToRequestBody(It.IsIn(values)), Times.Exactly(entries.Count) ); }
public Task <SendMessageBatchResponse> SendMessageBatchAsync( SendMessageBatchRequest request, CancellationToken cancellationToken = new CancellationToken()) { return(Task.FromResult(new SendMessageBatchResponse())); }
public Task <SendMessageBatchResponse> SendMessageBatchAsync(SendMessageBatchRequest request, CancellationToken token = default(CancellationToken)) { return(SendMessageBatch(request).AsTaskResult()); }
public SendMessageBatchResponse SendMessageBatch(SendMessageBatchRequest request) { if (request.Entries == null || request.Entries.Count <= 0) { throw new EmptyBatchRequestException("No entires in request"); } if (request.Entries.Count > SqsQueueDefinition.MaxBatchSendItems) { throw new TooManyEntriesInBatchRequestException($"Count of [{request.Entries.Count}] exceeds limit of [{SqsQueueDefinition.MaxBatchSendItems}]"); } var q = GetQueue(request.QueueUrl); var response = new SendMessageBatchResponse { Failed = new List <BatchResultErrorEntry>(), Successful = new List <SendMessageBatchResultEntry>() }; var entryIds = new HashSet <string>(); foreach (var entry in request.Entries) { string id = null; BatchResultErrorEntry batchError = null; try { if (entryIds.Contains(entry.Id)) { throw new BatchEntryIdsNotDistinctException($"Duplicate Id of [{entry.Id}]"); } entryIds.Add(entry.Id); id = q.Send(new SendMessageRequest { QueueUrl = q.QueueDefinition.QueueUrl, MessageAttributes = entry.MessageAttributes, MessageBody = entry.MessageBody, }); } catch (ReceiptHandleIsInvalidException rhex) { batchError = new BatchResultErrorEntry { Id = entry.Id, Message = rhex.Message, Code = rhex.ErrorCode }; } if (id == null) { var entryToQueue = batchError ?? new BatchResultErrorEntry { Id = entry.Id, Message = "FakeSendError", Code = "789" }; response.Failed.Add(entryToQueue); } else { response.Successful.Add(new SendMessageBatchResultEntry { Id = entry.Id, MessageId = id }); } } return(response); }
public static SendMessageBatchResponse SendMessageBatch(this IAmazonSQS client, SendMessageBatchRequest request) { return(client.SendMessageBatchAsync(request).GetResult()); }
public virtual Task <SendMessageBatchResponse> SendMessageBatchAsync(SendMessageBatchRequest request, CancellationToken cancellationToken = default(CancellationToken)) { return(amazonSqsToBeExtended.SendMessageBatchAsync(request, cancellationToken)); }
public virtual SendMessageBatchResponse SendMessageBatch(SendMessageBatchRequest request) { return(amazonSqsToBeExtended.SendMessageBatch(request)); }
public Task <SendMessageBatchResponse> SendMessageBatchAsync(SendMessageBatchRequest request, CancellationToken cancellationToken = new CancellationToken()) { BatchRequestsSent.Add(request); return(Task.FromResult(BatchRequestResponse(request))); }