public async Task SendMessageBatchExample()
{
// Create service client using the SDK's default logic for determining AWS credentials and region to use.
// For information configuring service clients checkout the .NET developer guide: https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/net-dg-config.html
AmazonSQSClient client = new AmazonSQSClient();
var sendMessageBatchRequest = new SendMessageBatchRequest
{
Entries = new List <SendMessageBatchRequestEntry>
{
new SendMessageBatchRequestEntry("message1", "FirstMessageContent"),
new SendMessageBatchRequestEntry("message2", "SecondMessageContent"),
new SendMessageBatchRequestEntry("message3", "ThirdMessageContent")
},
QueueUrl = "SQS_QUEUE_URL"
};
var response = await client.SendMessageBatchAsync(sendMessageBatchRequest);
Console.WriteLine("Messages successfully sent:");
foreach (var success in response.Successful)
{
Console.WriteLine(" Message id : {0}", success.MessageId);
Console.WriteLine(" Message content MD5 : {0}", success.MD5OfMessageBody);
}
Console.WriteLine("Messages failed to send:");
foreach (var failed in response.Failed)
{
Console.WriteLine(" Message id : {0}", failed.Id);
Console.WriteLine(" Message content : {0}", failed.Message);
Console.WriteLine(" Sender's fault? : {0}", failed.SenderFault);
}
}
public async Task <bool> SendBatchMessage(ICollection <IResult> lstMessages)
{
try
{
var slice = 10;
var result = true;
for (int i = 0; i < lstMessages.Count(); i = i + slice)
{
var response = await _client.SendMessageBatchAsync(
new SendMessageBatchRequest(
_queueUrl,
lstMessages
.Skip(i)
.Take(slice)
.Select(item => new SendMessageBatchRequestEntry(Guid.NewGuid().ToString(), item.ToJson()))
.ToList()
));
result = response.Failed.Count() == 0;
}
return(result);
}
catch (Exception ex)
{
Console.WriteLine("Fail to send messages.");
Console.WriteLine(ex.Message);
throw ex;
}
}
public async Task <string> CreateMessages([FromBody] string[] messages)
{
var queueUrl = await QueueUrl.Build(_client, QueueName, _sqsConfigParam.QueueAttributes);
var sendMessageBatchRequest = new SendMessageBatchRequest
{
Entries = messages
.Select((message, index) => new SendMessageBatchRequestEntry(index.ToString(), message)).ToList(),
QueueUrl = queueUrl.Value
};
var response = await _client.SendMessageBatchAsync(sendMessageBatchRequest);
if (response.HttpStatusCode == HttpStatusCode.OK)
{
var result = new
{
StatusCode = "正常終了",
Message = "SQSへメッセージの登録に成功しました。",
SuccessfulResponse = response.Successful,
FailedResponse = response.Failed
};
return(JsonConvert.SerializeObject(result));
}
else
{
var result = new
{
StatusCode = "エラー",
Message = "SQSへメッセージの登録に失敗しました。"
};
return(JsonConvert.SerializeObject(result));
}
}
static async Task Main()
{
var client = new AmazonSQSClient();
const string queueUrl = "<please input your queue's url>";
// メッセージをエンキューする
await client.SendMessageAsync(queueUrl, "Hello SQS !!");
// 複数のメッセージをエンキューする
await client.SendMessageBatchAsync(queueUrl, Enumerable.Range(0, 5).Select(i => new SendMessageBatchRequestEntry
{
Id = i.ToString(),
MessageBody = "Hello SQS !! - " + i,
}).ToList());
// メッセージをデキューする
var receiveMessageRequest = new ReceiveMessageRequest
{
QueueUrl = queueUrl,
MaxNumberOfMessages = 10,
};
var messages = (await client.ReceiveMessageAsync(receiveMessageRequest)).Messages;
foreach (var message in messages)
{
// メッセージの内容を表示する
Console.WriteLine(message.Body);
// メッセージを削除する
await client.DeleteMessageAsync(queueUrl, message.ReceiptHandle);
}
}
/// <inheritdoc/>
public async Task PublishAsync(IEnumerable <Message> messages, CancellationToken cancellationToken = default)
{
_ = messages ?? throw new ArgumentNullException(nameof(messages));
if (messages.Any(m => m.EnqueueTime.HasValue))
{
// TODO: It might be possible to implement this with repeating DelaySeconds
throw new ArgumentException("The EnqueueTime property is not supported with Amazon SQS.");
}
var batch = new SendMessageBatchRequest
{
QueueUrl = await GetQueueUrl().ConfigureAwait(false),
Entries = messages.Select(m => new SendMessageBatchRequestEntry
{
Id = m.Id ?? Guid.NewGuid().ToString(),
MessageBody = Convert.ToBase64String(m.Content),
MessageGroupId = m.PartitionId,
MessageAttributes = m.Properties.ToDictionary(p => p.Key, p => new MessageAttributeValue
{
StringValue = p.Value.ToString(),
DataType = "String"
}),
}).ToList()
};
var response = await _client.SendMessageBatchAsync(batch, cancellationToken).ConfigureAwait(false);
if (response.Failed.Any())
{
throw new AmazonSQSException("Not all messages have been sent.");
}
}
public async Task SendAsync(string channelName, IEnumerable <QueueMessage> messages, CancellationToken cancellationToken = default)
{
if (channelName is null)
{
throw new ArgumentNullException(nameof(channelName));
}
if (messages is null)
{
throw new ArgumentNullException(nameof(messages));
}
string queueUri = GetQueueUri(channelName);
// SQS request size is limited
foreach (IEnumerable <QueueMessage> chunk in messages.Chunk(MaxEntriesPerRequest))
{
var request = new SendMessageBatchRequest(
queueUri,
chunk.Select(Converter.ToSQSMessage).ToList());
try
{
await _client.SendMessageBatchAsync(request, cancellationToken).ConfigureAwait(false);
}
catch (AmazonSQSException ex) when(ex.ErrorCode == "AWS.SimpleQueueService.NonExistentQueue")
{
throw new InvalidOperationException(
$"the queue '{channelName}' doesn't exist.", ex);
}
}
}
public void EnqueueObjects <T>(List <T> objects) where T : Models.Dynamo.ReceiptBase
{
if (objects.Count == 0)
{
return;
}
using (var sqsClient = new AmazonSQSClient())
{
var success = 0;
var failed = 0;
var count = objects.Count < _maxQueue ? 1 : (int)Math.Ceiling(((double)objects.Count) / ((double)_maxQueue));
for (var i = 0; i < count; i++)
{
var result = sqsClient.SendMessageBatchAsync(new SendMessageBatchRequest
{
Entries = objects.Skip(i * _maxQueue).Take(_maxQueue).Select(x => new SendMessageBatchRequestEntry
{
Id = Guid.NewGuid().ToString(),
MessageBody = JsonConvert.SerializeObject(x)
})
.ToList(),
QueueUrl = Environment.GetEnvironmentVariable("Queue")
}).Result;
success += result.Successful.Count;
failed += result.Failed.Count;
}
Console.WriteLine($"Successful message count is {success}");
Console.WriteLine($"Failed message count is {failed}");
}
}
private Task <SendMessageBatchResponse> sendCityFeed(List <CityFeed> cityFeeds)
{
// Prone to failure: requires a formal SQSClient designed to handle Errors and failures
var batch = from feed in cityFeeds
select new SendMessageBatchRequestEntry(Guid.NewGuid().ToString(), JsonSerializer.Serialize(feed));
return(_sqsClient.SendMessageBatchAsync(_sqsCityFeedUrl, batch.ToList()));
}
public static void SendMessageBatch(List <SendMessageBatchRequestEntry> messages, string queueUrl)
{
var request = new SendMessageBatchRequest
{
Entries = messages,
QueueUrl = queueUrl
};
var task = client.SendMessageBatchAsync(request);
task.Wait();
}
public async Task EnqueueAsync <T>(IEnumerable <T> messages, CancellationToken cancellationToken) where T : class, IMessage
{
var group = messages.Select(x => new SendMessageBatchRequestEntry {
MessageGroupId = x.QName, Id = Guid.NewGuid().ToString(), MessageBody = _serializationManager.Serialize(x)
}).ToList();
await _sqsClient.SendMessageBatchAsync(new SendMessageBatchRequest
{
Entries = group,
QueueUrl = "http://localhost:9324/queue/default",
});
}
static void Main(string[] args)
{
var num_messages = int.Parse(args[0]);
var queue_url = args[1];
var ids = new int[] { 0, 0, 0, 0 };
System.Console.WriteLine($"Sending {num_messages} messages to SQS queue {queue_url}");
var sqs = new AmazonSQSClient(); //need keys and region
var testMessage = new Faker <TestMessage>()
.RuleFor(u => u.client_id, f => f.Random.Number(1, 4)) //"1")
.RuleFor(u => u.id, (f, u) => ids[u.client_id - 1]++)
.RuleFor(u => u.member_id, f => f.Random.Number(1, 1000000).ToString())
.FinishWith((f, u) => System.Console.WriteLine($"Message created for client_id {u.client_id}"));
try
{
var messages = new List <TestMessage>();
for (int i = 0; i < num_messages; i++)
{
var message = testMessage.Generate();
messages.Add(message);
if (messages.Count == 10)
{
var result = sqs.SendMessageBatchAsync(new SendMessageBatchRequest {
QueueUrl = queue_url,
Entries = messages.Select(x => new SendMessageBatchRequestEntry {
Id = Guid.NewGuid().ToString(),
MessageBody = JsonConvert.SerializeObject(x),
MessageGroupId = x.client_id.ToString(),
MessageAttributes = new Dictionary <string, MessageAttributeValue>()
}).ToList()
}).Result;
if (result.Failed.Any())
{
foreach (var entry in result.Failed)
{
System.Console.WriteLine($"Item {entry.Id} failed: {entry.Message}");
}
}
messages.Clear();
}
}
}
catch (System.Exception)
{
throw;
}
}
/// <summary>
/// Puts a batch of messages on the queue
///
/// Supports a maximum of 10 messages.
/// </summary>
/// <param name="batchMessages">An array of messages to be put on the queue</param>
/// <param name="queueName">The name of the queue</param>
/// <returns></returns>
public async Task <IBatchResponse> SendMessageBatchAsync(BatchMessageRequest[] batchMessages, string queueName)
{
if (batchMessages.Length > 10)
{
throw new ArgumentException($"AWS SQS supports a max message number of 10 messages. {batchMessages.Length} were received.", nameof(batchMessages));
}
var queueUrl = await GetQueueUrlAsync(queueName);
var sendMessageBatchRequest = new SendMessageBatchRequest
{
Entries = batchMessages.Select((v, i) => new SendMessageBatchRequestEntry(i.ToString(), v.Message)
{
MessageAttributes = v.MessageAttributes.ToDictionary(attribute => attribute.Key, attribute => new MessageAttributeValue {
StringValue = attribute.Value, DataType = "String"
}),
MessageGroupId = queueName.EndsWith(".fifo") ? queueUrl : null
}).ToList(),
QueueUrl = queueUrl
};
var retryCounter = 0;
SendMessageBatchResponse awsBatchResponse = null;
while (awsBatchResponse == null)
{
try
{
awsBatchResponse = await _client.SendMessageBatchAsync(sendMessageBatchRequest);
}
catch (AmazonSQSException e)
{
if (e.Message.EndsWith("Throttled") && retryCounter < 10)
{
retryCounter += 1;
await Task.Delay(retryCounter * 3);
}
else
{
throw;
}
}
}
return(BatchResponse.FromAwsBatchResponse(awsBatchResponse, batchMessages));
}
public async Task <IEnumerable <string> > PublishBatch(IEnumerable <string> messages, int retry = 0)
{
var request = new SendMessageBatchRequest
{
QueueUrl = QueueUrl,
};
messages.Do(message =>
request.Entries.Add(new SendMessageBatchRequestEntry
{
MessageBody = message,
Id = JsonConvert.DeserializeObject <JObject>(message)["Id"]?.ToString()
?? Guid.NewGuid().ToString(),
}));
if (IsFifo)
{
request.Entries.ForEach(message =>
{
message.MessageDeduplicationId =
JsonConvert.DeserializeObject <JObject>(message.MessageBody)["DeduplicationId"]?.ToString();
message.MessageGroupId = "Default";
});
}
await Limiter.Add(request.Entries.Count);
var response = await Client.SendMessageBatchAsync(request);
var successfuls = response.Successful.Select(m => m.MessageId).ToList();
if (response.Failed.Any())
{
if (retry > MAX_RETRY)
{
throw new Exception("Failed to send all requests because : " + response.Failed.Select(f => f.Code).ToString(Environment.NewLine));
}
Log.For(this).Warning($"Failed to send {response.Failed.Select(c => c.Message).ToLinesString()} because : {response.Failed.Select(c => c.Code).ToLinesString()}. Retrying for {retry}/{MAX_RETRY}.");
var toSend = response.Failed.Select(f => f.Message);
successfuls.AddRange(await PublishBatch(toSend, retry++));
}
return(successfuls);
}
public async Task PutMessagesAsync(IReadOnlyCollection <QueueMessage> messages, CancellationToken cancellationToken = default)
{
if (messages == null)
{
return;
}
// SQS request size is limited
foreach (IEnumerable <QueueMessage> chunk in messages.Chunk(MaxEntriesPerRequest))
{
var request = new SendMessageBatchRequest(
_queueUrl,
chunk.Select(Converter.ToSQSMessage).ToList());
SendMessageBatchResponse r = await _client.SendMessageBatchAsync(request, cancellationToken);
}
}
private static async Task SendBatch(SendMessageBatchRequest sendMessageBatchRequest)
{
SendMessageBatchResponse response = await client.SendMessageBatchAsync(sendMessageBatchRequest);
Console.WriteLine("Messages successfully sent:");
foreach (var success in response.Successful)
{
Console.WriteLine(" Message id : {0}", success.MessageId);
Console.WriteLine(" Message content MD5 : {0}", success.MD5OfMessageBody);
}
Console.WriteLine("Messages failed to send:");
foreach (var failed in response.Failed)
{
Console.WriteLine(" Message id : {0}", failed.Id);
Console.WriteLine(" Message content : {0}", failed.Message);
Console.WriteLine(" Sender's fault? : {0}", failed.SenderFault);
}
}
public async Task <HttpStatusCode> SendMessages(IEnumerable <string> messages, string url = null)
{
if (url == null)
{
url = awsSQSOptions.Url;
}
var sendMessageResponse = await amazonSQSClient.SendMessageBatchAsync(new SendMessageBatchRequest()
{
QueueUrl = url,
Entries = messages.Select((message, index) => new SendMessageBatchRequestEntry()
{
Id = index.ToString(),
MessageBody = message,
DelaySeconds = 0
}).ToList()
});
return(sendMessageResponse.HttpStatusCode);
}
private static async Task <bool> WriteSplit <T>(string streamName, List <T> list)
{
var client = new AmazonSQSClient(creds, config);
var data = new List <SendMessageBatchRequestEntry>();
foreach (var l2 in list)
{
var json = JsonConvert.SerializeObject(l2);
data.Add(new SendMessageBatchRequestEntry(data.Count.ToString(), json));
}
var res = await client.SendMessageBatchAsync(new SendMessageBatchRequest()
{
Entries = data,
QueueUrl = streamName
});
return(!res.Failed.Any());
}
public void TestSendMessageInBatch()
{
// Create the queue
queueName = setQueueName();
queueURL = createQueueURLAsync(queueName);
List <SendMessageBatchRequestEntry> batchMessage = new List <SendMessageBatchRequestEntry>();
//batchMessage.AddRange("one");
//batchMessage.AddRange("two");
//batchMessage.Add("three");
//batchMessage.Add("four");
//batchMessage.Add("four");
// Send message to the queue
var request = new SendMessageBatchRequest(queueURL.Result.ToString(), batchMessage);
// Get the response when sending the message request
var response = sqsClient.SendMessageBatchAsync(request);
// Verify the message body is correct between the request and the response
ValidateMD5(request.Entries.ToString(), response.Result.ToString());
}
public async Task <CloudPlatformResponse> EnqueueMessagesAsync(CloudPlatformRequest request)
{
var cResponse = new CloudPlatformResponse();
try
{
SendMessageBatchResponse response;
var messages = (IEnumerable <string>)request.Data[ServerlessBenchmark.Constants.Message];
var batchMessageEntry =
messages.Select(message => new SendMessageBatchRequestEntry(Guid.NewGuid().ToString("N"), message))
.ToList();
using (var client = new AmazonSQSClient())
{
response = await client.SendMessageBatchAsync(client.GetQueueUrl(request.Source).QueueUrl, batchMessageEntry);
}
if (response.Failed.Any())
{
var groupedFailures = response.Failed.GroupBy(failure => failure.Message);
foreach (var group in groupedFailures)
{
cResponse.ErrorDetails.Add(group.Key, group.Count());
}
cResponse.HttpStatusCode = HttpStatusCode.InternalServerError;
}
else
{
cResponse.HttpStatusCode = HttpStatusCode.OK;
}
}
catch (InvalidCastException)
{
Console.WriteLine("Data needs to be IEnumberable of strings");
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
return(cResponse);
}
public void RequestPictureTransformations(PicturesTransformationModel transformations)
{
using (var client = new AmazonSQSClient(_accessKeyId, _secretAccessKey, AmazonRegion.USWest2))
{
var batchRequest = new SendMessageBatchRequest()
{
QueueUrl = _queueUrl,
Entries = new List <SendMessageBatchRequestEntry>()
};
foreach (var file in transformations.FileNames)
{
batchRequest.Entries.Add(new SendMessageBatchRequestEntry()
{
Id = Guid.NewGuid().ToString(),
MessageBody = $"File '{file}' transformation [{transformations.Transformation.ToString()}] request",
MessageAttributes = new Dictionary <string, MessageAttributeValue>
{
{ "fileName", new MessageAttributeValue()
{
StringValue = file, DataType = "String"
} },
{ "transformation", new MessageAttributeValue()
{
StringValue = transformations.Transformation.ToString(), DataType = "String"
} },
{ "date", new MessageAttributeValue()
{
StringValue = DateTime.Now.ToString("HH:mm:ss dd.MM.yyyy"), DataType = "String"
} }
}
});
}
client.SendMessageBatchAsync(batchRequest).Wait();
}
}
public virtual async Task <SendMessageBatchResponse> SendMessageBatchAsync(SendMessageBatchRequest sendMessageBatchRequest)
{
var sendMessageBatchResponse = await _amazonSqsClient.SendMessageBatchAsync(sendMessageBatchRequest);
return(sendMessageBatchResponse);
}
public async Task PutMessagesAsync(IEnumerable <QueueMessage> messages, CancellationToken cancellationToken = default)
{
var sqs = messages.Select(ToSQSMessage).ToList();
SendMessageBatchResponse r = await _client.SendMessageBatchAsync(_queueName, sqs, cancellationToken);
}
