/// <summary>
/// Creates a <see cref="Flow{TIn,TOut,TMat}"/> to publish messages in batches to a SQS queue using a <paramref name="client"/>.
/// </summary>
public static Flow <IEnumerable <SendMessageRequest>, IReadOnlyList <SqsPublishResultEntry>, NotUsed> Batch(
IAmazonSQS client, string queueUrl, SqsPublishBatchSettings settings = null)
{
settings = settings ?? SqsPublishBatchSettings.Default;
return(Flow.Create <IEnumerable <SendMessageRequest> >()
.Select(requests =>
{
var batch = new List <SendMessageRequest>();
var entries = new List <SendMessageBatchRequestEntry>();
var i = 0;
foreach (var request in requests)
{
batch.Add(request);
entries.Add(new SendMessageBatchRequestEntry(i.ToString(), request.MessageBody)
{
MessageDeduplicationId = request.MessageDeduplicationId,
MessageGroupId = request.MessageGroupId,
MessageAttributes = request.MessageAttributes
});
i++;
}
return Tuple.Create(batch, new SendMessageBatchRequest(queueUrl, entries));
})
.SelectAsync(settings.ConcurrentRequests, async tuple =>
{
var requests = tuple.Item1;
var batchRequest = tuple.Item2;
var response = await client.SendMessageBatchAsync(batchRequest);
if (response.Failed.Count == 0)
{
var responseMetadata = response.ResponseMetadata;
var resultEntries = response.Successful.ToDictionary(e => int.Parse(e.Id), e => e);
var results = new List <SqsPublishResultEntry>(requests.Count);
var i = 0;
foreach (var request in requests)
{
var result = resultEntries[i];
results.Add(new SqsPublishResultEntry(request, result, responseMetadata));
}
return (IReadOnlyList <SqsPublishResultEntry>)results;
}
else
{
var numberOfMessages = batchRequest.Entries.Count;
var nrOfFailedMessages = response.Failed.Count;
throw new SqsBatchException($"Some messages are failed to send. {nrOfFailedMessages} of {numberOfMessages} messages are failed");
}
}));
}
/// <summary>
/// Creates a <see cref="Sink{TIn,TMat}"/> to publish messages in batches
/// to a SQS queue using a <paramref name="client"/>.
/// See also: https://getakka.net/articles/streams/builtinstages.html#groupedwithin
/// </summary>
public static Sink <IEnumerable <SendMessageRequest>, Task> BatchedMessageSink(IAmazonSQS client, string queueUrl, SqsPublishBatchSettings settings = null) =>
SqsPublishFlow.Batch(client, queueUrl, settings)
.ToMaterialized(Sink.Ignore <IReadOnlyList <SqsPublishResultEntry> >(), Keep.Right);
/// <summary>
/// Creates a <see cref="Sink{TIn,TMat}"/> that accepts an iterable of strings and publish
/// them as messages in batches to a SQS queue using a <paramref name="client"/>.
/// See also: https://getakka.net/articles/streams/builtinstages.html#groupedwithin
/// </summary>
public static Sink <IEnumerable <string>, Task> Batch(IAmazonSQS client, string queueUrl,
SqsPublishBatchSettings settings = null) =>
Flow.FromFunction((IEnumerable <string> msgs) =>
msgs.Select(msg => new SendMessageRequest(queueUrl, msg)))
.ToMaterialized(BatchedMessageSink(client, queueUrl, settings), Keep.Right);