ByPartitionAndData(string streamName, KinesisFlowSettings settings = null, Func <IAmazonKinesis> client = null)
{
return(Flow.Create <(string, MemoryStream)>()
.Select(tuple => new PutRecordsRequestEntry
{
PartitionKey = tuple.Item1,
Data = tuple.Item2
})
.Via(Create(streamName, settings, client)));
}
Create(string streamName, KinesisFlowSettings settings = null, Func <IAmazonKinesis> client = null)
{
settings = settings ?? KinesisFlowSettings.Default;
client = client ?? DefaultClientFactory;
return(Flow.Create <PutRecordsRequestEntry>()
.Throttle(settings.MaxRecordsPerSecond, Second, settings.MaxRecordsPerSecond, ThrottleMode.Shaping)
.Throttle(settings.MaxBytesPerSecond, Second, settings.MaxBytesPerSecond, GetPayloadByteSize, ThrottleMode.Shaping)
.Batch(settings.MaxBatchSize, ImmutableQueue.Create, (queue, request) => queue.Enqueue(request))
.Via(new KinesisFlowStage(streamName, settings.MaxRetries, settings.BackoffStrategy, settings.RetryInitialTimeout, client))
.SelectAsync(settings.Parallelism, task => task)
.SelectMany(result => result));
}
ByPartitionAndBytes(string streamName, KinesisFlowSettings settings = null, Func <IAmazonKinesis> client = null)
{
return(Flow.Create <(string, ByteString)>()
.Select(tuple =>
{
var bytes = tuple.Item2;
var stream = new MemoryStream(bytes.Count);
bytes.WriteTo(stream);
return new PutRecordsRequestEntry
{
PartitionKey = tuple.Item1,
Data = stream
};
})
.Via(Create(streamName, settings, client)));
}
ByPartitionAndBytes(string streamName, KinesisFlowSettings settings = null, Func <IAmazonKinesis> client = null) =>
Create(string streamName, KinesisFlowSettings settings = null, Func <IAmazonKinesis> client = null) => KinesisFlow.Create(streamName, settings, client).To(Sink.Ignore <PutRecordsResultEntry>());