public Task RunAsync()
{
var concurrency = Environment.ProcessorCount;
var batchSize = 1000;
var download = new TransformBlock <string, TempFile>(
key => _s3.DownloadFileAsync(_options.BucketName, key),
new ExecutionDataflowBlockOptions {
BoundedCapacity = concurrency
});
var batch = new BatchBlock <ElbLogEntry>(batchSize, new GroupingDataflowBlockOptions {
BoundedCapacity = batchSize * concurrency * 2
});
var fileReader = new ActionBlock <TempFile>(
file => PostEntriesAsync(file, batch),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = concurrency, BoundedCapacity = concurrency
});
var serializer = new TransformBlock <ElbLogEntry[], HttpRequestMessage>(
(Func <ElbLogEntry[], HttpRequestMessage>)SerializeBatch,
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = concurrency, BoundedCapacity = concurrency
});
var upload = new ActionBlock <HttpRequestMessage>(
UploadBatch,
new ExecutionDataflowBlockOptions {
BoundedCapacity = concurrency * 2, MaxDegreeOfParallelism = concurrency
});
var linkOpts = new DataflowLinkOptions {
PropagateCompletion = true
};
download.LinkTo(fileReader, linkOpts);
fileReader.PropagateCompletion(batch);
batch.LinkTo(serializer, linkOpts);
serializer.LinkTo(upload, linkOpts);
_s3.PostFileNamesAsync(_options.BucketName, _options.Prefix, download)
.PropagateCompletion(download);
return(upload.Completion);
}