public Task RunAsync() { var concurrency = Environment.ProcessorCount; var batchSize = 1000; var download = new TransformBlock <string, TempFile>( key => _s3.DownloadFileAsync(_options.BucketName, key), new ExecutionDataflowBlockOptions { BoundedCapacity = concurrency }); var batch = new BatchBlock <ElbLogEntry>(batchSize, new GroupingDataflowBlockOptions { BoundedCapacity = batchSize * concurrency * 2 }); var fileReader = new ActionBlock <TempFile>( file => PostEntriesAsync(file, batch), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = concurrency, BoundedCapacity = concurrency }); var serializer = new TransformBlock <ElbLogEntry[], HttpRequestMessage>( (Func <ElbLogEntry[], HttpRequestMessage>)SerializeBatch, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = concurrency, BoundedCapacity = concurrency }); var upload = new ActionBlock <HttpRequestMessage>( UploadBatch, new ExecutionDataflowBlockOptions { BoundedCapacity = concurrency * 2, MaxDegreeOfParallelism = concurrency }); var linkOpts = new DataflowLinkOptions { PropagateCompletion = true }; download.LinkTo(fileReader, linkOpts); fileReader.PropagateCompletion(batch); batch.LinkTo(serializer, linkOpts); serializer.LinkTo(upload, linkOpts); _s3.PostFileNamesAsync(_options.BucketName, _options.Prefix, download) .PropagateCompletion(download); return(upload.Completion); }