protected ITargetBlock <TSource> BuildPipeline(object context, out Task completionTask) { var readerBlock = new BufferBlock <TSource>(new DataflowBlockOptions { BoundedCapacity = Options.ReadChunkSize }); var processorBlock = new TransformBlock <TSource, TTarget>(async item => { return(await Processor().Process(item, context)); }, new ExecutionDataflowBlockOptions { BoundedCapacity = Options.ReadChunkSize, MaxDegreeOfParallelism = Options.MaxDegreeOfParallelism, SingleProducerConstrained = true }); var batchblock = new BatchBlock <TTarget>(Options.ReadChunkSize); var writerBlock = new ActionBlock <IEnumerable <TTarget> >(async items => { var writer = Writer(); await writer.Write(items.ToArray(), context); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Options.MaxDegreeOfParallelism }); readerBlock.LinkToWithPropagation(processorBlock); processorBlock.LinkToWithPropagation(batchblock); if (AccumulatingBlock != null) { var accumBlock = AccumulatingBlock(); batchblock.LinkToWithPropagation(accumBlock); accumBlock.LinkToWithPropagation(writerBlock); } else { batchblock.LinkToWithPropagation(writerBlock); } completionTask = new Task(() => { writerBlock.Completion.Wait(); Logger.LogTrace("writerBlock.Completion.Wait() returned"); if (CompletionWriter != null) { Logger.LogTrace("Calling CompletionWriter"); CompletionWriter.Write(null, context).Wait(); Logger.LogTrace("CompletionWriter returned"); } }); completionTask.Start(); return(readerBlock); }
protected ITargetBlock <TSource> BuildPipeline(object context, out Task completionTask) { var readerBlock = new BufferBlock <TSource>(new DataflowBlockOptions { BoundedCapacity = Options.ReadChunkSize }); var processorBlock = new TransformBlock <TSource, TTarget>(async item => { return(await Processor().Process(item, context)); }, new ExecutionDataflowBlockOptions { BoundedCapacity = Options.ReadChunkSize, MaxDegreeOfParallelism = Options.MaxDegreeOfParallelism, SingleProducerConstrained = true }); var batchblock = new BatchBlock <TTarget>(Options.ReadChunkSize); var taskScheduler = new ConcurrentExclusiveSchedulerPair(); var writerBlock = new ActionBlock <ICollection <TTarget> >(async items => { var writer = Writer(); await writer.Write(items, context); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Options.MaxDegreeOfParallelism, TaskScheduler = taskScheduler.ExclusiveScheduler }); readerBlock.LinkToWithPropagation(processorBlock); processorBlock.LinkToWithPropagation(batchblock); batchblock.LinkToWithPropagation(writerBlock); completionTask = new Task(() => { writerBlock.Completion.Wait(); Logger.LogDebug("writerBlock.Completion.Wait() returned"); if (CompletionWriter != null && Accumulator != null) { Logger.LogDebug("Calling CompletionWriter"); CompletionWriter.Write(Accumulator, context).Wait(); Logger.LogDebug("CompletionWriter returned"); } }); completionTask.Start(); return(readerBlock); }