protected ITargetBlock <TSource> BuildPipeline(object context, out Task completionTask)
{
var readerBlock = new BufferBlock <TSource>(new DataflowBlockOptions {
BoundedCapacity = Options.ReadChunkSize
});
var processorBlock = new TransformBlock <TSource, TTarget>(async item =>
{
return(await Processor().Process(item, context));
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = Options.ReadChunkSize,
MaxDegreeOfParallelism = Options.MaxDegreeOfParallelism,
SingleProducerConstrained = true
});
var batchblock = new BatchBlock <TTarget>(Options.ReadChunkSize);
var writerBlock = new ActionBlock <IEnumerable <TTarget> >(async items =>
{
var writer = Writer();
await writer.Write(items.ToArray(), context);
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Options.MaxDegreeOfParallelism
});
readerBlock.LinkToWithPropagation(processorBlock);
processorBlock.LinkToWithPropagation(batchblock);
if (AccumulatingBlock != null)
{
var accumBlock = AccumulatingBlock();
batchblock.LinkToWithPropagation(accumBlock);
accumBlock.LinkToWithPropagation(writerBlock);
}
else
{
batchblock.LinkToWithPropagation(writerBlock);
}
completionTask = new Task(() =>
{
writerBlock.Completion.Wait();
Logger.LogTrace("writerBlock.Completion.Wait() returned");
if (CompletionWriter != null)
{
Logger.LogTrace("Calling CompletionWriter");
CompletionWriter.Write(null, context).Wait();
Logger.LogTrace("CompletionWriter returned");
}
});
completionTask.Start();
return(readerBlock);
}
protected ITargetBlock <TSource> BuildPipeline(object context, out Task completionTask)
{
var readerBlock = new BufferBlock <TSource>(new DataflowBlockOptions {
BoundedCapacity = Options.ReadChunkSize
});
var processorBlock = new TransformBlock <TSource, TTarget>(async item =>
{
return(await Processor().Process(item, context));
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = Options.ReadChunkSize,
MaxDegreeOfParallelism = Options.MaxDegreeOfParallelism,
SingleProducerConstrained = true
});
var batchblock = new BatchBlock <TTarget>(Options.ReadChunkSize);
var taskScheduler = new ConcurrentExclusiveSchedulerPair();
var writerBlock = new ActionBlock <ICollection <TTarget> >(async items =>
{
var writer = Writer();
await writer.Write(items, context);
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Options.MaxDegreeOfParallelism,
TaskScheduler = taskScheduler.ExclusiveScheduler
});
readerBlock.LinkToWithPropagation(processorBlock);
processorBlock.LinkToWithPropagation(batchblock);
batchblock.LinkToWithPropagation(writerBlock);
completionTask = new Task(() =>
{
writerBlock.Completion.Wait();
Logger.LogDebug("writerBlock.Completion.Wait() returned");
if (CompletionWriter != null && Accumulator != null)
{
Logger.LogDebug("Calling CompletionWriter");
CompletionWriter.Write(Accumulator, context).Wait();
Logger.LogDebug("CompletionWriter returned");
}
});
completionTask.Start();
return(readerBlock);
}
