public StartableBlock <DataBatch> Create(string peopleJsonFilePath,
string targetFilePath,
IProgress <PipelineProgress> progress,
CancellationTokenSource cancellationSource)
{
var dataPool = new DataPool();
// Create blocks
var readBlock = _readingBlockFactory.Create(peopleJsonFilePath, dataPool, cancellationSource.Token);
var processBlock = new ProcessingBlock <DataBatch>(DataflowFacade.TransformBlock("Process",
DataBatch.IdGetter,
x =>
{
x.Data.Where(item => item.IsValid).ForEach(_personValidator.Validate);
x.Data.Where(item => item.IsValid).ForEach(_personFieldsComputer.Compute);
for (var i = 0; i < Settings.ExtraProcessingBlocksCount; i++)
{
x.Data.ForEach(_ => ComputationsHelper.PerformTimeConsumingOperation());
}
},
cancellationSource.Token,
Settings.DegreeOfParallelism));
var writeBlock = _writingBlockFactory.Create(targetFilePath, cancellationSource.Token);
var progressBlock = _progressReportingBlockFactory.Create("ReportProgress",
DataBatch.IdGetter,
progress,
readBlock.EstimatedOutputCount,
1,
cancellationSource.Token);
var disposeBlock = new ProcessingBlock <DataBatch>(DataflowFacade.TransformBlock("DisposeData",
DataBatch.IdGetter,
x => x.Data.ForEach(dataPool.Return),
cancellationSource.Token));
return(_straightPipelineFactory.Create(readBlock,
new[] { processBlock, writeBlock, progressBlock, disposeBlock },
cancellationSource));
}
private IEnumerable <ProcessingBlock <DataBatch> > CreateExtraProcessingBlocks(CancellationTokenSource cancellationSource)
{
return(Enumerable.Range(1, Settings.ExtraProcessingBlocksCount)
.Select(x => ProcessingBlock <DataBatch> .Create($"ExtraProcessing {x}",
DataBatch.IdGetter,
batch => batch.Data.ForEach(_ => ComputationsHelper.PerformTimeConsumingOperation()),
cancellationSource.Token,
Settings.ProcessInParallel ? Settings.MaxDegreeOfParallelism : 1)));
}
