private IPropagatorBlock <int, Data> UseLinesReaderAndParser(StreamReader peopleJsonStream, DataPool dataPool, CancellationToken cancellation)
{
// Create blocks
// NOTE:
// - extract part which must be single-thread
// - ability to replace just reading (e.g. from db)
var readLinesBlock = DataflowFacade.TransformManyBlock <int, Data>(
"ReadLines",
x => - 1,
x => _streamLinesReader.Read(peopleJsonStream, x)
.Select(line =>
{
var data = dataPool.Rent();
data.PersonJson = line;
return(data);
}),
cancellation);
// NOTE: can be multi-thread
var parseDataBlock = DataflowFacade.TransformBlock(
"ParseData",
Data.IdGetter,
x => _dataParser.Parse(x),
cancellation);
// Link blocks
readLinesBlock.LinkWithCompletion(parseDataBlock);
return(DataflowBlock.Encapsulate(readLinesBlock, parseDataBlock));
}
private IPropagatorBlock <int, Data> UseDataReader(StreamReader peopleJsonStream, DataPool dataPool, CancellationToken cancellation)
{
return(DataflowFacade.TransformManyBlock <int, Data>(
"ReadData",
x => - 1,
x => _dataReader.Read(peopleJsonStream, x, dataPool),
cancellation));
}