public ITargetBlock <string> Build(DataflowBuilderConfiguration config)
{
var sinkCompletions = new List <Task>();
var fileParser =
new TransformManyBlock <string, UnparsedRecord>(fileLocation => config.FileParser.Parse(fileLocation));
var invalidParsedRecordExtractor = new TransformBlock <Validity <ParsedRecord>, InvalidRecord>(validity =>
new InvalidRecord(validity.Data.Id, validity.InvalidReason));
ITargetBlock <InvalidRecord> invalidRecordSink =
DataflowExtensions.CreateBatchedActionBlock <InvalidRecord>(BatchSize, config.InvalidRecordSink.Persist);
invalidParsedRecordExtractor.LinkTo(invalidRecordSink);
sinkCompletions.Add(invalidRecordSink.Completion);
foreach (RecordPipeline recordPipeline in config.RecordPipelines)
{
ITargetBlock <Validity <ParsedRecord> > sink = DataflowExtensions.ToTargetPipeline(
new TransformBlock <Validity <ParsedRecord>, ParsedRecord>(v => v.Data),
DataflowExtensions.CreateBatchedActionBlock <ParsedRecord>(BatchSize,
recordPipeline.ValidRecordSink.Persist));
fileParser.LinkTo(recordPipeline.Pipeline, DataflowExtensions.PropagateCompletion,
r => recordPipeline.RoutingPredicate(r));
recordPipeline.Pipeline.LinkTo(sink, DataflowExtensions.PropagateCompletion, validity => validity.IsValid);
recordPipeline.Pipeline.LinkTo(invalidParsedRecordExtractor, validity => !validity.IsValid);
sinkCompletions.Add(sink.Completion);
}
// We need to add unroutable last since it will greedily accept all messages. This won't work if any of
// the record pipeline have a limited capacity since they could overflow into this link
var unroutable = new TransformBlock <UnparsedRecord, InvalidRecord>(unparsed =>
new InvalidRecord(unparsed.Id, "Record did not match any of the pipelines' predicates."));
fileParser.LinkTo(unroutable, DataflowExtensions.PropagateCompletion);
invalidParsedRecordExtractor.CompleteWhenAllOrAnyFaulted(config.RecordPipelines.Select(p => p.Pipeline.Completion).ToList());
invalidRecordSink.CompleteWhenAllOrAnyFaulted(new List <Task> {
invalidParsedRecordExtractor.Completion, unroutable.Completion
});
return(DataflowExtensions.EncapsulateTarget(fileParser, Task.WhenAll(sinkCompletions)));
}
