private (ITargetBlock <string>, IDataflowBlock) BuildPipeline()
{
var dataFlowOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = configuration.MaxDegreeOfParallelism
};
TransformBlock <string, LogEvent> parseJson = new TransformBlock <string, LogEvent>(input =>
{
return(ParseLogEvent(input));
}, dataFlowOptions);
var matchEvents = new TransformBlock <LogEvent, LogEventDetails>(input =>
{
return(ProcessLogEvent(input));
}, dataFlowOptions);
var batchEventDetails = new BatchBlock <LogEventDetails>(configuration.InsertBatchSize);
var insertEventDetails = new ActionBlock <LogEventDetails[]>(async chunk =>
{
await eventPersistence.Persist(chunk);
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = configuration.MaxDegreeOfParallelism * 4
});
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
parseJson.LinkTo(matchEvents, linkOptions, input => input != null);
// drain null values
parseJson.LinkTo(DataflowBlock.NullTarget <LogEvent>(), linkOptions);
matchEvents.LinkTo(batchEventDetails, linkOptions, input => input != null);
// drain null values
matchEvents.LinkTo(DataflowBlock.NullTarget <LogEventDetails>(), linkOptions);
batchEventDetails.LinkTo(insertEventDetails, linkOptions);
// When the batch block completes, set the action block also to complete.
batchEventDetails.Completion.ContinueWith(delegate { insertEventDetails.Complete(); });
return(parseJson, insertEventDetails);
}
