public StartableBlock <TData> Create <TData>(StartableBlock <TData> source,
ProcessingBlock <TData>[] processors,
CancellationTokenSource cancellationSource)
{
// The pipeline looks like this:
// source -> processor1 -> processor2 -> processor3 (output)
// Link blocks
source.Output.LinkWithCompletion(processors[0].Processor);
for (var i = 0; i < processors.Length - 1; i++)
{
processors[i].Processor.LinkWithCompletion(processors[i + 1].Processor);
}
// Create global completion
var completion = TaskExtensions.CreateGlobalCompletion(new[] { source.Completion }.Concat(processors.Select(x => x.Completion)),
cancellationSource);
return(new StartableBlock <TData>
{
Start = source.Start,
Output = processors.Last().Processor,
EstimatedOutputCount = source.EstimatedOutputCount,
Completion = completion
});
}
public StartableBlock <TData> Create <TData>(StartableBlock <TData> source,
ProcessingBlock <TData>[] processors,
ProcessingBlock <TData> errorHandler,
ProcessingBlock <TData> output,
Predicate <TData> validDataPredicate,
CancellationTokenSource cancellationSource)
{
// The pipeline looks like this:
// source -> processor1 -> processor2 -> output
// \ | | ^
// \ v v /
// \--> errorHandler ----------/
// Link blocks
source.Output.LinkWithCompletion(processors[0].Processor, validDataPredicate);
source.Output.LinkTo(errorHandler.Processor, x => !validDataPredicate(x));
for (var i = 0; i < processors.Length - 1; i++)
{
processors[i].Processor.LinkWithCompletion(processors[i + 1].Processor, validDataPredicate);
processors[i].Processor.LinkTo(errorHandler.Processor, x => !validDataPredicate(x));
}
var lastProcessor = processors.Last();
lastProcessor.Processor.LinkTo(output.Processor, validDataPredicate);
lastProcessor.Processor.LinkTo(errorHandler.Processor, x => !validDataPredicate(x));
errorHandler.Processor.LinkTo(output.Processor);
// Propagate completions of multiple inputs
errorHandler.Processor.DeriveCompletionOrFaultFrom(new[] { source.Output }.Concat(processors.Select(x => x.Processor)));
output.Processor.DeriveCompletionOrFaultFrom(lastProcessor.Processor, errorHandler.Processor);
// Create global completion
var completion = TaskExtensions.CreateGlobalCompletion(new[] { source.Completion }.Concat(processors.Select(x => x.Completion))
.Concat(new[] { errorHandler.Completion, output.Completion }),
cancellationSource);
return(new StartableBlock <TData>
{
Start = source.Start,
Output = output.Processor,
EstimatedOutputCount = source.EstimatedOutputCount,
Completion = completion
});
}
public async Task <PipelineExecutionResult> Execute <TData>(StartableBlock <TData> pipeline)
{
var pipelineExecutionResult = new PipelineExecutionResult();
// Ignore pipeline output
pipeline.Output.IgnoreOutput();
// Handle completion
var completionHandler = pipeline.Completion.ContinueWith(
(completion, executionResult) => FillExecutionResult(completion, (PipelineExecutionResult)executionResult),
pipelineExecutionResult);
// Execute
pipelineExecutionResult.StartTs = DateTime.UtcNow;
pipeline.Start();
return(await completionHandler);
}
public StartableBlock <DataBatch <Person> > Create(string targetFilePath, IProgress <PipelineProgress> progress, CancellationToken cancellation)
{
File.Create(targetFilePath).Dispose();
var builtSummary = new PeopleSummary();
// Create pipelines
var summaryFromDbPipeline = CreateSummaryFormDbPipeline(targetFilePath, progress, cancellation);
var writeEmptyLineBlock1 = new StartableBlock <object>(
() =>
{
if (!summaryFromDbPipeline.Completion.IsFaulted && !summaryFromDbPipeline.Completion.IsCanceled)
{
File.AppendAllLines(targetFilePath, new[] { string.Empty });
}
});
var peoplePipeline = CreatePeoplePipeline(targetFilePath, builtSummary, progress, cancellation);
var writeEmptyLineBlock2 = new StartableBlock <object>(
() =>
{
if (!peoplePipeline.Completion.IsFaulted && !peoplePipeline.Completion.IsCanceled)
{
File.AppendAllLines(targetFilePath, new[] { string.Empty });
}
});
var builtSummaryPipeline = CreateBuiltSummaryPipeline(targetFilePath, builtSummary, progress, cancellation);
// Link pipelines
summaryFromDbPipeline.ContinueWith(writeEmptyLineBlock1);
writeEmptyLineBlock1.ContinueWith(peoplePipeline);
peoplePipeline.ContinueWith(writeEmptyLineBlock2, false);
writeEmptyLineBlock2.ContinueWith(builtSummaryPipeline);
builtSummaryPipeline.Output.IgnoreOutput();
return(new StartableBlock <DataBatch <Person> >(
summaryFromDbPipeline.Start,
peoplePipeline.Output,
peoplePipeline.EstimatedOutputCount,
builtSummaryPipeline.Completion,
true));
}
