public StartableBlock <TData> Create <TData>(StartableBlock <TData> source, ProcessingBlock <TData>[] processors, CancellationTokenSource cancellationSource) { // The pipeline looks like this: // source -> processor1 -> processor2 -> processor3 (output) // Link blocks source.Output.LinkWithCompletion(processors[0].Processor); for (var i = 0; i < processors.Length - 1; i++) { processors[i].Processor.LinkWithCompletion(processors[i + 1].Processor); } // Create global completion var completion = TaskExtensions.CreateGlobalCompletion(new[] { source.Completion }.Concat(processors.Select(x => x.Completion)), cancellationSource); return(new StartableBlock <TData> { Start = source.Start, Output = processors.Last().Processor, EstimatedOutputCount = source.EstimatedOutputCount, Completion = completion }); }
public StartableBlock <TData> Create <TData>(StartableBlock <TData> source, ProcessingBlock <TData>[] processors, ProcessingBlock <TData> errorHandler, ProcessingBlock <TData> output, Predicate <TData> validDataPredicate, CancellationTokenSource cancellationSource) { // The pipeline looks like this: // source -> processor1 -> processor2 -> output // \ | | ^ // \ v v / // \--> errorHandler ----------/ // Link blocks source.Output.LinkWithCompletion(processors[0].Processor, validDataPredicate); source.Output.LinkTo(errorHandler.Processor, x => !validDataPredicate(x)); for (var i = 0; i < processors.Length - 1; i++) { processors[i].Processor.LinkWithCompletion(processors[i + 1].Processor, validDataPredicate); processors[i].Processor.LinkTo(errorHandler.Processor, x => !validDataPredicate(x)); } var lastProcessor = processors.Last(); lastProcessor.Processor.LinkTo(output.Processor, validDataPredicate); lastProcessor.Processor.LinkTo(errorHandler.Processor, x => !validDataPredicate(x)); errorHandler.Processor.LinkTo(output.Processor); // Propagate completions of multiple inputs errorHandler.Processor.DeriveCompletionOrFaultFrom(new[] { source.Output }.Concat(processors.Select(x => x.Processor))); output.Processor.DeriveCompletionOrFaultFrom(lastProcessor.Processor, errorHandler.Processor); // Create global completion var completion = TaskExtensions.CreateGlobalCompletion(new[] { source.Completion }.Concat(processors.Select(x => x.Completion)) .Concat(new[] { errorHandler.Completion, output.Completion }), cancellationSource); return(new StartableBlock <TData> { Start = source.Start, Output = output.Processor, EstimatedOutputCount = source.EstimatedOutputCount, Completion = completion }); }
public async Task <PipelineExecutionResult> Execute <TData>(StartableBlock <TData> pipeline) { var pipelineExecutionResult = new PipelineExecutionResult(); // Ignore pipeline output pipeline.Output.IgnoreOutput(); // Handle completion var completionHandler = pipeline.Completion.ContinueWith( (completion, executionResult) => FillExecutionResult(completion, (PipelineExecutionResult)executionResult), pipelineExecutionResult); // Execute pipelineExecutionResult.StartTs = DateTime.UtcNow; pipeline.Start(); return(await completionHandler); }
public StartableBlock <DataBatch <Person> > Create(string targetFilePath, IProgress <PipelineProgress> progress, CancellationToken cancellation) { File.Create(targetFilePath).Dispose(); var builtSummary = new PeopleSummary(); // Create pipelines var summaryFromDbPipeline = CreateSummaryFormDbPipeline(targetFilePath, progress, cancellation); var writeEmptyLineBlock1 = new StartableBlock <object>( () => { if (!summaryFromDbPipeline.Completion.IsFaulted && !summaryFromDbPipeline.Completion.IsCanceled) { File.AppendAllLines(targetFilePath, new[] { string.Empty }); } }); var peoplePipeline = CreatePeoplePipeline(targetFilePath, builtSummary, progress, cancellation); var writeEmptyLineBlock2 = new StartableBlock <object>( () => { if (!peoplePipeline.Completion.IsFaulted && !peoplePipeline.Completion.IsCanceled) { File.AppendAllLines(targetFilePath, new[] { string.Empty }); } }); var builtSummaryPipeline = CreateBuiltSummaryPipeline(targetFilePath, builtSummary, progress, cancellation); // Link pipelines summaryFromDbPipeline.ContinueWith(writeEmptyLineBlock1); writeEmptyLineBlock1.ContinueWith(peoplePipeline); peoplePipeline.ContinueWith(writeEmptyLineBlock2, false); writeEmptyLineBlock2.ContinueWith(builtSummaryPipeline); builtSummaryPipeline.Output.IgnoreOutput(); return(new StartableBlock <DataBatch <Person> >( summaryFromDbPipeline.Start, peoplePipeline.Output, peoplePipeline.EstimatedOutputCount, builtSummaryPipeline.Completion, true)); }