private DataflowPipeline <TItem> BuildNotEmpty <TItem>(
PipelineTaskStep <TItem> step,
DataflowExecutionContext context,
IProgress <byte> progress,
CancellationToken cancellation)
{
var firstBlock = ToTransformBlock(step, 0, context, cancellation);
var previousBlock = firstBlock;
for (var i = 1; i <= step.Blocks.Count - 1; i++)
{
var currentBlock = ToTransformBlock(step, i, context, cancellation);
previousBlock.LinkTo(currentBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
previousBlock = currentBlock;
}
var progressBlock = CreateProgressBlock(step, context, progress, cancellation);
previousBlock.LinkTo(progressBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
return(new DataflowPipeline <TItem>
{
InputBlock = firstBlock,
Completion = progressBlock.Completion
});
}
private ActionBlock <PipelineItem <TItem> > BuildProgressBlock <TItem>(
PipelineTaskStep <TItem> step,
DataflowExecutionContext context,
CancellationToken cancellation)
{
return(new ActionBlock <PipelineItem <TItem> >(
x =>
{
if (x == null)
{
return;
}
x.ProcessingStopwatch.Stop();
context.Events?.Raise(e => e.OnItemEnded(x.Number, x.Item, step, context.StepContext.Task, x.ProcessingStopwatch.Elapsed));
context.TaskEvents?.Raise(e => e.OnStepProgressed(x.Number, context.ExpectedItemsCount, x.ProcessingStopwatch.Elapsed, step, context.StepContext.Task));
},
new ExecutionDataflowBlockOptions
{
NameFormat = "Progress",
BoundedCapacity = 1,
EnsureOrdered = true,
CancellationToken = cancellation
}));
}
public DataflowPipeline <TItem> Build <TItem>(
PipelineTaskStep <TItem> step,
DataflowExecutionContext context,
IProgress <byte> progress,
CancellationToken cancellation)
{
return(step.Blocks.Count != 0
? BuildNotEmpty(step, context, progress, cancellation)
: BuildEmpty(step, context, progress, cancellation));
}
private TransformBlock <int, PipelineItem <TItem> > BuildMaterializeAndPostNextBlock <TItem>(
IEnumerator <TItem> inputEnumerator,
ITargetBlock <int> inputBlock,
PipelineTaskStep <TItem> step,
DataflowExecutionContext context,
CancellationToken cancellation)
{
return(new TransformBlock <int, PipelineItem <TItem> >(
i =>
{
var itemStartTs = DateTimeOffset.UtcNow;
var sw = Stopwatch.StartNew();
bool hasNextItem;
try
{
hasNextItem = inputEnumerator.MoveNext();
}
catch (Exception e)
{
throw new TaskExecutionException(e, step, step.GetInputExceptionData(i));
}
if (hasNextItem)
{
var pipelineItem = new PipelineItem <TItem>
{
Number = i,
Item = inputEnumerator.Current,
ProcessingStopwatch = sw
};
var materializationDuration = sw.Elapsed;
context.TotalInputMaterializationDuration += materializationDuration;
context.Events?.Raise(x => x.OnItemMaterialized(pipelineItem.Number, pipelineItem.Item, itemStartTs, materializationDuration, step, context.StepContext.Task));
PostNextInput(inputBlock, pipelineItem.Number + 1);
return pipelineItem;
}
else
{
sw.Stop();
inputBlock.Complete();
return null;
}
},
new ExecutionDataflowBlockOptions
{
NameFormat = "MaterializeAndPostNextBlock",
BoundedCapacity = 1,
EnsureOrdered = true,
CancellationToken = cancellation
}));
}
private DataflowPipeline <TItem> BuildEmpty <TItem>(
PipelineTaskStep <TItem> step,
DataflowExecutionContext context,
IProgress <byte> progress,
CancellationToken cancellation)
{
var progressBlock = CreateProgressBlock(step, context, progress, cancellation);
return(new DataflowPipeline <TItem>
{
InputBlock = progressBlock,
Completion = progressBlock.Completion
});
}
private TransformBlock <PipelineItem <TItem>, PipelineItem <TItem> > BuildProcessingBlock <TItem>(
PipelineBlock <TItem> block,
PipelineTaskStep <TItem> step,
DataflowExecutionContext context,
CancellationToken cancellation)
{
return(new TransformBlock <PipelineItem <TItem>, PipelineItem <TItem> >(
x =>
{
if (x == null)
{
return x;
}
context.Events?.Raise(e => e.OnBlockStarted(block, x.Number, x.Item, step, context.StepContext.Task));
var sw = Stopwatch.StartNew();
try
{
block.Body(x.Item);
}
catch (Exception e)
{
throw new TaskExecutionException(e, step, block.GetExceptionData(x.Number));
}
sw.Stop();
context.TotalBlockDurations.AddOrUpdate(block.Name, sw.Elapsed, (_, duration) => duration + sw.Elapsed);
context.Events?.Raise(e => e.OnBlockEnded(block, x.Number, x.Item, step, context.StepContext.Task, sw.Elapsed));
return x;
},
new ExecutionDataflowBlockOptions
{
NameFormat = block.Name,
MaxDegreeOfParallelism = block.MaxDegreeOfParallelism,
BoundedCapacity = block.MaxDegreeOfParallelism,
EnsureOrdered = true,
CancellationToken = cancellation
}));
}
public DataflowPipeline Build <TItem>(
IEnumerator <TItem> inputEnumerator,
PipelineTaskStep <TItem> step,
DataflowExecutionContext context,
CancellationToken cancellation)
{
var inputBlock = BuildInputBlock(cancellation);
var materializeAndPostNextBlock = BuildMaterializeAndPostNextBlock(inputEnumerator, inputBlock, step, context, cancellation);
inputBlock.LinkTo(materializeAndPostNextBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
ISourceBlock <PipelineItem <TItem> > previousBlock = materializeAndPostNextBlock;
foreach (var block in step.Blocks)
{
var currentBlock = BuildProcessingBlock(block, step, context, cancellation);
previousBlock.LinkTo(currentBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
previousBlock = currentBlock;
}
var progressBlock = BuildProgressBlock(step, context, cancellation);
previousBlock.LinkTo(progressBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
return(new DataflowPipeline
{
Execute = () =>
{
PostNextInput(inputBlock, 1);
return progressBlock.Completion;
}
});
}
private ActionBlock <PipelineItem <TItem> > CreateProgressBlock <TItem>(
PipelineTaskStep <TItem> step,
DataflowExecutionContext context,
IProgress <byte> progress,
CancellationToken cancellation)
{
return(new ActionBlock <PipelineItem <TItem> >(
x =>
{
x.ProcessingStopwatch.Stop();
context.Events?.OnItemEnded(x.Number, x.Item, step, context.StepContext.Task, x.ProcessingStopwatch.Elapsed);
PipelineProcessingUtils.ReportProgress(x.Number, step.Input.ExpectedItemsCount, progress);
},
new ExecutionDataflowBlockOptions
{
NameFormat = "Progress",
BoundedCapacity = 1,
EnsureOrdered = true,
CancellationToken = cancellation
}));
}
