public void Execute(
BasicTaskStep step,
TaskStepExecutionContext context,
CancellationToken cancellation)
{
var events = context.EventsBag.TryGetEvents <TaskExecutionEvents>();
var sw = new Stopwatch();
var progress = new SynchronousProgress <float>(p => events?.Raise(x => x.OnStepProgressed(p, sw.Elapsed, step, context.Task)));
sw.Start();
try
{
step.Body(context.OutcomeSoFar, progress, cancellation);
}
catch (OperationCanceledException e)
{
throw;
}
catch (Exception e)
{
throw new TaskExecutionException(e, step);
}
sw.Stop();
progress.Report(1f);
cancellation.ThrowIfCancellationRequested();
}
public void Execute(
PipelineTaskStep <TItem> step,
TaskStepExecutionContext context,
IProgress <byte> progress,
CancellationToken cancellation)
{
var itemNumber = 0;
var events = context.EventsBag.TryGetEvents <PipelineExecutionEvents>();
var itemSw = new Stopwatch();
var blockSw = new Stopwatch();
var totalInputMaterializationDuration = TimeSpan.Zero;
var totalBlockDurations = step.Blocks.Select(x => x.Name).Distinct().ToDictionary(x => x, _ => TimeSpan.Zero);
using (var inputEnumerator = step.Input.Input.GetEnumerator())
{
while (true)
{
itemSw.Restart();
if (!inputEnumerator.MoveNext())
{
itemSw.Stop();
break;
}
itemNumber++;
var item = inputEnumerator.Current;
var materializationDuration = itemSw.Elapsed;
totalInputMaterializationDuration += materializationDuration;
events?.OnItemStarted(itemNumber, item, materializationDuration, step, context.Task);
foreach (var block in step.Blocks)
{
events?.OnBlockStarted(block, itemNumber, item, step, context.Task);
blockSw.Restart();
try
{
block.Body(item);
}
catch (Exception e)
{
throw new TaskExecutionException(e, step, block.GetExceptionData());
}
blockSw.Stop();
totalBlockDurations[block.Name] += blockSw.Elapsed;
events?.OnBlockEnded(block, itemNumber, item, step, context.Task, blockSw.Elapsed);
cancellation.ThrowIfCancellationRequested();
}
itemSw.Stop();
events?.OnItemEnded(itemNumber, item, step, context.Task, itemSw.Elapsed);
PipelineProcessingUtils.ReportProgress(itemNumber, step.Input.ExpectedItemsCount, progress);
}
}
events?.OnPipelineEnded(totalInputMaterializationDuration, totalBlockDurations, step, context.Task);
}
public void Execute(
PipelineTaskStep <TItem> step,
TaskStepExecutionContext context,
CancellationToken cancellation)
{
var items = step.Input.ItemsFactory();
var events = context.EventsBag.TryGetEvents <PipelineExecutionEvents>();
var dataflowExecutionContext = new DataflowExecutionContext
{
StepContext = context,
ExpectedItemsCount = items.ExpectedCount,
TaskEvents = context.EventsBag.TryGetEvents <TaskExecutionEvents>(),
Events = events,
TotalInputMaterializationDuration = TimeSpan.Zero,
TotalBlockDurations = new ConcurrentDictionary <string, TimeSpan>(
step.Blocks.Select(x => x.Name).Distinct().Select(
x => new KeyValuePair <string, TimeSpan>(x, TimeSpan.Zero)))
};
using (var inputEnumerator = items.Items.GetEnumerator())
{
var pipeline = _dataflowPipelineBuilder.Build(
inputEnumerator,
step,
dataflowExecutionContext,
cancellation);
try
{
pipeline.Execute().Wait();
}
catch (AggregateException e)
{
var flat = e.Flatten();
if (flat.InnerExceptions.Count == 1)
{
throw flat.InnerExceptions[0];
}
throw flat;
}
}
events?.Raise(x => x.OnPipelineEnded(dataflowExecutionContext.TotalInputMaterializationDuration, dataflowExecutionContext.TotalBlockDurations, step, context.Task));
}
private void ExecuteStep <TStep>(
TStep step,
TaskDefinition task,
TaskOutcome outcomeSoFar,
CancellationToken cancellation,
ExecutionEventsBag eventsBag)
where TStep : ITaskStep
{
var stepExecutor = _taskStepExecutorResolver.Resolve <TStep>();
var context = new TaskStepExecutionContext(
task,
eventsBag,
outcomeSoFar);
stepExecutor.Execute(step, context, cancellation);
}
public void Execute(
BasicTaskStep step,
TaskStepExecutionContext context,
IProgress <byte> progress,
CancellationToken cancellation)
{
try
{
step.Body();
}
catch (Exception e)
{
throw new TaskExecutionException(e, step);
}
progress.Report(100);
cancellation.ThrowIfCancellationRequested();
}
private void ExecuteStep <TStep>(
TStep step,
TaskDefinition task,
IProgress <TaskProgress> progress,
CancellationToken cancellation)
where TStep : ITaskStep
{
var stepExecutor = _taskStepExecutorResolver.Resolve <TStep>();
var context = new TaskStepExecutionContext
{
Task = task,
EventsBag = _executionEventsBag
};
var stepProgress = new SynchronousProgress <byte>(
x => progress.Report(new TaskProgress
{
StepName = step.Name,
ProgressPercentage = x
}));
stepExecutor.Execute(step, context, stepProgress, cancellation);
}
public void Execute(TTaskStep step, TaskStepExecutionContext context, CancellationToken cancellation) => throw Error;
public void Execute(TTaskStep step, TaskStepExecutionContext context, IProgress <byte> progress, CancellationToken cancellation) => throw Error;
public void Execute(
PipelineTaskStep <TItem> step,
TaskStepExecutionContext context,
CancellationToken cancellation)
{
var items = step.Input.ItemsFactory();
var itemNumber = 0;
var taskEvents = context.EventsBag.TryGetEvents <TaskExecutionEvents>();
var events = context.EventsBag.TryGetEvents <PipelineExecutionEvents>();
var itemSw = new Stopwatch();
var blockSw = new Stopwatch();
var totalInputMaterializationDuration = TimeSpan.Zero;
var totalBlockDurations = step.Blocks.Select(x => x.Name).Distinct().ToDictionary(x => x, _ => TimeSpan.Zero);
using (var inputEnumerator = items.Items.GetEnumerator())
{
while (true)
{
itemNumber++;
var itemStartTs = DateTimeOffset.UtcNow;
itemSw.Restart();
bool hasNextItem;
try
{
hasNextItem = inputEnumerator.MoveNext();
}
catch (Exception e)
{
throw new TaskExecutionException(e, step, step.GetInputExceptionData(itemNumber));
}
if (!hasNextItem)
{
itemSw.Stop();
break;
}
var item = inputEnumerator.Current;
var materializationDuration = itemSw.Elapsed;
totalInputMaterializationDuration += materializationDuration;
events?.Raise(x => x.OnItemMaterialized(itemNumber, item, itemStartTs, materializationDuration, step, context.Task));
foreach (var block in step.Blocks)
{
events?.Raise(x => x.OnBlockStarted(block, itemNumber, item, step, context.Task));
blockSw.Restart();
try
{
block.Body(item);
}
catch (Exception e)
{
throw new TaskExecutionException(e, step, block.GetExceptionData(itemNumber));
}
blockSw.Stop();
totalBlockDurations[block.Name] += blockSw.Elapsed;
events?.Raise(x => x.OnBlockEnded(block, itemNumber, item, step, context.Task, blockSw.Elapsed));
cancellation.ThrowIfCancellationRequested();
}
itemSw.Stop();
events?.Raise(x => x.OnItemEnded(itemNumber, item, step, context.Task, itemSw.Elapsed));
taskEvents?.Raise(x => x.OnStepProgressed(itemNumber, items.ExpectedCount, itemSw.Elapsed, step, context.Task));
}
}
events?.Raise(x => x.OnPipelineEnded(totalInputMaterializationDuration, totalBlockDurations, step, context.Task));
}
public void Execute(
PipelineTaskStep <TItem> step,
TaskStepExecutionContext context,
IProgress <byte> progress,
CancellationToken cancellation)
{
var events = context.EventsBag.TryGetEvents <PipelineExecutionEvents>();
var dataflowExecutionContext = new DataflowExecutionContext
{
StepContext = context,
Events = events,
TotalBlockDurations = new ConcurrentDictionary <string, TimeSpan>(
step.Blocks.Select(x => x.Name).Distinct().Select(
x => new KeyValuePair <string, TimeSpan>(x, TimeSpan.Zero)))
};
var pipeline = _dataflowPipelineBuilder.Build(step, dataflowExecutionContext, progress, cancellation);
var itemNumber = 0;
var totalInputMaterializationDuration = TimeSpan.Zero;
using (var inputEnumerator = step.Input.Input.GetEnumerator())
{
while (true)
{
var sw = Stopwatch.StartNew();
if (!inputEnumerator.MoveNext())
{
sw.Stop();
break;
}
var pipelineItem = new PipelineItem <TItem>
{
Number = ++itemNumber,
Item = inputEnumerator.Current,
ProcessingStopwatch = sw
};
var materializationDuration = sw.Elapsed;
totalInputMaterializationDuration += materializationDuration;
events?.OnItemStarted(pipelineItem.Number, pipelineItem.Item, materializationDuration, step, context.Task);
pipeline.InputBlock.SendAsync(pipelineItem).Wait();
cancellation.ThrowIfCancellationRequested();
}
}
pipeline.InputBlock.Complete();
try
{
pipeline.Completion.Wait();
}
catch (AggregateException e)
{
var flat = e.Flatten();
if (flat.InnerExceptions.Count == 1)
{
throw flat.InnerExceptions[0];
}
throw flat;
}
events?.OnPipelineEnded(totalInputMaterializationDuration, dataflowExecutionContext.TotalBlockDurations, step, context.Task);
}
