public void Send(StepData stepData, IPipelineStep pipelineStep)
{
if (pipelineStep == _getTaskStep)
{
_initializingWebClientStep.SetData(stepData);
_gettingEntityStep.SetData(stepData);
}
else if (pipelineStep == _gettingEntityStep)
{
_downloadStep.SetData(stepData);
}
else if (pipelineStep == _downloadStep)
{
_creatorStep.SetData(stepData);
}
else if (pipelineStep == _creatorStep)
{
_encodeStep.SetData(stepData);
}
else if (pipelineStep == _encodeStep)
{
_uploadStep.SetData(stepData);
}
else if (pipelineStep == _uploadStep)
{
_finishStep.SetData(stepData);
}
}
/// <summary>
/// Executes the pipeline with an initial input.
/// </summary>
/// <param name="inInput">The input value.</param>
/// <returns>Returns the task which being run and holds the result of the pipeline.</returns>
public Task <TPipeOut> Execute(TPipeIn inInput)
{
// Before execute starts, check if the last step outputs the same type as the pipeline should output.
object last = PipelineSteps[PipelineSteps.Count - 1];
Type t = last.GetType();
if (t.IsGenericType)
{
Type returnType = t.GenericTypeArguments.Last();
if (returnType != typeof(TPipeOut))
{
throw new Exception("Last step in the pipeline does not have the correct output type\nThe last step outputs a " + returnType.Name + ", while the pipeline should output a " + typeof(TPipeOut).Name + " type");
}
}
IPipelineStep <TPipeIn> first = PipelineSteps[0] as IPipelineStep <TPipeIn>;
TaskCompletionSource <TPipeOut> task = new TaskCompletionSource <TPipeOut>();
first.Buffer.Add(new Item <TPipeIn>()
{
Input = inInput,
TaskCompletionSource = task
});
return(task.Task);
}
public GenericBCPipelineStep <TStepIn, TStepOut> GenerateStep <TStepIn, TStepOut>()
{
var pipelineStep = new GenericBCPipelineStep <TStepIn, TStepOut>();
var stepIndex = _pipelineSteps.Count;
Task.Run(() =>
{
IPipelineStep <TStepOut> nextPipelineStep = default;
foreach (var input in pipelineStep.Buffer.GetConsumingEnumerable())
{
bool isLastStep = stepIndex == _pipelineSteps.Count - 1;
var outPut = pipelineStep.StepAction(input);
if (isLastStep)
{
Finished?.Invoke((TPipeOut)(object)outPut);
}
else
{
nextPipelineStep ??= _pipelineSteps[stepIndex + 1] as IPipelineStep <TStepOut>;
nextPipelineStep.Buffer.Add(outPut);
}
}
});
_pipelineSteps.Add(pipelineStep);
return(pipelineStep);
}
public GenericBCPipelineStep <TStepIn, TStepOut> GenerateStep <TStepIn, TStepOut>()
{
var pipelineStep = new GenericBCPipelineStep <TStepIn, TStepOut>();
var stepIndex = _pipelineSteps.Count;
Task.Run(() =>
{
IPipelineStep <TStepOut> nextPipelineStep = null;
foreach (var input in pipelineStep.Buffer.GetConsumingEnumerable())
{
bool isLastStep = stepIndex == _pipelineSteps.Count - 1;
var output = pipelineStep.StepAction(input);
if (isLastStep)
{
// This is dangerous as the invocation is added to the last step
// Alternatively, you can utilize BeginInvoke like here: https://stackoverflow.com/a/16336361/1229063
Finished?.Invoke((TPipeOut)(object)output);
}
else
{
nextPipelineStep = nextPipelineStep // no need to evaluate more than once
?? (isLastStep ? null : _pipelineSteps[stepIndex + 1] as IPipelineStep <TStepOut>);
nextPipelineStep.Buffer.Add(output);
}
}
});
_pipelineSteps.Add(pipelineStep);
return(pipelineStep);
}
/// <summary>
/// Add a step to process the input and transform it into an output
/// </summary>
/// <param name="input">Input</param>
/// <typeparam name="TInput">Type of the input</typeparam>
/// <param name="step">Pipeline step</param>
/// <typeparam name="TOutput">Type of the output</typeparam>
/// <param name="arg1">Additional argument passed to the process</param>
/// <typeparam name="TArg1">Type of the first argument</typeparam>
/// <param name="arg2">Additional argument passed to the process</param>
/// <typeparam name="TArg1">Type of the second argument</typeparam>
/// <param name="arg3">Additional argument passed to the process</param>
/// <typeparam name="TArg3">Type of the third argument</typeparam>
/// <returns>Returns the output of processing the input</returns>
public static TOutput AddStep <TInput, TArg1, TArg2, TArg3, TOutput>(
this TInput input,
IPipelineStep <TInput, TArg1, TArg2, TArg3, TOutput> step,
TArg1 arg1, TArg2 arg2, TArg3 arg3)
{
return(step.Process(input, arg1, arg2, arg3));
}
private async Task ExecutePipeLineStep(IPipelineStep pipelineStep, PropertyBag propertyBag)
{
try
{
var sw = Stopwatch.StartNew();
this.m_Logger.Debug("Executing pipeline step {0}", pipelineStep.GetType().Name);
if (pipelineStep is IPipelineStepWithTimeout stepWithTimeout)
{
this.m_Logger.Debug("Running pipeline step {0} with timeout {1}",
pipelineStep.GetType().Name, stepWithTimeout.ProcessorTimeout);
this.m_TaskRunner.RunSync(async cancelArgs =>
{
if (!cancelArgs.Cancel)
{
await pipelineStep.ProcessAsync(this, propertyBag);
}
}, stepWithTimeout.ProcessorTimeout);
}
else
{
await pipelineStep.ProcessAsync(this, propertyBag);
}
this.m_Logger.Debug("Executed pipeline step {0} in {1}", pipelineStep.GetType().Name, sw.Elapsed);
}
catch (Exception ex)
{
OnProcessorException(propertyBag, ex);
}
}
public ActionPipelineStep(string name, IPipelineStep <TOutput> input, Action <TOutput> action)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
Input = input ?? throw new ArgumentNullException(nameof(input));
Action = action ?? throw new ArgumentNullException(nameof(action));
Signaler.RegisterMultiple(this, Input);
}
public static IEnumerable <OUTPUT> Step <INPUT, OUTPUT>(this IEnumerable <INPUT> input,
IPipelineStep <INPUT, OUTPUT> step)
{
LoopStep <INPUT, OUTPUT> loopDecorator = new LoopStep <INPUT, OUTPUT>(step);
return(loopDecorator.Process(input));
}
public CachePipelineStep(IPipelineStep <TItem> step)
{
Step = step ?? throw new ArgumentNullException(nameof(step));
Name = $"Cache for step '{step.Name}'";
Signaler = Step.Signaler;
Signaler.RegisterDependency(Step, this);
}
public PausePipelineStep(string name, IPipelineStep <TOutput> input)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
Input = input ?? throw new ArgumentNullException(nameof(input));
Signaler = Input.Signaler;
Signaler.RegisterDependency(Input, this);
}
public static INPUTOUTPUT Step <INPUTOUTPUT>(this INPUTOUTPUT input, IPipelineStep <INPUTOUTPUT> step)
{
if (step == null)
{
return(input);
}
return(step.Process(input));
}
public static void RegisterMultiple(this ISignaler signaler, IPipelineStep downstream, params IPipelineStep[] upstreams)
{
AssertSameSignaler(upstreams.Select(u => u.Signaler).ToArray());
foreach (var up in upstreams)
{
signaler.RegisterDependency(up, downstream);
}
}
public UnaryOperationPipelineStep(string name, IPipelineStep <TInput> input, Func <TInput, TOutput> operation)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
Input = input ?? throw new ArgumentNullException(nameof(input));
Operation = operation ?? throw new ArgumentNullException(nameof(operation));
Signaler = Input.Signaler;
Signaler.RegisterDependency(Input, this);
}
public DemultiplexingPipelineStep(string name, IPipelineStep <TOutput> input, IPipelineStep <TSelect> select)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
Input = input ?? throw new ArgumentNullException(nameof(input));
Select = select ?? throw new ArgumentNullException(nameof(select));
Signaler = Input.Signaler;
Signaler.RegisterMultiple(this, Input, Select);
Outputs = new Dictionary <TSelect, IPipelineStep <TOutput> >();
}
public BinaryOperationPipelineStep(string name, IPipelineStep <TInput1> input1, IPipelineStep <TInput2> input2, Func <TInput1, TInput2, TOutput> operation)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
Input1 = input1 ?? throw new ArgumentNullException(nameof(input1));
Input2 = input2 ?? throw new ArgumentNullException(nameof(input2));
Operation = operation ?? throw new ArgumentNullException(nameof(operation));
Signaler = Input1.Signaler;
Signaler.RegisterMultiple(this, Input1, Input2);
}
public PassThroughPipelineStep(string name, IPipelineStep <TOutput> input, bool invalidateOnInvalidate, bool dependOnInput)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
Input = input ?? throw new ArgumentNullException(nameof(input));
InvalidateOnInvalidate = invalidateOnInvalidate;
if (dependOnInput)
{
Signaler.RegisterDependency(Input, this);
}
}
public GooeyCrawler AddPipelineStep(IPipelineStep step)
{
step.ImportSiteHash = siteHash;
if (step is ICheckedPipelineStep)
((ICheckedPipelineStep)step).PerformErrorCheck();
this.pipelineSteps.Add(step);
return this;
}
internal void AddStep(IPipelineStep step)
{
Steps.Add((PipelineStep)step);
bool isComplete = step.OutputQNames.All(s => PipelineQueues.ContainsKey(s));
isComplete &= PipelineQueues.ContainsKey(step.InputQName);
if (isComplete)
{
((PipelineStep)step).UpdateStatus(StepStatus.Complete);
}
}
public PipelineStep <U> Then <U>(Func <T, Task <U> > nextStep)
{
var newStep = new PipelineStep <U>(async() =>
{
var previousStepValue = await GetLatestAsync();
return(await nextStep(previousStepValue));
});
_nextStep = newStep;
return(newStep);
}
public void AddInput(TSelect select, IPipelineStep <TOutput> input)
{
SignalerHelper.AssertSameSignaler(Signaler, input.Signaler);
var pauseStep = new PausePipelineStep <TOutput>($"Multiplexer '{Name}' - Branch {select}", input);
pauseStep.Pause();
Inputs.Add(select, pauseStep);
Signaler.RegisterDependency(input, this);
}
public static OUTPUT Step <INPUT, OUTPUT>(this INPUT input, IPipelineStep <INPUT, OUTPUT> step, Action <INPUT> inputEvent = null, Action <OUTPUT> outputEvent = null)
{
if (inputEvent != null || outputEvent != null)
{
var eventDecorator = new EventStep <INPUT, OUTPUT>(step);
eventDecorator.OnInput += inputEvent;
eventDecorator.OnOutput += outputEvent;
return(eventDecorator.Process(input));
}
return(step.Process(input));
}
public IPipelineStep <T> Register(IPipelineStep <T> step)
{
if (_nextStep == null)
{
_nextStep = step;
}
else
{
_nextStep.Register(step);
}
return(this);
}
public IPipelineStep <T> Register(IPipelineStep <T> step)
{
if (_rootStep == null)
{
_rootStep = step;
}
else
{
_rootStep.Register(step);
}
return(_rootStep);
}
/// <summary>
/// Add a step to process the input and transform it into an output, allowing
/// events to happen before and after processing
/// </summary>
/// <typeparam name="TInput">Type of the input</typeparam>
/// <typeparam name="TOutput">Type of the output</typeparam>
/// <param name="input">Input</param>
/// <param name="step">Pipeline step</param>
/// <param name="inputEvent">An action to execute on the input before processing it</param>
/// <param name="outputEvent">An action to execute on the output after processing</param>
/// <returns>Returns the output of processing the input</returns>
public static TOutput AddStep <TInput, TOutput>(
this TInput input,
IPipelineStep <TInput, TOutput> step,
Action <TInput> inputEvent = null,
Action <TOutput> outputEvent = null)
{
var eventDecorator = new PipelineStep <TInput, TOutput>(step);
eventDecorator.OnInput += inputEvent;
eventDecorator.OnOutput += outputEvent;
return(eventDecorator.Process(input));
}
private void WriteInformationForStep(string stepName, IPipelineStep <T> step, T context)
{
var stopwatch = new Stopwatch();
var logger = _kernel.Resolve <ILogger>();
logger.WriteBanner();
logger.InfoFormat("Running step '{0}'...", stepName);
stopwatch.Start();
step.Execute(context);
stopwatch.Stop();
logger.InfoFormat("Step '{0}' completed.", stepName);
logger.InfoFormat("Execution Time: ({0}).", stopwatch.Elapsed.ToString());
logger.WriteBanner();
}
public PipelineStepStatus GetStatusPreviousStep(IPipelineStep step)
{
var steps = Steps.Where(s => s.Number == step.Number - 1);
var stepsCount = steps.Count();
if (stepsCount == 0)
{
return(PipelineStepStatus.Complited);
}
var complitedSteps = Steps.Count(s => s.Number == step.Number - 1 && s.Status == PipelineStepStatus.Complited);
return(complitedSteps == stepsCount ? PipelineStepStatus.Complited : PipelineStepStatus.Running);
}
public GenericPipelineStep <TStepIn, TStepOut> GenerateStep <TStepIn, TStepOut>()
{
var pipelineStep = new GenericPipelineStep <TStepIn, TStepOut>();
var stepIndex = _pipelineSteps.Count;
Task.Run(() =>
{
IPipelineStep <TStepOut> nextPipelineStep = null;
foreach (var input in pipelineStep.Buffer.GetConsumingEnumerable())
{
bool isLastStep = stepIndex == _pipelineSteps.Count - 1;
TStepOut output;
try
{
output = pipelineStep.StepAction(input.Input);
}
catch (Exception e)
{
input.TaskCompletionSource.SetException(e);
continue;
}
if (isLastStep)
{
input.TaskCompletionSource.SetResult((TPipeOut)(object)output);
}
else
{
nextPipelineStep = nextPipelineStep ?? (isLastStep ? null : _pipelineSteps[stepIndex + 1] as IPipelineStep <TStepOut>);
nextPipelineStep.Buffer.Add(new Item <TStepOut>()
{
Input = output, TaskCompletionSource = input.TaskCompletionSource
});
}
}
});
_pipelineSteps.Add(pipelineStep);
return(pipelineStep);
}
private void ExecutePipeLineStep(IPipelineStep pipelineStep, PropertyBag propertyBag)
{
try
{
if (pipelineStep is IPipelineStepWithTimeout)
{
IPipelineStepWithTimeout stepWithTimeout = (IPipelineStepWithTimeout)pipelineStep;
m_Logger.Debug("Running pipeline step {0} with timeout {1}",
pipelineStep.GetType().Name, stepWithTimeout.ProcessorTimeout);
m_TaskRunner.RunSync(() => pipelineStep.Process(this, propertyBag), stepWithTimeout.ProcessorTimeout);
}
else
{
m_Logger.Debug("Running pipeline step {0}", pipelineStep.GetType().Name);
pipelineStep.Process(this, propertyBag);
}
}
catch (Exception ex)
{
OnProcessorException(propertyBag, ex);
}
}
/// <summary>
/// Processes the step.
/// </summary>
/// <param name="inStepIndex">Which step is being processed in the pipeline.</param>
/// <param name="inPipelineStep">The pipeline step instance.</param>
private void StartStep <TStepIn, TStepOut>(int inStepIndex, PipelineStep <TStepIn, TStepOut> inPipelineStep)
{
IPipelineStep <TStepOut> nextPipelineStep = null;
// Iterate over all inputs in the buffer for this step
foreach (var input in inPipelineStep.Buffer.GetConsumingEnumerable())
{
TStepOut output;
// Try to perform the step.
try
{
output = inPipelineStep.StepAction(input.Input);
}
catch (Exception e)
{
input.TaskCompletionSource.SetException(e);
continue;
}
FinishStep(input, output, inStepIndex, ref nextPipelineStep);
}
}
/// <summary>
/// Processes the optional step.
/// </summary>
/// <param name="inStepIndex">The index of this step in the pipeline.</param>
/// <param name="inOptionalStep">The optional step being processed.</param>
/// <typeparam name="TOptionalInOut">The In and Output type of the optional step.</typeparam>
private void StartOptionalStep <TOptionalInOut>(int inStepIndex, OptionalPipelineStep <TOptionalInOut, TOptionalInOut> inOptionalStep)
{
IPipelineStep <TOptionalInOut> nextPipelineStep = null;
foreach (var input in inOptionalStep.Buffer.GetConsumingEnumerable())
{
TOptionalInOut output = default(TOptionalInOut);
try
{
if (inOptionalStep.Choice(input.Input))
{
output = inOptionalStep.StepAction(input.Input);
}
}
catch (Exception e)
{
input.TaskCompletionSource.SetException(e);
continue;
}
FinishStep(input, output, inStepIndex, ref nextPipelineStep);
}
}
public void AddUploadStep(IPipelineStep pipelineStep)
{
_uploadStep = pipelineStep;
}
public NotFoundPipelineStepException(IPipelineStep step)
: this($"No step with the given key ({step.Key}) is found", step.Key)
{
}
public void AddGettingEntityStep(IPipelineStep pipelineStep)
{
_gettingEntityStep = pipelineStep;
}
public void AddGetTaskStep(IPipelineStep pipelineStep)
{
_getTaskStep = pipelineStep;
}
private void ExecutePipeLineStep(IPipelineStep pipelineStep, PropertyBag propertyBag)
{
try
{
Stopwatch sw = Stopwatch.StartNew();
m_Logger.Debug("Executing pipeline step {0}", pipelineStep.GetType().Name);
if (pipelineStep is IPipelineStepWithTimeout)
{
IPipelineStepWithTimeout stepWithTimeout = (IPipelineStepWithTimeout) pipelineStep;
m_Logger.Debug("Running pipeline step {0} with timeout {1}",
pipelineStep.GetType().Name, stepWithTimeout.ProcessorTimeout);
m_TaskRunner.RunSync(cancelArgs =>
{
if (!cancelArgs.Cancel)
{
pipelineStep.Process(this, propertyBag);
}
}, stepWithTimeout.ProcessorTimeout);
}
else
{
pipelineStep.Process(this, propertyBag);
}
m_Logger.Debug("Executed pipeline step {0} in {1}", pipelineStep.GetType().Name, sw.Elapsed);
}
catch (Exception ex)
{
OnProcessorException(propertyBag, ex);
}
}
public void AddCreatorStep(IPipelineStep pipelineStep)
{
_creatorStep = pipelineStep;
}
public void AddInitializingWebClientStep(IPipelineStep pipelineStep)
{
_initializingWebClientStep = pipelineStep;
}
public void AddEncodeStep(IPipelineStep pipelineStep)
{
_encodeStep = pipelineStep;
}
private void ExecutePipeLineStep(IPipelineStep pipelineStep, PropertyBag propertyBag)
{
try
{
if (pipelineStep is IPipelineStepWithTimeout)
{
IPipelineStepWithTimeout stepWithTimeout = (IPipelineStepWithTimeout) pipelineStep;
m_Logger.Debug("Running pipeline step {0} with timeout {1}",
pipelineStep.GetType().Name, stepWithTimeout.ProcessorTimeout);
m_TaskRunner.RunSync(() => pipelineStep.Process(this, propertyBag), stepWithTimeout.ProcessorTimeout);
}
else
{
m_Logger.Debug("Running pipeline step {0}", pipelineStep.GetType().Name);
pipelineStep.Process(this, propertyBag);
}
}
catch (Exception ex)
{
OnProcessorException(propertyBag, ex);
}
}
public void AddFinishStep(IPipelineStep pipelineStep)
{
_finishStep = pipelineStep;
}
public void AddDownloadStep(IPipelineStep pipelineStep)
{
_downloadStep = pipelineStep;
}
public void Add(IPipelineStep pipelineStep)
{
_list.Add(pipelineStep);
}
