public void Send(StepData stepData, IPipelineStep pipelineStep) { if (pipelineStep == _getTaskStep) { _initializingWebClientStep.SetData(stepData); _gettingEntityStep.SetData(stepData); } else if (pipelineStep == _gettingEntityStep) { _downloadStep.SetData(stepData); } else if (pipelineStep == _downloadStep) { _creatorStep.SetData(stepData); } else if (pipelineStep == _creatorStep) { _encodeStep.SetData(stepData); } else if (pipelineStep == _encodeStep) { _uploadStep.SetData(stepData); } else if (pipelineStep == _uploadStep) { _finishStep.SetData(stepData); } }
/// <summary> /// Executes the pipeline with an initial input. /// </summary> /// <param name="inInput">The input value.</param> /// <returns>Returns the task which being run and holds the result of the pipeline.</returns> public Task <TPipeOut> Execute(TPipeIn inInput) { // Before execute starts, check if the last step outputs the same type as the pipeline should output. object last = PipelineSteps[PipelineSteps.Count - 1]; Type t = last.GetType(); if (t.IsGenericType) { Type returnType = t.GenericTypeArguments.Last(); if (returnType != typeof(TPipeOut)) { throw new Exception("Last step in the pipeline does not have the correct output type\nThe last step outputs a " + returnType.Name + ", while the pipeline should output a " + typeof(TPipeOut).Name + " type"); } } IPipelineStep <TPipeIn> first = PipelineSteps[0] as IPipelineStep <TPipeIn>; TaskCompletionSource <TPipeOut> task = new TaskCompletionSource <TPipeOut>(); first.Buffer.Add(new Item <TPipeIn>() { Input = inInput, TaskCompletionSource = task }); return(task.Task); }
public GenericBCPipelineStep <TStepIn, TStepOut> GenerateStep <TStepIn, TStepOut>() { var pipelineStep = new GenericBCPipelineStep <TStepIn, TStepOut>(); var stepIndex = _pipelineSteps.Count; Task.Run(() => { IPipelineStep <TStepOut> nextPipelineStep = default; foreach (var input in pipelineStep.Buffer.GetConsumingEnumerable()) { bool isLastStep = stepIndex == _pipelineSteps.Count - 1; var outPut = pipelineStep.StepAction(input); if (isLastStep) { Finished?.Invoke((TPipeOut)(object)outPut); } else { nextPipelineStep ??= _pipelineSteps[stepIndex + 1] as IPipelineStep <TStepOut>; nextPipelineStep.Buffer.Add(outPut); } } }); _pipelineSteps.Add(pipelineStep); return(pipelineStep); }
public GenericBCPipelineStep <TStepIn, TStepOut> GenerateStep <TStepIn, TStepOut>() { var pipelineStep = new GenericBCPipelineStep <TStepIn, TStepOut>(); var stepIndex = _pipelineSteps.Count; Task.Run(() => { IPipelineStep <TStepOut> nextPipelineStep = null; foreach (var input in pipelineStep.Buffer.GetConsumingEnumerable()) { bool isLastStep = stepIndex == _pipelineSteps.Count - 1; var output = pipelineStep.StepAction(input); if (isLastStep) { // This is dangerous as the invocation is added to the last step // Alternatively, you can utilize BeginInvoke like here: https://stackoverflow.com/a/16336361/1229063 Finished?.Invoke((TPipeOut)(object)output); } else { nextPipelineStep = nextPipelineStep // no need to evaluate more than once ?? (isLastStep ? null : _pipelineSteps[stepIndex + 1] as IPipelineStep <TStepOut>); nextPipelineStep.Buffer.Add(output); } } }); _pipelineSteps.Add(pipelineStep); return(pipelineStep); }
/// <summary> /// Add a step to process the input and transform it into an output /// </summary> /// <param name="input">Input</param> /// <typeparam name="TInput">Type of the input</typeparam> /// <param name="step">Pipeline step</param> /// <typeparam name="TOutput">Type of the output</typeparam> /// <param name="arg1">Additional argument passed to the process</param> /// <typeparam name="TArg1">Type of the first argument</typeparam> /// <param name="arg2">Additional argument passed to the process</param> /// <typeparam name="TArg1">Type of the second argument</typeparam> /// <param name="arg3">Additional argument passed to the process</param> /// <typeparam name="TArg3">Type of the third argument</typeparam> /// <returns>Returns the output of processing the input</returns> public static TOutput AddStep <TInput, TArg1, TArg2, TArg3, TOutput>( this TInput input, IPipelineStep <TInput, TArg1, TArg2, TArg3, TOutput> step, TArg1 arg1, TArg2 arg2, TArg3 arg3) { return(step.Process(input, arg1, arg2, arg3)); }
private async Task ExecutePipeLineStep(IPipelineStep pipelineStep, PropertyBag propertyBag) { try { var sw = Stopwatch.StartNew(); this.m_Logger.Debug("Executing pipeline step {0}", pipelineStep.GetType().Name); if (pipelineStep is IPipelineStepWithTimeout stepWithTimeout) { this.m_Logger.Debug("Running pipeline step {0} with timeout {1}", pipelineStep.GetType().Name, stepWithTimeout.ProcessorTimeout); this.m_TaskRunner.RunSync(async cancelArgs => { if (!cancelArgs.Cancel) { await pipelineStep.ProcessAsync(this, propertyBag); } }, stepWithTimeout.ProcessorTimeout); } else { await pipelineStep.ProcessAsync(this, propertyBag); } this.m_Logger.Debug("Executed pipeline step {0} in {1}", pipelineStep.GetType().Name, sw.Elapsed); } catch (Exception ex) { OnProcessorException(propertyBag, ex); } }
public ActionPipelineStep(string name, IPipelineStep <TOutput> input, Action <TOutput> action) { Name = name ?? throw new ArgumentNullException(nameof(name)); Input = input ?? throw new ArgumentNullException(nameof(input)); Action = action ?? throw new ArgumentNullException(nameof(action)); Signaler.RegisterMultiple(this, Input); }
public static IEnumerable <OUTPUT> Step <INPUT, OUTPUT>(this IEnumerable <INPUT> input, IPipelineStep <INPUT, OUTPUT> step) { LoopStep <INPUT, OUTPUT> loopDecorator = new LoopStep <INPUT, OUTPUT>(step); return(loopDecorator.Process(input)); }
public CachePipelineStep(IPipelineStep <TItem> step) { Step = step ?? throw new ArgumentNullException(nameof(step)); Name = $"Cache for step '{step.Name}'"; Signaler = Step.Signaler; Signaler.RegisterDependency(Step, this); }
public PausePipelineStep(string name, IPipelineStep <TOutput> input) { Name = name ?? throw new ArgumentNullException(nameof(name)); Input = input ?? throw new ArgumentNullException(nameof(input)); Signaler = Input.Signaler; Signaler.RegisterDependency(Input, this); }
public static INPUTOUTPUT Step <INPUTOUTPUT>(this INPUTOUTPUT input, IPipelineStep <INPUTOUTPUT> step) { if (step == null) { return(input); } return(step.Process(input)); }
public static void RegisterMultiple(this ISignaler signaler, IPipelineStep downstream, params IPipelineStep[] upstreams) { AssertSameSignaler(upstreams.Select(u => u.Signaler).ToArray()); foreach (var up in upstreams) { signaler.RegisterDependency(up, downstream); } }
public UnaryOperationPipelineStep(string name, IPipelineStep <TInput> input, Func <TInput, TOutput> operation) { Name = name ?? throw new ArgumentNullException(nameof(name)); Input = input ?? throw new ArgumentNullException(nameof(input)); Operation = operation ?? throw new ArgumentNullException(nameof(operation)); Signaler = Input.Signaler; Signaler.RegisterDependency(Input, this); }
public DemultiplexingPipelineStep(string name, IPipelineStep <TOutput> input, IPipelineStep <TSelect> select) { Name = name ?? throw new ArgumentNullException(nameof(name)); Input = input ?? throw new ArgumentNullException(nameof(input)); Select = select ?? throw new ArgumentNullException(nameof(select)); Signaler = Input.Signaler; Signaler.RegisterMultiple(this, Input, Select); Outputs = new Dictionary <TSelect, IPipelineStep <TOutput> >(); }
public BinaryOperationPipelineStep(string name, IPipelineStep <TInput1> input1, IPipelineStep <TInput2> input2, Func <TInput1, TInput2, TOutput> operation) { Name = name ?? throw new ArgumentNullException(nameof(name)); Input1 = input1 ?? throw new ArgumentNullException(nameof(input1)); Input2 = input2 ?? throw new ArgumentNullException(nameof(input2)); Operation = operation ?? throw new ArgumentNullException(nameof(operation)); Signaler = Input1.Signaler; Signaler.RegisterMultiple(this, Input1, Input2); }
public PassThroughPipelineStep(string name, IPipelineStep <TOutput> input, bool invalidateOnInvalidate, bool dependOnInput) { Name = name ?? throw new ArgumentNullException(nameof(name)); Input = input ?? throw new ArgumentNullException(nameof(input)); InvalidateOnInvalidate = invalidateOnInvalidate; if (dependOnInput) { Signaler.RegisterDependency(Input, this); } }
public GooeyCrawler AddPipelineStep(IPipelineStep step) { step.ImportSiteHash = siteHash; if (step is ICheckedPipelineStep) ((ICheckedPipelineStep)step).PerformErrorCheck(); this.pipelineSteps.Add(step); return this; }
internal void AddStep(IPipelineStep step) { Steps.Add((PipelineStep)step); bool isComplete = step.OutputQNames.All(s => PipelineQueues.ContainsKey(s)); isComplete &= PipelineQueues.ContainsKey(step.InputQName); if (isComplete) { ((PipelineStep)step).UpdateStatus(StepStatus.Complete); } }
public PipelineStep <U> Then <U>(Func <T, Task <U> > nextStep) { var newStep = new PipelineStep <U>(async() => { var previousStepValue = await GetLatestAsync(); return(await nextStep(previousStepValue)); }); _nextStep = newStep; return(newStep); }
public void AddInput(TSelect select, IPipelineStep <TOutput> input) { SignalerHelper.AssertSameSignaler(Signaler, input.Signaler); var pauseStep = new PausePipelineStep <TOutput>($"Multiplexer '{Name}' - Branch {select}", input); pauseStep.Pause(); Inputs.Add(select, pauseStep); Signaler.RegisterDependency(input, this); }
public static OUTPUT Step <INPUT, OUTPUT>(this INPUT input, IPipelineStep <INPUT, OUTPUT> step, Action <INPUT> inputEvent = null, Action <OUTPUT> outputEvent = null) { if (inputEvent != null || outputEvent != null) { var eventDecorator = new EventStep <INPUT, OUTPUT>(step); eventDecorator.OnInput += inputEvent; eventDecorator.OnOutput += outputEvent; return(eventDecorator.Process(input)); } return(step.Process(input)); }
public IPipelineStep <T> Register(IPipelineStep <T> step) { if (_nextStep == null) { _nextStep = step; } else { _nextStep.Register(step); } return(this); }
public IPipelineStep <T> Register(IPipelineStep <T> step) { if (_rootStep == null) { _rootStep = step; } else { _rootStep.Register(step); } return(_rootStep); }
/// <summary> /// Add a step to process the input and transform it into an output, allowing /// events to happen before and after processing /// </summary> /// <typeparam name="TInput">Type of the input</typeparam> /// <typeparam name="TOutput">Type of the output</typeparam> /// <param name="input">Input</param> /// <param name="step">Pipeline step</param> /// <param name="inputEvent">An action to execute on the input before processing it</param> /// <param name="outputEvent">An action to execute on the output after processing</param> /// <returns>Returns the output of processing the input</returns> public static TOutput AddStep <TInput, TOutput>( this TInput input, IPipelineStep <TInput, TOutput> step, Action <TInput> inputEvent = null, Action <TOutput> outputEvent = null) { var eventDecorator = new PipelineStep <TInput, TOutput>(step); eventDecorator.OnInput += inputEvent; eventDecorator.OnOutput += outputEvent; return(eventDecorator.Process(input)); }
private void WriteInformationForStep(string stepName, IPipelineStep <T> step, T context) { var stopwatch = new Stopwatch(); var logger = _kernel.Resolve <ILogger>(); logger.WriteBanner(); logger.InfoFormat("Running step '{0}'...", stepName); stopwatch.Start(); step.Execute(context); stopwatch.Stop(); logger.InfoFormat("Step '{0}' completed.", stepName); logger.InfoFormat("Execution Time: ({0}).", stopwatch.Elapsed.ToString()); logger.WriteBanner(); }
public PipelineStepStatus GetStatusPreviousStep(IPipelineStep step) { var steps = Steps.Where(s => s.Number == step.Number - 1); var stepsCount = steps.Count(); if (stepsCount == 0) { return(PipelineStepStatus.Complited); } var complitedSteps = Steps.Count(s => s.Number == step.Number - 1 && s.Status == PipelineStepStatus.Complited); return(complitedSteps == stepsCount ? PipelineStepStatus.Complited : PipelineStepStatus.Running); }
public GenericPipelineStep <TStepIn, TStepOut> GenerateStep <TStepIn, TStepOut>() { var pipelineStep = new GenericPipelineStep <TStepIn, TStepOut>(); var stepIndex = _pipelineSteps.Count; Task.Run(() => { IPipelineStep <TStepOut> nextPipelineStep = null; foreach (var input in pipelineStep.Buffer.GetConsumingEnumerable()) { bool isLastStep = stepIndex == _pipelineSteps.Count - 1; TStepOut output; try { output = pipelineStep.StepAction(input.Input); } catch (Exception e) { input.TaskCompletionSource.SetException(e); continue; } if (isLastStep) { input.TaskCompletionSource.SetResult((TPipeOut)(object)output); } else { nextPipelineStep = nextPipelineStep ?? (isLastStep ? null : _pipelineSteps[stepIndex + 1] as IPipelineStep <TStepOut>); nextPipelineStep.Buffer.Add(new Item <TStepOut>() { Input = output, TaskCompletionSource = input.TaskCompletionSource }); } } }); _pipelineSteps.Add(pipelineStep); return(pipelineStep); }
private void ExecutePipeLineStep(IPipelineStep pipelineStep, PropertyBag propertyBag) { try { if (pipelineStep is IPipelineStepWithTimeout) { IPipelineStepWithTimeout stepWithTimeout = (IPipelineStepWithTimeout)pipelineStep; m_Logger.Debug("Running pipeline step {0} with timeout {1}", pipelineStep.GetType().Name, stepWithTimeout.ProcessorTimeout); m_TaskRunner.RunSync(() => pipelineStep.Process(this, propertyBag), stepWithTimeout.ProcessorTimeout); } else { m_Logger.Debug("Running pipeline step {0}", pipelineStep.GetType().Name); pipelineStep.Process(this, propertyBag); } } catch (Exception ex) { OnProcessorException(propertyBag, ex); } }
/// <summary> /// Processes the step. /// </summary> /// <param name="inStepIndex">Which step is being processed in the pipeline.</param> /// <param name="inPipelineStep">The pipeline step instance.</param> private void StartStep <TStepIn, TStepOut>(int inStepIndex, PipelineStep <TStepIn, TStepOut> inPipelineStep) { IPipelineStep <TStepOut> nextPipelineStep = null; // Iterate over all inputs in the buffer for this step foreach (var input in inPipelineStep.Buffer.GetConsumingEnumerable()) { TStepOut output; // Try to perform the step. try { output = inPipelineStep.StepAction(input.Input); } catch (Exception e) { input.TaskCompletionSource.SetException(e); continue; } FinishStep(input, output, inStepIndex, ref nextPipelineStep); } }
/// <summary> /// Processes the optional step. /// </summary> /// <param name="inStepIndex">The index of this step in the pipeline.</param> /// <param name="inOptionalStep">The optional step being processed.</param> /// <typeparam name="TOptionalInOut">The In and Output type of the optional step.</typeparam> private void StartOptionalStep <TOptionalInOut>(int inStepIndex, OptionalPipelineStep <TOptionalInOut, TOptionalInOut> inOptionalStep) { IPipelineStep <TOptionalInOut> nextPipelineStep = null; foreach (var input in inOptionalStep.Buffer.GetConsumingEnumerable()) { TOptionalInOut output = default(TOptionalInOut); try { if (inOptionalStep.Choice(input.Input)) { output = inOptionalStep.StepAction(input.Input); } } catch (Exception e) { input.TaskCompletionSource.SetException(e); continue; } FinishStep(input, output, inStepIndex, ref nextPipelineStep); } }
public void AddUploadStep(IPipelineStep pipelineStep) { _uploadStep = pipelineStep; }
public NotFoundPipelineStepException(IPipelineStep step) : this($"No step with the given key ({step.Key}) is found", step.Key) { }
public void AddGettingEntityStep(IPipelineStep pipelineStep) { _gettingEntityStep = pipelineStep; }
public void AddGetTaskStep(IPipelineStep pipelineStep) { _getTaskStep = pipelineStep; }
private void ExecutePipeLineStep(IPipelineStep pipelineStep, PropertyBag propertyBag) { try { Stopwatch sw = Stopwatch.StartNew(); m_Logger.Debug("Executing pipeline step {0}", pipelineStep.GetType().Name); if (pipelineStep is IPipelineStepWithTimeout) { IPipelineStepWithTimeout stepWithTimeout = (IPipelineStepWithTimeout) pipelineStep; m_Logger.Debug("Running pipeline step {0} with timeout {1}", pipelineStep.GetType().Name, stepWithTimeout.ProcessorTimeout); m_TaskRunner.RunSync(cancelArgs => { if (!cancelArgs.Cancel) { pipelineStep.Process(this, propertyBag); } }, stepWithTimeout.ProcessorTimeout); } else { pipelineStep.Process(this, propertyBag); } m_Logger.Debug("Executed pipeline step {0} in {1}", pipelineStep.GetType().Name, sw.Elapsed); } catch (Exception ex) { OnProcessorException(propertyBag, ex); } }
public void AddCreatorStep(IPipelineStep pipelineStep) { _creatorStep = pipelineStep; }
public void AddInitializingWebClientStep(IPipelineStep pipelineStep) { _initializingWebClientStep = pipelineStep; }
public void AddEncodeStep(IPipelineStep pipelineStep) { _encodeStep = pipelineStep; }
private void ExecutePipeLineStep(IPipelineStep pipelineStep, PropertyBag propertyBag) { try { if (pipelineStep is IPipelineStepWithTimeout) { IPipelineStepWithTimeout stepWithTimeout = (IPipelineStepWithTimeout) pipelineStep; m_Logger.Debug("Running pipeline step {0} with timeout {1}", pipelineStep.GetType().Name, stepWithTimeout.ProcessorTimeout); m_TaskRunner.RunSync(() => pipelineStep.Process(this, propertyBag), stepWithTimeout.ProcessorTimeout); } else { m_Logger.Debug("Running pipeline step {0}", pipelineStep.GetType().Name); pipelineStep.Process(this, propertyBag); } } catch (Exception ex) { OnProcessorException(propertyBag, ex); } }
public void AddFinishStep(IPipelineStep pipelineStep) { _finishStep = pipelineStep; }
public void AddDownloadStep(IPipelineStep pipelineStep) { _downloadStep = pipelineStep; }
public void Add(IPipelineStep pipelineStep) { _list.Add(pipelineStep); }