public EtlOperationError(IEtlOperation sourceOperation, INode sourceNode, Exception exception, object sourceItem) { SourceOperation = sourceOperation; SourceNode = sourceNode; Exception = exception; SourceItem = sourceItem; }
private void AddOperation(IEtlOperation operation) { if (_currentPredicate != null) { var conditionalOperation = new ConditionalEtlOperation(_currentPredicate, operation); _steps.Add(conditionalOperation); } else { _steps.Add(operation); } }
public EtlOperationError(IEtlOperation sourceOperation, INode sourceNode, Exception exception) : this(sourceOperation, sourceNode, exception, null) { }
public EtlOperationError(IEtlOperation sourceOperation, Exception exception) : this(sourceOperation, exception, null) { }
public EtlPipelineResult AbortedOn(IEtlOperation operation) { AbortedOnOperation = operation; return(this); }
public PipelineAbortException(IEtlOperation operation, IEnumerable <EtlOperationError> errors, string message) : base(message) { Operation = operation; _errors.AddRange(errors); }
public PipelineAbortException(IEtlOperation operation, IEnumerable <EtlOperationError> errors) : this(operation, "The ETL pipeline is aborting.") { _errors.AddRange(errors); }
public PipelineAbortException(IEtlOperation operation, EtlOperationError error, string message) : base(message) { Operation = operation; _errors.Add(error); }
public PipelineAbortException(IEtlOperation operation, EtlOperationError error) : this(operation, "The ETL pipeline is aborting.") { _errors.Add(error); }
public PipelineAbortException(IEtlOperation operation, string message) : base(message) { Operation = operation; }
public EtlOperationGroup AddOperation(IEtlOperation operation) { _operations.Add(operation); return(this); }
public EtlOperationResult WithError(IEtlOperation sourceOperation, Exception exception) { _errors.Add(new EtlOperationError(sourceOperation, exception)); return(this); }
private IEtlPipeline RegisterOperation(IEtlOperation operation) { AddOperation(operation); return(this); }
public IEtlPipeline Run(IEtlOperation operation) { return(RegisterOperation(operation)); }
private IEtlOperationResult ExecuteOperation(IEtlOperation operation, EtlPipelineContext context) { _currentlyExecutingOperation = operation; IEtlOperationResult result = null; try { switch (operation) { case IConditionalLoopEtlOperation loop: { var multiResult = new EtlOperationResult(true); do { foreach (var op in loop.GetOperations()) { _executionResults[op] = LastResult = ExecuteOperation(op, context); multiResult .WithErrors(LastResult.Errors) .QuiesceSuccess(LastResult.IsSuccess); } } while (loop.Predicate(Context)); return(multiResult); } case IEtlOperationCollection collection: { var multiResult = new EtlOperationResult(true); foreach (var op in collection.GetOperations()) { _executionResults[op] = LastResult = ExecuteOperation(op, context); multiResult .WithErrors(LastResult.Errors) .QuiesceSuccess(LastResult.IsSuccess); } return(multiResult); } default: result = operation.Execute(context); break; } } catch (Exception e) { if (EtlLibConfig.EnableDebug) { Debugger.Break(); } _log.Error( $"An error occured while executing operation '{operation.Name}' ({operation.GetType().Name}): {e}"); var error = new EtlOperationError(operation, e); if (!_settings.OnErrorFn.Invoke(context, new[] { error })) { throw new PipelineAbortException(operation, error); } } finally { if (result?.Errors.Count > 0) { context.ReportErrors(result.Errors); if (!_settings.OnErrorFn.Invoke(context, result.Errors.ToArray())) { throw new PipelineAbortException(operation, result.Errors); } } if (operation is IDisposable disposable) { _log.Debug("Disposing of resources used by step."); disposable.Dispose(); } _log.Debug("Cleaning up (globally)."); GC.Collect(); } return(result); }
public ConditionalEtlOperation(Func <EtlPipelineContext, bool> predicate, IEtlOperation operation) { _predicate = predicate; _operation = operation; }