public EtlOperationError(IEtlOperation sourceOperation, INode sourceNode, Exception exception,
object sourceItem)
{
SourceOperation = sourceOperation;
SourceNode = sourceNode;
Exception = exception;
SourceItem = sourceItem;
}
private void AddOperation(IEtlOperation operation)
{
if (_currentPredicate != null)
{
var conditionalOperation = new ConditionalEtlOperation(_currentPredicate, operation);
_steps.Add(conditionalOperation);
}
else
{
_steps.Add(operation);
}
}
public EtlOperationError(IEtlOperation sourceOperation, INode sourceNode, Exception exception)
: this(sourceOperation, sourceNode, exception, null)
{
}
public EtlOperationError(IEtlOperation sourceOperation, Exception exception)
: this(sourceOperation, exception, null)
{
}
public EtlPipelineResult AbortedOn(IEtlOperation operation)
{
AbortedOnOperation = operation;
return(this);
}
public PipelineAbortException(IEtlOperation operation, IEnumerable <EtlOperationError> errors, string message)
: base(message)
{
Operation = operation;
_errors.AddRange(errors);
}
public PipelineAbortException(IEtlOperation operation, IEnumerable <EtlOperationError> errors)
: this(operation, "The ETL pipeline is aborting.")
{
_errors.AddRange(errors);
}
public PipelineAbortException(IEtlOperation operation, EtlOperationError error, string message)
: base(message)
{
Operation = operation;
_errors.Add(error);
}
public PipelineAbortException(IEtlOperation operation, EtlOperationError error)
: this(operation, "The ETL pipeline is aborting.")
{
_errors.Add(error);
}
public PipelineAbortException(IEtlOperation operation, string message) : base(message)
{
Operation = operation;
}
public EtlOperationGroup AddOperation(IEtlOperation operation)
{
_operations.Add(operation);
return(this);
}
public EtlOperationResult WithError(IEtlOperation sourceOperation, Exception exception)
{
_errors.Add(new EtlOperationError(sourceOperation, exception));
return(this);
}
private IEtlPipeline RegisterOperation(IEtlOperation operation)
{
AddOperation(operation);
return(this);
}
public IEtlPipeline Run(IEtlOperation operation)
{
return(RegisterOperation(operation));
}
private IEtlOperationResult ExecuteOperation(IEtlOperation operation, EtlPipelineContext context)
{
_currentlyExecutingOperation = operation;
IEtlOperationResult result = null;
try
{
switch (operation)
{
case IConditionalLoopEtlOperation loop:
{
var multiResult = new EtlOperationResult(true);
do
{
foreach (var op in loop.GetOperations())
{
_executionResults[op] = LastResult = ExecuteOperation(op, context);
multiResult
.WithErrors(LastResult.Errors)
.QuiesceSuccess(LastResult.IsSuccess);
}
} while (loop.Predicate(Context));
return(multiResult);
}
case IEtlOperationCollection collection:
{
var multiResult = new EtlOperationResult(true);
foreach (var op in collection.GetOperations())
{
_executionResults[op] = LastResult = ExecuteOperation(op, context);
multiResult
.WithErrors(LastResult.Errors)
.QuiesceSuccess(LastResult.IsSuccess);
}
return(multiResult);
}
default:
result = operation.Execute(context);
break;
}
}
catch (Exception e)
{
if (EtlLibConfig.EnableDebug)
{
Debugger.Break();
}
_log.Error(
$"An error occured while executing operation '{operation.Name}' ({operation.GetType().Name}): {e}");
var error = new EtlOperationError(operation, e);
if (!_settings.OnErrorFn.Invoke(context, new[] { error }))
{
throw new PipelineAbortException(operation, error);
}
}
finally
{
if (result?.Errors.Count > 0)
{
context.ReportErrors(result.Errors);
if (!_settings.OnErrorFn.Invoke(context, result.Errors.ToArray()))
{
throw new PipelineAbortException(operation, result.Errors);
}
}
if (operation is IDisposable disposable)
{
_log.Debug("Disposing of resources used by step.");
disposable.Dispose();
}
_log.Debug("Cleaning up (globally).");
GC.Collect();
}
return(result);
}
public ConditionalEtlOperation(Func <EtlPipelineContext, bool> predicate, IEtlOperation operation)
{
_predicate = predicate;
_operation = operation;
}
