private EtlPipelineResult InternalExecute(EtlPipelineContext context)
{
var result = new EtlPipelineResult();
PrintHeader();
var stopwatch = Stopwatch.StartNew();
if (_settings.ObjectPoolRegistrations.Count > 0)
{
_log.Info("Initializing object pools...");
foreach (var pool in _settings.ObjectPoolRegistrations)
{
_log.Info($" * Creating pool for '{pool.Type.Name}' (InitialSize={pool.InitialSize}, AutoGrow={pool.AutoGrow})");
context.ObjectPool.RegisterAndInitializeObjectPool(pool.Type, pool.InitialSize, pool.AutoGrow);
}
}
for (var i = 0; i < _steps.Count; i++)
{
_log.Info($"Executing step #{i + 1} ({_steps[i].GetType().Name}): '{_steps[i].Name}'");
var operation = _steps[i];
try
{
_executionResults[_steps[i]] = LastResult = ExecuteOperation(operation, context);
}
catch (PipelineAbortException e)
{
result
.WithErrors(e.Errors)
.AbortedOn(_currentlyExecutingOperation)
.QuiesceIsSuccess(false);
_log.Warn($"Error handling has indicated that the ETL process should be halted after error, terminating. Message: {e.Message}");
break;
}
}
_log.Debug("Deallocating all object pools:");
foreach (var pool in Context.ObjectPool.Pools)
{
_log.Debug($" * ObjectPool<{pool.Type.Name}> => Referenced: {pool.Referenced}, Free: {pool.Free}");
}
context.ObjectPool.DeAllocate();
stopwatch.Stop();
return(result.WithTotalRunTime(stopwatch.Elapsed));
}
private IEtlOperationResult ExecuteOperation(IEtlOperation operation, EtlPipelineContext context)
{
_currentlyExecutingOperation = operation;
IEtlOperationResult result = null;
try
{
switch (operation)
{
case IConditionalLoopEtlOperation loop:
{
var multiResult = new EtlOperationResult(true);
do
{
foreach (var op in loop.GetOperations())
{
_executionResults[op] = LastResult = ExecuteOperation(op, context);
multiResult
.WithErrors(LastResult.Errors)
.QuiesceSuccess(LastResult.IsSuccess);
}
} while (loop.Predicate(Context));
return(multiResult);
}
case IEtlOperationCollection collection:
{
var multiResult = new EtlOperationResult(true);
foreach (var op in collection.GetOperations())
{
_executionResults[op] = LastResult = ExecuteOperation(op, context);
multiResult
.WithErrors(LastResult.Errors)
.QuiesceSuccess(LastResult.IsSuccess);
}
return(multiResult);
}
default:
result = operation.Execute(context);
break;
}
}
catch (Exception e)
{
if (EtlLibConfig.EnableDebug)
{
Debugger.Break();
}
_log.Error(
$"An error occured while executing operation '{operation.Name}' ({operation.GetType().Name}): {e}");
var error = new EtlOperationError(operation, e);
if (!_settings.OnErrorFn.Invoke(context, new[] { error }))
{
throw new PipelineAbortException(operation, error);
}
}
finally
{
if (result?.Errors.Count > 0)
{
context.ReportErrors(result.Errors);
if (!_settings.OnErrorFn.Invoke(context, result.Errors.ToArray()))
{
throw new PipelineAbortException(operation, result.Errors);
}
}
if (operation is IDisposable disposable)
{
_log.Debug("Disposing of resources used by step.");
disposable.Dispose();
}
_log.Debug("Cleaning up (globally).");
GC.Collect();
}
return(result);
}