public void OnError_is_called_when_etl_operation_returns_errors()
{
var exception = new Exception("Whoops!");
var errorReturningOp = new ActionEtlOperation(ctx => false);
errorReturningOp.WithErrors(new EtlOperationError(errorReturningOp, exception));
var errorHandlerCalled = false;
var pipeline = EtlPipeline.Create(settings => settings
.OnError((ctx, errors) =>
{
errorHandlerCalled = true;
errors.Count().Should().Be(1);
var err = errors.Single();
err.Exception.Should().Be(exception);
err.SourceOperation.Should().Be(errorReturningOp);
err.HasSourceItem.Should().BeFalse();
err.SourceNode.Should().BeNull();
return(true);
})
)
.Run(errorReturningOp)
.Execute();
errorHandlerCalled.Should().BeTrue();
}
public void Do_while_stops_execution_when_loop_returns_error_with_default_error_handling()
{
var context = new EtlPipelineContext();
var items = new Queue <string>(new[] { "The", "Quick", "Brown", "Foxed", "Jumps", "Over", "The", "Lazy", "Dog" });
var iterations = 0;
var getCountOperation = new ActionEtlOperation(ctx =>
{
ctx.State["remaining_count"] = items.Count;
return(true);
});
EtlPipeline.Create(settings => settings
.UseExistingContext(context)
.Named("Do-While Pipeline Test"))
.Run(getCountOperation)
.Do(pipeline =>
{
pipeline
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
items.Dequeue();
iterations++;
return(iterations != 5);
}))
.Run(getCountOperation);
})
.While(ctx => (int)ctx.State["remaining_count"] > 5)
.Execute();
iterations.Should().Be(4);
}
public void OnError_stops_etl_pipeline_execution_by_default()
{
var exception = new Exception("Whoops!");
var exceptionThrowingOp = new ExceptionThrowingEtlOperation(exception);
var operationAfterErrorRun = false;
var pipeline = EtlPipeline.Create(settings => {})
.Run(exceptionThrowingOp)
.Run(ctx => new ActionEtlOperation(context => operationAfterErrorRun = true))
.Execute();
operationAfterErrorRun.Should().BeFalse();
}
public void EtlPipeline_continues_executing_when_OnError_returns_true()
{
var exception = new Exception("Whoops!");
var exceptionThrowingOp = new ExceptionThrowingEtlOperation(exception);
var operationAfterErrorRun = false;
var pipeline = EtlPipeline.Create(settings => settings
.OnError((ctx, errors) => true)
)
.Run(exceptionThrowingOp)
.Run(ctx => new ActionEtlOperation(context => operationAfterErrorRun = true))
.Execute();
operationAfterErrorRun.Should().BeTrue();
}
public void Pipeline_if_clause_runs_contents_if_predicate_returns_true()
{
var wasRun1 = false;
var wasRun2 = false;
var wasRunBefore = false;
var wasRunAfter = false;
var context = new EtlPipelineContext();
var pipeline = EtlPipeline.Create(settings => settings
.UseExistingContext(context)
.Named("Run If Test"))
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
wasRunBefore = true;
return(true);
}))
.If(ctx => (string)ctx.State["hello"] == "world", p =>
{
p
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
wasRun1 = true;
return(true);
}).Named("If 1"))
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
wasRun2 = true;
return(true);
}).Named("If 2"));
})
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
wasRunAfter = true;
return(true);
}));
context.State["hello"] = "world";
pipeline.Execute();
wasRunBefore.Should().BeTrue();
wasRun1.Should().BeTrue();
wasRun2.Should().BeTrue();
wasRunAfter.Should().BeTrue();
}
public void Pipeline_runif_does_not_run_operation_if_predicate_returns_false()
{
var wasRun = false;
var context = new EtlPipelineContext();
context.State["hello"] = "not world";
EtlPipeline.Create(settings => settings
.UseExistingContext(context)
.Named("Runif Test"))
.RunIf(ctx => (string)ctx.State["hello"] == "world",
ctx => new ActionEtlOperation(ctx2 =>
{
wasRun = true;
return(true);
})
)
.Execute();
wasRun.Should().BeFalse();
}
public void Pipeline_aborts_when_error_encountered_when_executing_a_nested_pipeline()
{
var run1 = false;
var run2 = false;
var run3 = false;
var innerPipeline = EtlPipeline.Create(settings => settings
.Named("Pipeline 2"))
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
run2 = true;
throw new Exception("Uh oh!");
}));
var result = EtlPipeline.Create(settings => settings
.Named("Pipeline 1"))
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
run1 = true;
return(true);
}))
.Run(ctx => innerPipeline)
.Run(new ActionEtlOperation(ctx =>
{
run3 = true;
return(true);
}))
.Execute();
run1.Should().BeTrue();
run2.Should().BeTrue();
run3.Should().BeFalse();
result.Should().NotBeNull();
result.IsSuccess.Should().BeFalse();
}
public void Pipeline_can_execute_another_pipeline()
{
var run1 = false;
var run2 = false;
var run3 = false;
var innerPipeline = EtlPipeline.Create(settings => settings
.Named("Pipeline 2"))
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
run2 = true;
return(true);
}));
var result = EtlPipeline.Create(settings => settings
.Named("Pipeline 1"))
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
run1 = true;
return(true);
}))
.Run(ctx => innerPipeline)
.Run(new ActionEtlOperation(ctx =>
{
run3 = true;
return(true);
}))
.Execute();
run1.Should().BeTrue();
run2.Should().BeTrue();
run3.Should().BeTrue();
result.Should().NotBeNull();
result.IsSuccess.Should().BeTrue();
}
public void OnError_is_called_when_etl_process_node_raises_error()
{
var exception = new Exception("Whoops!");
var exceptionThrowingOp = new ExceptionThrowingEtlOperation(exception);
var errorHandlerCalled = false;
var input = new List <Row>
{
new Row {
["number"] = 1
},
new Row {
["number"] = 2
}
};
var inputNode = new TestSourceNode((ctx, emitter) =>
{
foreach (var item in input)
{
emitter.Emit(item);
}
emitter.SignalEnd();
});
var context = new EtlPipelineContext();
var transformNode = new GenericTransformationNode <Row>((objects, row) => throw exception);
var process = EtlProcessBuilder.Create(context)
.Input(ctx => inputNode)
.Continue(ctx => transformNode)
.Complete(ctx => new TestSinkNode())
.Build();
var pipeline = EtlPipeline.Create(settings => settings
.UseExistingContext(context)
.OnError((ctx, errors) =>
{
errorHandlerCalled = true;
errors.Length.Should().Be(2);
errors[0].Exception.Should().Be(exception);
errors[0].SourceOperation.Should().Be(process);
errors[0].HasSourceItem.Should().BeTrue();
errors[0].SourceNode.Should().Be(transformNode);
errors[0].SourceItem.Should().Be(input[1]);
errors[1].Exception.Should().Be(exception);
errors[1].SourceOperation.Should().Be(process);
errors[1].HasSourceItem.Should().BeTrue();
errors[1].SourceNode.Should().Be(transformNode);
errors[1].SourceItem.Should().Be(input[0]);
return(true);
})
)
.Run(ctx => process)
.Execute();
errorHandlerCalled.Should().BeTrue();
}
