public void InputOutputAdapter_works()
{
var outputNode = new TestSourceNode((ctx, e) =>
{
for (var i = 1; i <= 5; i++)
{
e.Emit(new Row {
["number"] = i
});
}
e.SignalEnd();
});
var inputNode = new TestSinkNode();
var nodeStatistics = new NodeStatistics();
nodeStatistics.RegisterNode(outputNode);
nodeStatistics.RegisterNode(inputNode);
var ioAdapter = new InputOutputAdapter <Row>(outputNode);
ioAdapter.SetNodeStatisticsCollector(nodeStatistics);
ioAdapter.AttachConsumer(inputNode);
var context = new EtlPipelineContext();
outputNode.Execute(context);
inputNode.Execute(context);
inputNode.ReceivedItems.Count.Should().Be(5);
for (var i = 1; i <= 5; i++)
{
inputNode.ReceivedItems[i - 1]["number"].Should().Be(i);
}
nodeStatistics.TotalReads.Should().Be(5);
nodeStatistics.TotalWrites.Should().Be(5);
nodeStatistics.TotalErrors.Should().Be(0);
}
public void OnError_is_called_when_etl_process_node_raises_error()
{
var exception = new Exception("Whoops!");
var exceptionThrowingOp = new ExceptionThrowingEtlOperation(exception);
var errorHandlerCalled = false;
var input = new List <Row>
{
new Row {
["number"] = 1
},
new Row {
["number"] = 2
}
};
var inputNode = new TestSourceNode((ctx, emitter) =>
{
foreach (var item in input)
{
emitter.Emit(item);
}
emitter.SignalEnd();
});
var context = new EtlPipelineContext();
var transformNode = new GenericTransformationNode <Row>((objects, row) => throw exception);
var process = EtlProcessBuilder.Create(context)
.Input(ctx => inputNode)
.Continue(ctx => transformNode)
.Complete(ctx => new TestSinkNode())
.Build();
var pipeline = EtlPipeline.Create(settings => settings
.UseExistingContext(context)
.OnError((ctx, errors) =>
{
errorHandlerCalled = true;
errors.Length.Should().Be(2);
errors[0].Exception.Should().Be(exception);
errors[0].SourceOperation.Should().Be(process);
errors[0].HasSourceItem.Should().BeTrue();
errors[0].SourceNode.Should().Be(transformNode);
errors[0].SourceItem.Should().Be(input[1]);
errors[1].Exception.Should().Be(exception);
errors[1].SourceOperation.Should().Be(process);
errors[1].HasSourceItem.Should().BeTrue();
errors[1].SourceNode.Should().Be(transformNode);
errors[1].SourceItem.Should().Be(input[0]);
return(true);
})
)
.Run(ctx => process)
.Execute();
errorHandlerCalled.Should().BeTrue();
}