public void InputOutputAdapter_works() { var outputNode = new TestSourceNode((ctx, e) => { for (var i = 1; i <= 5; i++) { e.Emit(new Row { ["number"] = i }); } e.SignalEnd(); }); var inputNode = new TestSinkNode(); var nodeStatistics = new NodeStatistics(); nodeStatistics.RegisterNode(outputNode); nodeStatistics.RegisterNode(inputNode); var ioAdapter = new InputOutputAdapter <Row>(outputNode); ioAdapter.SetNodeStatisticsCollector(nodeStatistics); ioAdapter.AttachConsumer(inputNode); var context = new EtlPipelineContext(); outputNode.Execute(context); inputNode.Execute(context); inputNode.ReceivedItems.Count.Should().Be(5); for (var i = 1; i <= 5; i++) { inputNode.ReceivedItems[i - 1]["number"].Should().Be(i); } nodeStatistics.TotalReads.Should().Be(5); nodeStatistics.TotalWrites.Should().Be(5); nodeStatistics.TotalErrors.Should().Be(0); }
public void OnError_is_called_when_etl_process_node_raises_error() { var exception = new Exception("Whoops!"); var exceptionThrowingOp = new ExceptionThrowingEtlOperation(exception); var errorHandlerCalled = false; var input = new List <Row> { new Row { ["number"] = 1 }, new Row { ["number"] = 2 } }; var inputNode = new TestSourceNode((ctx, emitter) => { foreach (var item in input) { emitter.Emit(item); } emitter.SignalEnd(); }); var context = new EtlPipelineContext(); var transformNode = new GenericTransformationNode <Row>((objects, row) => throw exception); var process = EtlProcessBuilder.Create(context) .Input(ctx => inputNode) .Continue(ctx => transformNode) .Complete(ctx => new TestSinkNode()) .Build(); var pipeline = EtlPipeline.Create(settings => settings .UseExistingContext(context) .OnError((ctx, errors) => { errorHandlerCalled = true; errors.Length.Should().Be(2); errors[0].Exception.Should().Be(exception); errors[0].SourceOperation.Should().Be(process); errors[0].HasSourceItem.Should().BeTrue(); errors[0].SourceNode.Should().Be(transformNode); errors[0].SourceItem.Should().Be(input[1]); errors[1].Exception.Should().Be(exception); errors[1].SourceOperation.Should().Be(process); errors[1].HasSourceItem.Should().BeTrue(); errors[1].SourceNode.Should().Be(transformNode); errors[1].SourceItem.Should().Be(input[0]); return(true); }) ) .Run(ctx => process) .Execute(); errorHandlerCalled.Should().BeTrue(); }