Ejemplo n.º 1
0
        public void InputOutputAdapter_works()
        {
            var outputNode = new TestSourceNode((ctx, e) =>
            {
                for (var i = 1; i <= 5; i++)
                {
                    e.Emit(new Row {
                        ["number"] = i
                    });
                }

                e.SignalEnd();
            });

            var inputNode      = new TestSinkNode();
            var nodeStatistics = new NodeStatistics();

            nodeStatistics.RegisterNode(outputNode);
            nodeStatistics.RegisterNode(inputNode);

            var ioAdapter = new InputOutputAdapter <Row>(outputNode);

            ioAdapter.SetNodeStatisticsCollector(nodeStatistics);

            ioAdapter.AttachConsumer(inputNode);

            var context = new EtlPipelineContext();

            outputNode.Execute(context);
            inputNode.Execute(context);

            inputNode.ReceivedItems.Count.Should().Be(5);
            for (var i = 1; i <= 5; i++)
            {
                inputNode.ReceivedItems[i - 1]["number"].Should().Be(i);
            }

            nodeStatistics.TotalReads.Should().Be(5);
            nodeStatistics.TotalWrites.Should().Be(5);
            nodeStatistics.TotalErrors.Should().Be(0);
        }
Ejemplo n.º 2
0
        public void OnError_is_called_when_etl_process_node_raises_error()
        {
            var exception           = new Exception("Whoops!");
            var exceptionThrowingOp = new ExceptionThrowingEtlOperation(exception);
            var errorHandlerCalled  = false;

            var input = new List <Row>
            {
                new Row {
                    ["number"] = 1
                },
                new Row {
                    ["number"] = 2
                }
            };

            var inputNode = new TestSourceNode((ctx, emitter) =>
            {
                foreach (var item in input)
                {
                    emitter.Emit(item);
                }
                emitter.SignalEnd();
            });

            var context = new EtlPipelineContext();

            var transformNode = new GenericTransformationNode <Row>((objects, row) => throw exception);

            var process = EtlProcessBuilder.Create(context)
                          .Input(ctx => inputNode)
                          .Continue(ctx => transformNode)
                          .Complete(ctx => new TestSinkNode())
                          .Build();

            var pipeline = EtlPipeline.Create(settings => settings
                                              .UseExistingContext(context)
                                              .OnError((ctx, errors) =>
            {
                errorHandlerCalled = true;
                errors.Length.Should().Be(2);

                errors[0].Exception.Should().Be(exception);
                errors[0].SourceOperation.Should().Be(process);
                errors[0].HasSourceItem.Should().BeTrue();
                errors[0].SourceNode.Should().Be(transformNode);
                errors[0].SourceItem.Should().Be(input[1]);

                errors[1].Exception.Should().Be(exception);
                errors[1].SourceOperation.Should().Be(process);
                errors[1].HasSourceItem.Should().BeTrue();
                errors[1].SourceNode.Should().Be(transformNode);
                errors[1].SourceItem.Should().Be(input[0]);
                return(true);
            })
                                              )
                           .Run(ctx => process)
                           .Execute();

            errorHandlerCalled.Should().BeTrue();
        }