public void DapperReaderNode_can_perform_simple_select_in_etlprocess(TestTable[] tests) { DapperExtensions.DapperExtensions.SqlDialect = new SqliteDialect(); using (var con = new SQLiteConnection("Data Source=:memory:;Version=3;New=True;")) { con.Open(); con.Execute("create table TestTable(Id int, Timestamp datetime, Name varchar(50));"); foreach (var test in tests) { con.Insert(test); } var context = new EtlPipelineContext(); var result = EtlProcessBuilder.Create(context) .Input(ctx => new DapperReaderNode <TestTable>(() => con, "select Id, Timestamp, Name from TestTable;")) .CompleteWithResult() .Build() .ExecuteWithResult(context); var results = result.Result.ToArray(); results.Length.Should().Be(tests.Length); results.Should().HaveSameCount(tests); results.Should().BeEquivalentTo(tests, config => config.Excluding(x => x.IsFrozen)); } }
public IEtlPipeline Run(Action <EtlPipelineContext, IEtlProcessBuilder> builder) { var b = EtlProcessBuilder.Create(); builder(Context, b); return(this); }
public override IEtlOperationResult Execute(EtlPipelineContext context) { _builder = EtlProcessBuilder.Create(context); _bootstrapBuilder(_builder); _etlProcess = ((EtlProcessBuilder)_builder).Build(); Named(_etlProcess.Name); return(_etlProcess.Execute(context)); }
public void Simple_etl_process_scenario_test() { var process = EtlProcessBuilder.Create() .GenerateInput <Row, int>(gen => gen.State < 5, (ctx, i, gen) => { gen.SetState(gen.State + 1); return(new Row { ["number"] = gen.State }); }) .Transform((ctx, row) => { var newRow = ctx.ObjectPool.Borrow <Row>(); row.CopyTo(newRow); newRow["transformed"] = true; ctx.ObjectPool.Return(row); return(newRow); }) .CompleteWithResult() .Build(); var context = new EtlPipelineContext(); var result = process.Execute(context); result.Should().BeAssignableTo <IEnumerableEtlOperationResult <Row> >(); var enumerableResult = result as IEnumerableEtlOperationResult <Row>; var results = enumerableResult.Result.ToList(); results.Count.Should().Be(5); for (var i = 1; i <= 5; i++) { results[i - 1]["number"].Should().Be(i); results[i - 1]["transformed"].Should().Be(true); } }
public void OnError_is_called_when_etl_process_node_raises_error() { var exception = new Exception("Whoops!"); var exceptionThrowingOp = new ExceptionThrowingEtlOperation(exception); var errorHandlerCalled = false; var input = new List <Row> { new Row { ["number"] = 1 }, new Row { ["number"] = 2 } }; var inputNode = new TestSourceNode((ctx, emitter) => { foreach (var item in input) { emitter.Emit(item); } emitter.SignalEnd(); }); var context = new EtlPipelineContext(); var transformNode = new GenericTransformationNode <Row>((objects, row) => throw exception); var process = EtlProcessBuilder.Create(context) .Input(ctx => inputNode) .Continue(ctx => transformNode) .Complete(ctx => new TestSinkNode()) .Build(); var pipeline = EtlPipeline.Create(settings => settings .UseExistingContext(context) .OnError((ctx, errors) => { errorHandlerCalled = true; errors.Length.Should().Be(2); errors[0].Exception.Should().Be(exception); errors[0].SourceOperation.Should().Be(process); errors[0].HasSourceItem.Should().BeTrue(); errors[0].SourceNode.Should().Be(transformNode); errors[0].SourceItem.Should().Be(input[1]); errors[1].Exception.Should().Be(exception); errors[1].SourceOperation.Should().Be(process); errors[1].HasSourceItem.Should().BeTrue(); errors[1].SourceNode.Should().Be(transformNode); errors[1].SourceItem.Should().Be(input[0]); return(true); }) ) .Run(ctx => process) .Execute(); errorHandlerCalled.Should().BeTrue(); }