public void DapperReaderNode_can_perform_simple_select_in_etlprocess(TestTable[] tests)
{
DapperExtensions.DapperExtensions.SqlDialect = new SqliteDialect();
using (var con = new SQLiteConnection("Data Source=:memory:;Version=3;New=True;"))
{
con.Open();
con.Execute("create table TestTable(Id int, Timestamp datetime, Name varchar(50));");
foreach (var test in tests)
{
con.Insert(test);
}
var context = new EtlPipelineContext();
var result = EtlProcessBuilder.Create(context)
.Input(ctx => new DapperReaderNode <TestTable>(() => con, "select Id, Timestamp, Name from TestTable;"))
.CompleteWithResult()
.Build()
.ExecuteWithResult(context);
var results = result.Result.ToArray();
results.Length.Should().Be(tests.Length);
results.Should().HaveSameCount(tests);
results.Should().BeEquivalentTo(tests, config => config.Excluding(x => x.IsFrozen));
}
}
public IEtlPipeline Run(Action <EtlPipelineContext, IEtlProcessBuilder> builder)
{
var b = EtlProcessBuilder.Create();
builder(Context, b);
return(this);
}
public override IEtlOperationResult Execute(EtlPipelineContext context)
{
_builder = EtlProcessBuilder.Create(context);
_bootstrapBuilder(_builder);
_etlProcess = ((EtlProcessBuilder)_builder).Build();
Named(_etlProcess.Name);
return(_etlProcess.Execute(context));
}
public void Simple_etl_process_scenario_test()
{
var process = EtlProcessBuilder.Create()
.GenerateInput <Row, int>(gen => gen.State < 5, (ctx, i, gen) =>
{
gen.SetState(gen.State + 1);
return(new Row {
["number"] = gen.State
});
})
.Transform((ctx, row) =>
{
var newRow = ctx.ObjectPool.Borrow <Row>();
row.CopyTo(newRow);
newRow["transformed"] = true;
ctx.ObjectPool.Return(row);
return(newRow);
})
.CompleteWithResult()
.Build();
var context = new EtlPipelineContext();
var result = process.Execute(context);
result.Should().BeAssignableTo <IEnumerableEtlOperationResult <Row> >();
var enumerableResult = result as IEnumerableEtlOperationResult <Row>;
var results = enumerableResult.Result.ToList();
results.Count.Should().Be(5);
for (var i = 1; i <= 5; i++)
{
results[i - 1]["number"].Should().Be(i);
results[i - 1]["transformed"].Should().Be(true);
}
}
public void OnError_is_called_when_etl_process_node_raises_error()
{
var exception = new Exception("Whoops!");
var exceptionThrowingOp = new ExceptionThrowingEtlOperation(exception);
var errorHandlerCalled = false;
var input = new List <Row>
{
new Row {
["number"] = 1
},
new Row {
["number"] = 2
}
};
var inputNode = new TestSourceNode((ctx, emitter) =>
{
foreach (var item in input)
{
emitter.Emit(item);
}
emitter.SignalEnd();
});
var context = new EtlPipelineContext();
var transformNode = new GenericTransformationNode <Row>((objects, row) => throw exception);
var process = EtlProcessBuilder.Create(context)
.Input(ctx => inputNode)
.Continue(ctx => transformNode)
.Complete(ctx => new TestSinkNode())
.Build();
var pipeline = EtlPipeline.Create(settings => settings
.UseExistingContext(context)
.OnError((ctx, errors) =>
{
errorHandlerCalled = true;
errors.Length.Should().Be(2);
errors[0].Exception.Should().Be(exception);
errors[0].SourceOperation.Should().Be(process);
errors[0].HasSourceItem.Should().BeTrue();
errors[0].SourceNode.Should().Be(transformNode);
errors[0].SourceItem.Should().Be(input[1]);
errors[1].Exception.Should().Be(exception);
errors[1].SourceOperation.Should().Be(process);
errors[1].HasSourceItem.Should().BeTrue();
errors[1].SourceNode.Should().Be(transformNode);
errors[1].SourceItem.Should().Be(input[0]);
return(true);
})
)
.Run(ctx => process)
.Execute();
errorHandlerCalled.Should().BeTrue();
}
