public void OnError_stops_etl_pipeline_execution_by_default()
{
var exception = new Exception("Whoops!");
var exceptionThrowingOp = new ExceptionThrowingEtlOperation(exception);
var operationAfterErrorRun = false;
var pipeline = EtlPipeline.Create(settings => {})
.Run(exceptionThrowingOp)
.Run(ctx => new ActionEtlOperation(context => operationAfterErrorRun = true))
.Execute();
operationAfterErrorRun.Should().BeFalse();
}
public async Task RunsAllStepsInMultiStagePipeline()
{
using (var writer = new TestBulkWriter <PipelineTestsOtherTestClass>())
{
var items = Enumerable
.Range(1, 1000)
.Select(i => new PipelineTestsMyTestClass {
Id = i, Name = "Bob"
})
.ToAsyncEnumerable();
var pipeline = EtlPipeline
.StartWith(items)
.Aggregate(f => f.Max(c => c.Id))
.Pivot(i =>
{
var result = new List <PipelineTestsMyTestClass>();
for (var j = 1; j <= i; j++)
{
result.Add(new PipelineTestsMyTestClass {
Id = j, Name = $"Bob {j}"
});
}
return(result);
})
.Project(i =>
{
var nameParts = i.Name.Split(' ');
return(new PipelineTestsOtherTestClass {
Id = i.Id, FirstName = nameParts[0], LastName = nameParts[1]
});
})
.TransformInPlace(i =>
{
i.Id -= 1;
i.FirstName = "Alice";
i.LastName = $"{i.Id}";
})
.WriteTo(writer);
await pipeline.ExecuteAsync();
Assert.Equal(1000, writer.ItemsWritten.Count);
Assert.Equal(0, writer.ItemsWritten[0].Id);
Assert.Equal("Alice", writer.ItemsWritten[0].FirstName);
Assert.Equal("0", writer.ItemsWritten[0].LastName);
Assert.Equal(999, writer.ItemsWritten[999].Id);
Assert.Equal("Alice", writer.ItemsWritten[999].FirstName);
Assert.Equal("999", writer.ItemsWritten[999].LastName);
}
}
public void EtlPipeline_continues_executing_when_OnError_returns_true()
{
var exception = new Exception("Whoops!");
var exceptionThrowingOp = new ExceptionThrowingEtlOperation(exception);
var operationAfterErrorRun = false;
var pipeline = EtlPipeline.Create(settings => settings
.OnError((ctx, errors) => true)
)
.Run(exceptionThrowingOp)
.Run(ctx => new ActionEtlOperation(context => operationAfterErrorRun = true))
.Execute();
operationAfterErrorRun.Should().BeTrue();
}
public void Pipeline_if_clause_runs_contents_if_predicate_returns_true()
{
var wasRun1 = false;
var wasRun2 = false;
var wasRunBefore = false;
var wasRunAfter = false;
var context = new EtlPipelineContext();
var pipeline = EtlPipeline.Create(settings => settings
.UseExistingContext(context)
.Named("Run If Test"))
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
wasRunBefore = true;
return(true);
}))
.If(ctx => (string)ctx.State["hello"] == "world", p =>
{
p
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
wasRun1 = true;
return(true);
}).Named("If 1"))
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
wasRun2 = true;
return(true);
}).Named("If 2"));
})
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
wasRunAfter = true;
return(true);
}));
context.State["hello"] = "world";
pipeline.Execute();
wasRunBefore.Should().BeTrue();
wasRun1.Should().BeTrue();
wasRun2.Should().BeTrue();
wasRunAfter.Should().BeTrue();
}
public async Task RunsAggregatorStep()
{
using (var writer = new TestBulkWriter <int>())
{
var items = Enumerable.Range(1, 1000).Select(i => new PipelineTestsMyTestClass {
Id = i, Name = "Bob"
});
var pipeline = EtlPipeline.StartWith(items)
.Aggregate(f => f.Sum(c => c.Id))
.WriteTo(writer);
await pipeline.ExecuteAsync();
Assert.Single(writer.ItemsWritten);
Assert.Equal(Enumerable.Range(1, 1000).Sum(), writer.ItemsWritten[0]);
}
}
public async Task RaisesExceptionsForAllStepsThatThrow()
{
var tableName = TestHelpers.DropCreate(nameof(PipelineTestsMyTestClass));
using (var writer = new BulkWriter <PipelineTestsMyTestClass>(_connectionString))
{
var items = Enumerable
.Range(1, 1000)
.Select(i => new PipelineTestsMyTestClass {
Id = i, Name = "Bob"
})
.ToAsyncEnumerable();
var pipeline = EtlPipeline
.StartWith(items)
.Project(i =>
{
//pump a few values through to ensure the next pipeline step actually
//gets run
if (i.Id >= 400)
{
throw new Exception("Projection exception 1");
}
return(i);
})
.Project(i =>
{
if (i.Id >= 200)
{
throw new Exception("Projection exception 2");
}
return(i);
})
.WriteTo(writer);
var pipelineTask = pipeline.ExecuteAsync();
await Assert.ThrowsAsync <Exception>(() => pipelineTask);
Assert.Equal(2, pipelineTask.Exception.InnerExceptions.Count);
Assert.Equal(1, pipelineTask.Exception.InnerExceptions.Count(e => e.Message == "Projection exception 1"));
Assert.Equal(1, pipelineTask.Exception.InnerExceptions.Count(e => e.Message == "Projection exception 2"));
}
}
public async Task ThrowsWhenAStepThrows()
{
var tableName = TestHelpers.DropCreate(nameof(PipelineTestsMyTestClass));
using (var writer = new BulkWriter <PipelineTestsMyTestClass>(_connectionString))
{
var items = Enumerable.Range(1, 1000).Select(i => new PipelineTestsMyTestClass {
Id = i, Name = "Bob"
});
var pipeline = EtlPipeline
.StartWith(items)
.Project <PipelineTestsMyTestClass>(i => throw new Exception("Projection exception"))
.WriteTo(writer);
var pipelineTask = pipeline.ExecuteAsync();
var exception = await Assert.ThrowsAsync <Exception>(() => pipelineTask);
Assert.Equal("Projection exception", exception.Message);
}
}
public async Task WritesToBulkWriter()
{
var tableName = TestHelpers.DropCreate(nameof(PipelineTestsMyTestClass));
using (var writer = new BulkWriter <PipelineTestsMyTestClass>(_connectionString))
{
var items = Enumerable.Range(1, 1000).Select(i => new PipelineTestsMyTestClass {
Id = i, Name = "Bob"
});
var pipeline = EtlPipeline
.StartWith(items)
.WriteTo(writer);
await pipeline.ExecuteAsync();
var count = (int)await TestHelpers.ExecuteScalar(_connectionString, $"SELECT COUNT(1) FROM {tableName}");
Assert.Equal(1000, count);
}
}
public void Do_while_breaks_loop_when_predicate_returns_false()
{
var context = new EtlPipelineContext();
var items = new Queue <string>(new[] { "The", "Quick", "Brown", "Foxed", "Jumps", "Over", "The", "Lazy", "Dog" });
var iterations = 0;
var getCountOperation = new ActionEtlOperation(ctx =>
{
ctx.State["remaining_count"] = items.Count;
return(true);
});
var executedAfter = false;
EtlPipeline.Create(settings => settings
.UseExistingContext(context)
.Named("Do-While Pipeline Test"))
.Run(getCountOperation)
.Do(pipeline =>
{
pipeline
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
items.Dequeue();
iterations++;
return(true);
}))
.Run(getCountOperation);
})
.While(ctx => (int)ctx.State["remaining_count"] > 5)
.Run(new ActionEtlOperation(ctx =>
{
executedAfter = true;
return(true);
}))
.Execute();
iterations.Should().Be(4);
}
public async Task IgnoresNullLogger()
{
using (var writer = new TestBulkWriter <PipelineTestsMyTestClass>())
{
var items = Enumerable.Range(1, 1000).Select(i => new PipelineTestsMyTestClass {
Id = i, Name = "Bob"
});
var pipeline = EtlPipeline
.StartWith(items)
.TransformInPlace(i =>
{
i.Id -= 1;
i.Name = $"Alice {i.Id}";
})
.LogWith(null)
.WriteTo(writer);
await pipeline.ExecuteAsync();
Assert.Equal(1000, writer.ItemsWritten.Count);
}
}
public void Pipeline_runif_does_not_run_operation_if_predicate_returns_false()
{
var wasRun = false;
var context = new EtlPipelineContext();
context.State["hello"] = "not world";
EtlPipeline.Create(settings => settings
.UseExistingContext(context)
.Named("Runif Test"))
.RunIf(ctx => (string)ctx.State["hello"] == "world",
ctx => new ActionEtlOperation(ctx2 =>
{
wasRun = true;
return(true);
})
)
.Execute();
wasRun.Should().BeFalse();
}
public async Task RunsProjectorStep()
{
using (var writer = new TestBulkWriter <PipelineTestsMyTestClass>())
{
var items = Enumerable.Range(1, 1000).Select(i => new PipelineTestsOtherTestClass {
Id = i, FirstName = "Bob", LastName = $"{i}"
});
var pipeline = EtlPipeline
.StartWith(items)
.Project(i => new PipelineTestsMyTestClass {
Id = i.Id, Name = $"{i.FirstName} {i.LastName}"
})
.WriteTo(writer);
await pipeline.ExecuteAsync();
Assert.Equal(1000, writer.ItemsWritten.Count);
Assert.Equal(1, writer.ItemsWritten[0].Id);
Assert.Equal("Bob 1", writer.ItemsWritten[0].Name);
Assert.Equal(1000, writer.ItemsWritten[999].Id);
Assert.Equal("Bob 1000", writer.ItemsWritten[999].Name);
}
}
public async Task RunsPivotStep()
{
using (var writer = new TestBulkWriter <PipelineTestsMyTestClass>())
{
var idCounter = 0;
var list = Enumerable
.Range(1, 10)
.ToList();
var items = list
.ToAsyncEnumerable();
var pipeline = EtlPipeline.StartWith(items)
.Pivot(i =>
{
var result = new List <PipelineTestsMyTestClass>();
for (var j = 0; j < i; j++)
{
++idCounter;
result.Add(new PipelineTestsMyTestClass {
Id = idCounter, Name = $"Bob {idCounter}"
});
}
return(result);
})
.WriteTo(writer);
await pipeline.ExecuteAsync();
var expectedSum = list.Sum();
Assert.Equal(expectedSum, writer.ItemsWritten.Count);
Assert.Equal(1, writer.ItemsWritten[0].Id);
Assert.Equal("Bob 1", writer.ItemsWritten[0].Name);
Assert.Equal(expectedSum, writer.ItemsWritten[expectedSum - 1].Id);
Assert.Equal($"Bob {expectedSum}", writer.ItemsWritten[expectedSum - 1].Name);
}
}
public void OnError_is_called_when_etl_operation_throws_exception()
{
var exception = new Exception("Whoops!");
var exceptionThrowingOp = new ExceptionThrowingEtlOperation(exception);
var errorHandlerCalled = false;
var pipeline = EtlPipeline.Create(settings => settings
.OnError((ctx, errors) =>
{
errorHandlerCalled = true;
errors.Count().Should().Be(1);
var err = errors.Single();
err.Exception.Should().Be(exception);
err.SourceOperation.Should().Be(exceptionThrowingOp);
err.HasSourceItem.Should().BeFalse();
err.SourceNode.Should().BeNull();
return(true);
})
)
.Run(exceptionThrowingOp)
.Execute();
errorHandlerCalled.Should().BeTrue();
}
public void Pipeline_aborts_when_error_encountered_when_executing_a_nested_pipeline()
{
var run1 = false;
var run2 = false;
var run3 = false;
var innerPipeline = EtlPipeline.Create(settings => settings
.Named("Pipeline 2"))
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
run2 = true;
throw new Exception("Uh oh!");
}));
var result = EtlPipeline.Create(settings => settings
.Named("Pipeline 1"))
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
run1 = true;
return(true);
}))
.Run(ctx => innerPipeline)
.Run(new ActionEtlOperation(ctx =>
{
run3 = true;
return(true);
}))
.Execute();
run1.Should().BeTrue();
run2.Should().BeTrue();
run3.Should().BeFalse();
result.Should().NotBeNull();
result.IsSuccess.Should().BeFalse();
}
public void Pipeline_can_execute_another_pipeline()
{
var run1 = false;
var run2 = false;
var run3 = false;
var innerPipeline = EtlPipeline.Create(settings => settings
.Named("Pipeline 2"))
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
run2 = true;
return(true);
}));
var result = EtlPipeline.Create(settings => settings
.Named("Pipeline 1"))
.Run(ctx => new ActionEtlOperation(ctx2 =>
{
run1 = true;
return(true);
}))
.Run(ctx => innerPipeline)
.Run(new ActionEtlOperation(ctx =>
{
run3 = true;
return(true);
}))
.Execute();
run1.Should().BeTrue();
run2.Should().BeTrue();
run3.Should().BeTrue();
result.Should().NotBeNull();
result.IsSuccess.Should().BeTrue();
}
public async Task LogsExceptionInPipelineStep()
{
using (var writer = new TestBulkWriter <PipelineTestsMyTestClass>())
{
var items = Enumerable.Range(1, 1000).Select(i => new PipelineTestsMyTestClass {
Id = i, Name = "Bob"
});
var loggerFactory = new FakeLoggerFactory();
var pipeline = EtlPipeline
.StartWith(items)
.Project <PipelineTestsMyTestClass>(i => throw new Exception("This is my exception"))
.LogWith(loggerFactory)
.WriteTo(writer);
var pipelineTask = pipeline.ExecuteAsync();
await Assert.ThrowsAsync <Exception>(() => pipelineTask);
var errorMessages = loggerFactory.LoggedMessages.Where(m => m.LogLevel == LogLevel.Error).ToList();
Assert.Single(errorMessages);
Assert.Contains("Error", errorMessages[0].Message);
Assert.Equal("This is my exception", errorMessages[0].Exception.Message);
}
}
public void OnError_is_called_when_etl_process_node_raises_error()
{
var exception = new Exception("Whoops!");
var exceptionThrowingOp = new ExceptionThrowingEtlOperation(exception);
var errorHandlerCalled = false;
var input = new List <Row>
{
new Row {
["number"] = 1
},
new Row {
["number"] = 2
}
};
var inputNode = new TestSourceNode((ctx, emitter) =>
{
foreach (var item in input)
{
emitter.Emit(item);
}
emitter.SignalEnd();
});
var context = new EtlPipelineContext();
var transformNode = new GenericTransformationNode <Row>((objects, row) => throw exception);
var process = EtlProcessBuilder.Create(context)
.Input(ctx => inputNode)
.Continue(ctx => transformNode)
.Complete(ctx => new TestSinkNode())
.Build();
var pipeline = EtlPipeline.Create(settings => settings
.UseExistingContext(context)
.OnError((ctx, errors) =>
{
errorHandlerCalled = true;
errors.Length.Should().Be(2);
errors[0].Exception.Should().Be(exception);
errors[0].SourceOperation.Should().Be(process);
errors[0].HasSourceItem.Should().BeTrue();
errors[0].SourceNode.Should().Be(transformNode);
errors[0].SourceItem.Should().Be(input[1]);
errors[1].Exception.Should().Be(exception);
errors[1].SourceOperation.Should().Be(process);
errors[1].HasSourceItem.Should().BeTrue();
errors[1].SourceNode.Should().Be(transformNode);
errors[1].SourceItem.Should().Be(input[0]);
return(true);
})
)
.Run(ctx => process)
.Execute();
errorHandlerCalled.Should().BeTrue();
}
public async Task LogsStartAndFinishOfPipelineSteps()
{
using (var writer = new TestBulkWriter <PipelineTestsOtherTestClass>())
{
var items = Enumerable
.Range(1, 1000)
.Select(i => new PipelineTestsMyTestClass {
Id = i, Name = "Bob"
})
.ToAsyncEnumerable();
var loggerFactory = new FakeLoggerFactory();
var pipeline = EtlPipeline
.StartWith(items)
.LogWith(loggerFactory)
.Aggregate(f =>
{
Thread.Sleep(1);
return(f.Max(c => c.Id));
})
.Pivot(i =>
{
var result = new List <PipelineTestsMyTestClass>();
for (var j = 1; j <= i; j++)
{
result.Add(new PipelineTestsMyTestClass {
Id = j, Name = $"Bob {j}"
});
}
return(result);
})
.Project(i =>
{
var nameParts = i.Name.Split(' ');
return(new PipelineTestsOtherTestClass {
Id = i.Id, FirstName = nameParts[0], LastName = nameParts[1]
});
})
.TransformInPlace(i =>
{
i.Id -= 1;
i.FirstName = "Alice";
i.LastName = $"{i.Id}";
})
.WriteTo(writer);
await pipeline.ExecuteAsync();
var totalStepsExpected = 6;
Assert.True(2 * totalStepsExpected == loggerFactory.LoggedMessages.Count, string.Join("\r\n", loggerFactory.LoggedMessages.Select(m => m.Message))); //2 log messages for each step in the pipeline
for (var i = 1; i <= totalStepsExpected; i++)
{
var messagesForStep = loggerFactory.LoggedMessages.Where(m => m.Message.Contains($"step {i} of {totalStepsExpected}")).ToList();
Assert.True(messagesForStep.Count == 2, $"Found {messagesForStep.Count} messages for step {i}");
Assert.Equal(1, messagesForStep.Count(m => m.Message.Contains("Starting")));
Assert.Equal(1, messagesForStep.Count(m => m.Message.Contains("Completing")));
}
}
}