public void ExceptionInAggregationFunction() { //Arrange MemorySource <MyRow> source = new MemorySource <MyRow>(); source.DataAsList = new List <MyRow>() { new MyRow { Id = 1, DetailValue = 3.5 }, }; //Act Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>( (row, aggRow) => throw new Exception("Test") ); MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>(); //Assert source.LinkTo(agg); agg.LinkTo(dest); Assert.Throws <AggregateException>(() => { source.Execute(); dest.Wait(); }); }
public void DbDestination() { //Arrange string[] data = { "1", "2" }; MemorySource <string[]> source = new MemorySource <string[]>(); source.DataAsList.Add(data); DbDestination <string[]> dest = new DbDestination <string[]>("test"); source.LinkTo(dest); //Act & Assert Assert.Throws <ETLBoxException>(() => { try { source.Execute(); dest.Wait(); } catch (AggregateException e) { throw e.InnerException; } }); }
public void IgnoreWithStringArray() { //Arrange MemorySource source = new MemorySource(); source.Data = new List <string[]>() { null, new string[] { "1", "Test1" }, null, new string[] { "2", "Test2" }, new string[] { "3", "Test3" }, null }; //Act JsonDestination dest = new JsonDestination("./IgnoreNullValuesStringArray.json"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(File.ReadAllText("./IgnoreNullValuesStringArray.json"), File.ReadAllText("res/JsonDestination/TwoColumnsStringArray.json")); }
public void UsingRowTransformation() { Prepare(); var orderSource = new MemorySource <Order>(); orderSource.DataAsList.Add(new Order() { OrderNumber = 815, CustomerName = "John" }); orderSource.DataAsList.Add(new Order() { OrderNumber = 4711, CustomerName = "Jim" }); var rowTrans = new RowTransformation <Order>( row => { int?id = SqlTask.ExecuteScalar <int>(SqlConnection, sql: $"SELECT Id FROM CustomerTable WHERE Name='{row.CustomerName}'"); row.CustomerId = id; return(row); }); /* Delete below here */ var dest = new MemoryDestination <Order>(); orderSource.LinkTo(rowTrans).LinkTo(dest); orderSource.Execute(); }
public void UnknownTableViaTableDefinition() { //Arrange TableDefinition def = new TableDefinition("UnknownTable", new List <TableColumn>() { new TableColumn("id", "INT") }); //Arrange string[] data = { "1", "2" }; MemorySource <string[]> source = new MemorySource <string[]>(); source.Data.Add(data); DbDestination <string[]> dest = new DbDestination <string[]>(def, SqlConnection); source.LinkTo(dest); //Act & Assert Assert.Throws <System.InvalidOperationException>(() => { try { source.Execute(); dest.Wait(); } catch (AggregateException e) { throw e.InnerException; } }); }
public void ValidateSchemaForDynamicObject() { //Arrange var source = new MemorySource(); dynamic n1 = new ExpandoObject(); n1.Xml = _validXml; source.DataAsList.Add(n1); dynamic n2 = new ExpandoObject(); n2.Xml = _validXml; source.DataAsList.Add(n2); dynamic n3 = new ExpandoObject(); n3.Xml = _invalidXml; source.DataAsList.Add(n3); MemoryDestination dest = new MemoryDestination(); MemoryDestination <ETLBoxError> error = new MemoryDestination <ETLBoxError>(); //Act XmlSchemaValidation schemaValidation = new XmlSchemaValidation(); schemaValidation.XmlSelector = row => { dynamic r = row as ExpandoObject; return(r.Xml); }; schemaValidation.XmlSchema = xsdMarkup; source.LinkTo(schemaValidation); schemaValidation.LinkTo(dest); schemaValidation.LinkErrorTo(error); source.Execute(); dest.Wait(); error.Wait(); //Assert Assert.True(dest.Data.Count == 2); Assert.True(error.Data.Count == 1); }
public void ExceptionInStoreKeyFunction() { //Arrange MemorySource <MyRow> source = new MemorySource <MyRow>(); source.DataAsList = new List <MyRow>() { new MyRow { Id = 1, ClassName = "Class1", DetailValue = 3.5 } }; //Act Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>( (row, aggValue) => aggValue.AggValue += row.DetailValue, row => row.ClassName, (key, agg) => throw new Exception("Test") ); MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>(); source.LinkTo(agg); agg.LinkTo(dest); //Assert Assert.Throws <AggregateException>(() => { source.Execute(); dest.Wait(); }); }
public void NoErrorHandling() { //Arrange MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>(); source.DataAsList = new List <MySimpleRow>() { new MySimpleRow() { Col1 = "X" }, new MySimpleRow() { Col1 = "1" }, new MySimpleRow() { Col1 = null } }; CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("ErrorFileNoError.csv"); //Act //Assert Assert.ThrowsAny <Exception>(() => { source.LinkTo(dest); source.Execute(); dest.Wait(); }); }
public void WriteWithObject() { //Arrange var source = new MemorySource <MyTextRow>(); source.DataAsList.Add(new MyTextRow() { Text = "Line 1" }); source.DataAsList.Add(new MyTextRow() { Text = "Line 2" }); source.DataAsList.Add(new MyTextRow() { Text = "Line 3" }); //Act TextDestination <MyTextRow> dest = new TextDestination <MyTextRow>("res/TextDestination/TestFile.txt" , tr => tr.Text); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.True(File.Exists("res/TextDestination/TestFile.txt")); Assert.Equal(File.ReadAllText("res/TextDestination/ToBeTestFile.txt"), File.ReadAllText("res/TextDestination/TestFile.txt")); }
private static void RunExceptionFlowWithType <T>() { //Arrange MemorySource <InputDataRow> source = new MemorySource <InputDataRow>(); source.DataAsList.Add(new InputDataRow() { LookupId = 1 }); MemorySource <T> lookupSource = new MemorySource <T>(); var lookup = new LookupTransformation <InputDataRow, T>(lookupSource); MemoryDestination <InputDataRow> dest = new MemoryDestination <InputDataRow>(); source.LinkTo(lookup); lookup.LinkTo(dest); //Act && Assert Assert.Throws <ETLBoxException>(() => { try { source.Execute(); dest.Wait(); } catch (AggregateException e) { throw e.InnerException; } }); }
public void AggregateSimple() { //Arrange MemorySource <MyRow> source = new MemorySource <MyRow>(); source.DataAsList = new List <MyRow>() { new MyRow { Id = 1, DetailValue = 3.5 }, new MyRow { Id = 2, DetailValue = 4.5 }, new MyRow { Id = 3, DetailValue = 2.0 }, }; Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>( (row, aggRow) => aggRow.AggValue += row.DetailValue ); MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>(); //Act source.LinkTo(agg); agg.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Collection <MyAggRow>(dest.Data, ar => Assert.True(ar.AggValue == 10) ); }
public void NoLookupSource() { //Arrange MemorySource <MyDataRow> source = new MemorySource <MyDataRow>(); source.DataAsList.Add(new MyDataRow() { Col1 = 1, Col2 = "Test1" }); //Act var lookup = new LookupTransformation <MyDataRow, MyLookupRow>(); MemoryDestination <MyDataRow> dest = new MemoryDestination <MyDataRow>(); //Assert Assert.Throws <ETLBoxException>(() => { try { source.LinkTo(lookup); lookup.LinkTo(dest); source.Execute(); dest.Wait(); } catch (AggregateException e) { throw e.InnerException; } }); //Assert }
public void DataIsFromList() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("MemoryDestination"); MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>(); DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(SqlConnection, "MemoryDestination"); //Act source.Data = new List <MySimpleRow>() { new MySimpleRow() { Col1 = 1, Col2 = "Test1" }, new MySimpleRow() { Col1 = 2, Col2 = "Test2" }, new MySimpleRow() { Col1 = 3, Col2 = "Test3" } }; source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void GroupingUsingStringArray() { //Arrange MemorySource <string[]> source = new MemorySource <string[]>(); source.DataAsList.Add(new string[] { "Class1", "3.5" }); source.DataAsList.Add(new string[] { "Class1", "6.5" }); source.DataAsList.Add(new string[] { "Class2", "10" }); Aggregation <string[], MyAggRow> agg = new Aggregation <string[], MyAggRow>( (row, aggValue) => aggValue.AggValue += Convert.ToDouble(row[1]), row => row[0], (key, agg) => agg.GroupName = (string)key ); MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>(); //Act source.LinkTo(agg); agg.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Collection <MyAggRow>(dest.Data, ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class1"), ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class2") ); }
public void IgnoreWithObject() { //Arrange TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(SqlConnection, "DestIgnoreNullValues"); MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>(); source.DataAsList = new List <MySimpleRow>() { null, new MySimpleRow() { Col1 = 1, Col2 = "Test1" }, null, new MySimpleRow() { Col1 = 2, Col2 = "Test2" }, new MySimpleRow() { Col1 = 3, Col2 = "Test3" }, null }; DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(SqlConnection, "DestIgnoreNullValues"); //Act source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert d2c.AssertTestData(); }
public void WriteIntoHttpClient() { //Arrange Mock<HttpMessageHandler> handlerMock = CreateHandlerMoq(); HttpClient httpClient = CreateHttpClient(handlerMock); MemorySource<MySimpleRow> source = new MemorySource<MySimpleRow>(); source.DataAsList.Add(new MySimpleRow() { Col1 = 1, Col2 = "Test1" }); //Act JsonDestination<MySimpleRow> dest = new JsonDestination<MySimpleRow>("http://test.test", ResourceType.Http); dest.HttpClient = httpClient; source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert handlerMock.Protected().Verify( "SendAsync", Times.Exactly(1), ItExpr.Is<HttpRequestMessage>(req => req.Method == HttpMethod.Get && req.RequestUri.Equals(new Uri("http://test.test")) ), ItExpr.IsAny<CancellationToken>() ); }
public void IgnoreWithObject() { //Arrange MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>(); source.DataAsList = new List <MySimpleRow>() { null, new MySimpleRow() { Col1 = 1, Col2 = "Test1" }, null, new MySimpleRow() { Col1 = 2, Col2 = "Test2" }, new MySimpleRow() { Col1 = 3, Col2 = "Test3" }, null }; //Act CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("./IgnoreNullValues.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(File.ReadAllText("./IgnoreNullValues.csv"), File.ReadAllText("res/CsvDestination/TwoColumns.csv")); }
public void IgnoreWithStringArray() { //Arrange MemorySource <string[]> source = new MemorySource <string[]>(); source.DataAsList = new List <string[]>() { null, new string[] { "1", "Test1" }, null, new string[] { "2", "Test2" }, new string[] { "3", "Test3" }, null }; //Act CsvDestination <string[]> dest = new CsvDestination <string[]>("./IgnoreNullValuesStringArray.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(File.ReadAllText("./IgnoreNullValuesStringArray.csv"), File.ReadAllText("res/CsvDestination/TwoColumnsNoHeader.csv")); }
public void DeltaLoadWithDeletion() { //Arrange MemorySource source = new MemorySource(); source.DataAsList.Add(CreateDynamicRow(2, "Test2")); source.DataAsList.Add(CreateDynamicRow(3, "Test3")); source.DataAsList.Add(CreateDynamicRow(4, delete: true)); source.DataAsList.Add(CreateDynamicRow(10, delete: true)); TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(SqlConnection, "DBMergeDynamicDeltaDestination"); d2c.InsertTestDataSet3(); //Act DbMerge dest = new DbMerge(SqlConnection, "DBMergeDynamicDeltaDestination") { DeltaMode = MergeMode.Delta }; dest.MergeProperties.IdPropertyNames.Add("Col1"); dest.MergeProperties.ComparePropertyNames.Add("Col2"); dest.MergeProperties.DeletionProperties.Add("Delete", true); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert d2c.AssertTestData(); Assert.Collection <ExpandoObject>(dest.DeltaTable, row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Update && r.Col1 == 2); }, row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Insert && r.Col1 == 3); }, row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Delete && r.Col1 == 4); }, row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Delete && r.Col1 == 10); } ); }
public void ValidateSchemaForArray() { //Arrange var source = new MemorySource <string[]>(); source.DataAsList.Add(new string[] { _validXml }); source.DataAsList.Add(new string[] { _invalidXml }); source.DataAsList.Add(new string[] { _validXml }); MemoryDestination <string[]> dest = new MemoryDestination <string[]>(); MemoryDestination <ETLBoxError> error = new MemoryDestination <ETLBoxError>(); //Act XmlSchemaValidation <string[]> schemaValidation = new XmlSchemaValidation <string[]>(); schemaValidation.XmlSelector = row => row[0]; schemaValidation.XmlSchema = xsdMarkup; source.LinkTo(schemaValidation); schemaValidation.LinkTo(dest); schemaValidation.LinkErrorTo(error); source.Execute(); dest.Wait(); error.Wait(); //Assert Assert.True(dest.Data.Count == 2); Assert.True(error.Data.Count == 1); }
public void SimpleMergeWithDynamic() { //Arrange MemorySource source = new MemorySource(); source.DataAsList.Add(CreateDynamicRow(1, "Test1")); source.DataAsList.Add(CreateDynamicRow(2, "Test2")); source.DataAsList.Add(CreateDynamicRow(3, "Test3")); TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(SqlConnection, "DBMergeDynamicDestination"); d2c.InsertTestDataSet3(); //Act DbMerge dest = new DbMerge(SqlConnection, "DBMergeDynamicDestination"); dest.MergeProperties.IdPropertyNames.Add("Col1"); dest.MergeProperties.ComparePropertyNames.Add("Col2"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(3, RowCountTask.Count(SqlConnection, "DBMergeDynamicDestination", $"{d2c.QB}Col1{d2c.QE} BETWEEN 1 AND 7 AND {d2c.QB}Col2{d2c.QE} LIKE 'Test%'")); d2c.AssertTestData(); Assert.Collection <ExpandoObject>(dest.DeltaTable, row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Exists && r.Col1 == 1); }, row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Update && r.Col1 == 2); }, row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Insert && r.Col1 == 3); }, row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Delete && r.Col1 == 4); }, row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Delete && r.Col1 == 10); } ); }
public void MixedTypes(IConnectionManager connection) { CreateTestTable(connection, "datatypedestination"); //Arrange MemorySource <MyDataTypeRow> source = new MemorySource <MyDataTypeRow>( new List <MyDataTypeRow>() { new MyDataTypeRow() { IntCol = 1, LongCol = -1, DecimalCol = 2.3M, DoubleCol = 5.4, DateTimeCol = new DateTime(2010, 1, 1, 10, 10, 10), DateCol = new DateTime(2020, 1, 1), StringCol = "Test", CharCol = 'T', DecimalStringCol = "13.4566", NullCol = null, EnumCol = EnumType.Value2 } }); //Act DbDestination <MyDataTypeRow> dest = new DbDestination <MyDataTypeRow>(connection, "datatypedestination"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert AssertFirstRow(connection, "datatypedestination"); }
public void MixedTypesWithDynamic(IConnectionManager connection) { CreateTestTable(connection, "datatypedestinationdynamic"); //Arrange MemorySource source = new MemorySource(); dynamic d1 = new ExpandoObject(); d1.IntCol = 1; d1.LongCol = -1; d1.DecimalCol = 2.3M; d1.DoubleCol = 5.4; d1.DateTimeCol = new DateTime(2010, 1, 1, 10, 10, 10); d1.DateCol = new DateTime(2020, 1, 1); d1.StringCol = "Test"; d1.CharCol = 'T'; d1.DecimalStringCol = "13.4566"; d1.NullCol = null; d1.EnumCol = EnumType.Value2; //Act DbDestination dest = new DbDestination(connection, "datatypedestinationdynamic"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert AssertFirstRow(connection, "datatypedestinationdynamic"); }
public void UnknownTable() { //Arrange string[] data = { "1", "2" }; MemorySource source = new MemorySource(); source.Data.Add(data); DBDestination dest = new DBDestination(SqlConnection, "UnknownTable"); source.LinkTo(dest); //Act & Assert Assert.Throws <ETLBoxException>(() => { try { source.Execute(); dest.Wait(); } catch (AggregateException e) { throw e.InnerException; } }); }
public void IgnoreWithStringArray() { //Arrange TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(SqlConnection, "DestIgnoreNullValuesStringArray"); MemorySource <string[]> source = new MemorySource <string[]>(); source.DataAsList = new List <string[]>() { null, new string[] { "1", "Test1" }, null, new string[] { "2", "Test2" }, new string[] { "3", "Test3" }, null }; DbDestination <string[]> dest = new DbDestination <string[]>(SqlConnection, "DestIgnoreNullValuesStringArray"); //Act source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert d2c.AssertTestData(); }
public void MixedTypes(IConnectionManager connection) { CreateTableTask.Create(connection, "datatypedestination", new List <TableColumn>() { new TableColumn("IntCol", "INT", allowNulls: true), new TableColumn("LongCol", "BIGINT", allowNulls: true), new TableColumn("DecimalCol", "FLOAT", allowNulls: true), new TableColumn("DoubleCol", "FLOAT", allowNulls: true), new TableColumn("DateTimeCol", "DATETIME", allowNulls: true), new TableColumn("DateCol", "DATE", allowNulls: true), new TableColumn("StringCol", "VARCHAR(200)", allowNulls: true), new TableColumn("CharCol", "CHAR(1)", allowNulls: true), new TableColumn("DecimalStringCol", "DECIMAL(12,10)", allowNulls: true), new TableColumn("NullCol", "CHAR(1)", allowNulls: true), }); //Arrange MemorySource <MyDataTypeRow> source = new MemorySource <MyDataTypeRow>( new List <MyDataTypeRow>() { new MyDataTypeRow() { IntCol = 1, LongCol = -1, DecimalCol = 2.3M, DoubleCol = 5.4, DateTimeCol = new DateTime(2010, 1, 1, 10, 10, 10), DateCol = new DateTime(2020, 1, 1), StringCol = "Test", CharCol = 'T', DecimalStringCol = "13.4566", NullCol = null } }); //Act DbDestination <MyDataTypeRow> dest = new DbDestination <MyDataTypeRow>("datatypedestination", connection); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert // IntCol LongCol DecimalCol DoubleCol DateTimeCol DateCol StringCol CharCol DecimalStringCol NullCol //1 - 1 2.3 5.4 2010 - 01 - 01 10:10:10.100 2020 - 01 - 01 Test T 13.4566000000 NULL SqlTask.ExecuteReaderSingleLine(connection, "Check data", "SELECT * FROM datatypedestination", col => Assert.True(Convert.ToInt32(col) == 1), col => Assert.True(Convert.ToInt64(col) == -1), col => Assert.True(Convert.ToDecimal(col) == 2.3M), col => Assert.True(Convert.ToDecimal(col) == 5.4M), col => Assert.True(Convert.ToDateTime(col) == new DateTime(2010, 1, 1, 10, 10, 10)), col => Assert.True(Convert.ToDateTime(col) == new DateTime(2020, 1, 1)), col => Assert.True(Convert.ToString(col) == "Test"), col => Assert.True(Convert.ToString(col) == "T" || Convert.ToString(col) == "84"), col => Assert.True(Convert.ToString(col).Replace("0", "") == "13.4566"), col => Assert.True(col == null) ); }
public void GroupingAndKeepingKey() { //Arrange MemorySource <MyRow> source = new MemorySource <MyRow>(); source.Data = new List <MyRow>() { new MyRow { Id = 1, ClassName = "Class1", DetailValue = 3.5 }, new MyRow { Id = 2, ClassName = "Class1", DetailValue = 6.5 }, new MyRow { Id = 3, ClassName = "Class2", DetailValue = 1.2 }, new MyRow { Id = 4, ClassName = "Class2", DetailValue = 2.3 }, new MyRow { Id = 5, ClassName = "Class2", DetailValue = 16.5 }, new MyRow { Id = 6, ClassName = "Class3", DetailValue = 30.0 }, new MyRow { Id = 6, ClassName = null, DetailValue = 14.5 }, new MyRow { Id = 6, ClassName = null, DetailValue = 15.5 }, }; Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>( (row, aggValue) => aggValue.AggValue += row.DetailValue, row => row.ClassName, (key, agg) => agg.GroupName = (string)key ); MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>(); //Act source.LinkTo(agg); agg.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Collection <MyAggRow>(dest.Data, ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class1"), ar => Assert.True(ar.AggValue == 20 && ar.GroupName == "Class2"), ar => Assert.True(ar.AggValue == 30 && ar.GroupName == "Class3"), ar => Assert.True(ar.AggValue == 30 && ar.GroupName == string.Empty) ); }
public void WaitForLongRunningProcedure() { //Arrange CreateProcedureTask.CreateOrAlter(SqlConnection, new ProcedureDefinition() { Name = "longrunning", Definition = @" WAITFOR DELAY '00:00:01'; SELECT @input + 10;", Parameter = new List <ProcedureParameter>() { new ProcedureParameter() { Name = "input", DataType = "INT" } // new ProcedureParameter() { Name ="result", Out = true, DataType="INT"} } }); var source = new MemorySource <MySimpleRow>(); source.DataAsList.Add(new MySimpleRow() { Col1 = 1, Col2 = "Test1" }); source.DataAsList.Add(new MySimpleRow() { Col1 = 2, Col2 = "Test2" }); source.DataAsList.Add(new MySimpleRow() { Col1 = 3, Col2 = "Test3" }); var dest = new MemoryDestination <MySimpleRow>(); var trans = new RowTransformation <MySimpleRow>( row => { row.Col1 = SqlTask.ExecuteScalar <int>(SqlConnection, "Read from procedure", $"EXEC longrunning @input = {row.Col1}") ?? 0; return(row); } ); //Act source.LinkTo(trans); trans.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Collection <MySimpleRow>(dest.Data, row => Assert.Equal(11, row.Col1), row => Assert.Equal(12, row.Col1), row => Assert.Equal(13, row.Col1) ); }
public void RedirectBatch(IConnectionManager connection) { //Arrange TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DestLinkError"); MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>(); source.Data = new List <MySimpleRow>() { new MySimpleRow() { Col1 = null, Col2 = "ErrorRecord" }, new MySimpleRow() { Col1 = "X2", Col2 = "ErrorRecord" }, new MySimpleRow() { Col1 = "1", Col2 = "Test1" }, new MySimpleRow() { Col1 = "2", Col2 = "Test2" }, new MySimpleRow() { Col1 = "3", Col2 = "Test3 - good, but in error batch" }, new MySimpleRow() { Col1 = null, Col2 = "ErrorRecord" }, new MySimpleRow() { Col1 = "3", Col2 = "Test3" }, }; DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(connection, "DestLinkError", batchSize: 2); MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>(); //Act source.LinkTo(dest); dest.LinkErrorTo(errorDest); source.Execute(); dest.Wait(); errorDest.Wait(); //Assert d2c.AssertTestData(); Assert.Collection <ETLBoxError>(errorDest.Data, d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)), d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)) ); }
private void TransferTestDataIntoDestination(List <MyMergeRow> knownGuids) { MemorySource <MyMergeRow> source = new MemorySource <MyMergeRow>(); source.Data = knownGuids; DbDestination <MyMergeRow> dest = new DbDestination <MyMergeRow>(SqlConnection, "MergeDestination"); source.LinkTo(dest); source.Execute(); dest.Wait(); }