public void SortSimpleDataDescending() { //Arrange TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("SortSourceNonGeneric"); source2Columns.InsertTestData(); DbSource <string[]> source = new DbSource <string[]>(Connection, "SortSourceNonGeneric"); //Act List <string[]> actual = new List <string[]>(); CustomDestination <string[]> dest = new CustomDestination <string[]>( row => actual.Add(row) ); Comparison <string[]> comp = new Comparison <string[]>( (x, y) => int.Parse(y[0]) - int.Parse(x[0]) ); Sort <string[]> block = new Sort <string[]>(comp); source.LinkTo(block); block.LinkTo(dest); source.Execute(); dest.Wait(); //Assert List <int> expected = new List <int>() { 3, 2, 1 }; Assert.Equal(expected, actual.Select(row => int.Parse(row[0])).ToList()); }
public void SortSimpleDataDescending() { //Arrange TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("SortSource"); source2Columns.InsertTestData(); DBSource <MySimpleRow> source = new DBSource <MySimpleRow>(Connection, "SortSource"); //Act List <MySimpleRow> actual = new List <MySimpleRow>(); CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>( row => actual.Add(row) ); Comparison <MySimpleRow> comp = new Comparison <MySimpleRow>( (x, y) => y.Col1 - x.Col1 ); Sort <MySimpleRow> block = new Sort <MySimpleRow>(comp); source.LinkTo(block); block.LinkTo(dest); source.Execute(); dest.Wait(); //Assert List <int> expected = new List <int>() { 3, 2, 1 }; Assert.Equal(expected, actual.Select(row => row.Col1).ToList()); }
public void InsertIntoTable() { //Arrange TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("Source"); source2Columns.InsertTestData(); TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CustomDestination"); //Act DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(SqlConnection, "Source"); CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>( row => { SqlTask.ExecuteNonQuery(SqlConnection, "Insert row", $"INSERT INTO dbo.CustomDestination VALUES({row.Col1},'{row.Col2}')"); } ); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void ColumnMapping(IConnectionManager connection) { //Arrange FourColumnsTableFixture source4Columns = new FourColumnsTableFixture(connection, "SourceColumnMapping", identityColumnIndex: 0); source4Columns.InsertTestData(); //Act DBSource <MyExtendedRow> source = new DBSource <MyExtendedRow>(connection, "SourceColumnMapping"); CustomDestination <MyExtendedRow> dest = new CustomDestination <MyExtendedRow>( input => { //Assert Assert.InRange(input.Id, 1, 3); Assert.StartsWith("Test", input.Text); if (input.Id == 1) { Assert.Null(input.Value); } else { Assert.True(input.Value > 0); } Assert.InRange(input.Percentage, 1, 2); }); source.LinkTo(dest); source.Execute(); dest.Wait(); }
public void InsertIntoTable() { //Arrange TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("CustomDestinationDynamicSource"); source2Columns.InsertTestData(); TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CustomDestinationDynamicDestination"); //Act DbSource <ExpandoObject> source = new DbSource <ExpandoObject>("CustomDestinationDynamicSource", SqlConnection); CustomDestination <ExpandoObject> dest = new CustomDestination <ExpandoObject>( row => { dynamic r = row as ExpandoObject; SqlTask.ExecuteNonQuery(SqlConnection, "Insert row", $"INSERT INTO dbo.CustomDestinationDynamicDestination VALUES({r.Col1},'{r.Col2}')"); } ); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void InsertIntoTable() { //Arrange TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("CustomDestinationNonGenericSource"); source2Columns.InsertTestData(); TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CustomDestinationNonGenericDestination"); //Act DbSource <string[]> source = new DbSource <string[]>(SqlConnection, "CustomDestinationNonGenericSource"); CustomDestination <string[]> dest = new CustomDestination <string[]>( row => { SqlTask.ExecuteNonQuery(SqlConnection, "Insert row", $"INSERT INTO dbo.CustomDestinationNonGenericDestination VALUES({row[0]},'{row[1]}')"); } ); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void TestDuplicateCheckInRowTrans() { CSVSource <Poco> source = new CSVSource <Poco>("src/DataFlowExamples/Duplicate.csv"); source.Configuration.Delimiter = ";"; source.Configuration.TrimOptions = CsvHelper.Configuration.TrimOptions.Trim; source.Configuration.MissingFieldFound = null; List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source RowTransformation <Poco, Poco> rowTrans = new RowTransformation <Poco, Poco>(input => { if (IDs.Contains(input.ID)) { input.IsDuplicate = true; } else { IDs.Add(input.ID); } return(input); }); var multicast = new Multicast <Poco>(); var dest = new DBDestination <Poco>("dbo.Staging"); TableDefinition stagingTable = new TableDefinition("dbo.Staging", new List <TableColumn>() { new TableColumn("Key", "INT", allowNulls: false, isPrimaryKey: true, isIdentity: true), new TableColumn("ID", "INT", allowNulls: false), new TableColumn("Value", "NVARCHAR(100)", allowNulls: false), new TableColumn("Name", "NVARCHAR(100)", allowNulls: false) }); stagingTable.CreateTable(); var trash = new CustomDestination <Poco>(input => { LogTask.Warn($"Duplicate found. ID: {input.ID} Name: {input.Name}"); }); source.LinkTo(rowTrans); rowTrans.LinkTo(multicast); multicast.LinkTo(dest, input => input.IsDuplicate == false); multicast.LinkTo(trash, input => input.IsDuplicate == true); source.Execute(); dest.Wait(); trash.Wait(); }
public void TestErrorLink() { //Arrange MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>(); source.DataAsList = new List <MySimpleRow>() { new MySimpleRow() { Col1 = 1, Col2 = "Test1" }, new MySimpleRow() { Col1 = 2, Col2 = "ErrorRecord" }, new MySimpleRow() { Col1 = 3, Col2 = "Test3" }, }; CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>( row => { if (row.Col1 == 2) { throw new Exception("Error record!"); } } ); MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>(); //Act source.LinkTo(dest); dest.LinkErrorTo(errorDest); source.Execute(); dest.Wait(); errorDest.Wait(); //Assert Assert.Collection <ETLBoxError>(errorDest.Data, d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)) ); }
public void DBSourceWithColumnMapping() { SqlTask.ExecuteNonQuery("Create source table", @"CREATE TABLE test.Source (Col1 nvarchar(100) null, Col2 nvarchar(100) null)"); SqlTask.ExecuteNonQuery("Insert demo data", "insert into test.Source values('Test1','Test2')"); SqlTask.ExecuteNonQuery("Insert demo data", "insert into test.Source values('Test1','Test2')"); SqlTask.ExecuteNonQuery("Insert demo data", "insert into test.Source values('Test1','Test2')"); DBSource <ColumnMapRow> source = new DBSource <ColumnMapRow>("test.Source"); CustomDestination <ColumnMapRow> dest = new CustomDestination <ColumnMapRow>( input => { Assert.AreEqual("Test1", input.Col1); Assert.AreEqual("Test2", input.B); }); source.LinkTo(dest); source.Execute(); dest.Wait(); }
public void DB_CustDest() { TableDefinition sourceTableDefinition = CreateSourceTable("test.Source"); TableDefinition destinationTableDefinition = CreateDestinationTable("test.Destination"); DBSource <MySimpleRow> source = new DBSource <MySimpleRow>() { SourceTableDefinition = sourceTableDefinition }; CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>( row => { SqlTask.ExecuteNonQuery("Insert row", $"insert into test.Destination values('{row.Value1}',{row.Value2})"); } ); source.LinkTo(dest); source.Execute(); dest.Wait(); Assert.AreEqual(3, RowCountTask.Count("test.Destination")); }
public void Table2JsonFile() { TableDefinition sourceTableDefinition = CreateSourceTable("test.Source"); DBSource <MySimpleRow> source = new DBSource <MySimpleRow>(sourceTableDefinition); List <MySimpleRow> rows = new List <MySimpleRow>(); CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>( row => { rows.Add(row); } ); source.LinkTo(dest); source.Execute(); dest.Wait(); string json = JsonConvert.SerializeObject(rows, Formatting.Indented); Assert.AreEqual(json, File.ReadAllText("src/DataFlow/json_tobe.json")); }
public void IgnoreWithObject() { //Arrange MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>(); source.Data = new List <MySimpleRow>() { null, new MySimpleRow() { Col1 = 1, Col2 = "Test1" }, null, new MySimpleRow() { Col1 = 2, Col2 = "Test2" }, new MySimpleRow() { Col1 = 3, Col2 = "Test3" }, null }; //Act List <MySimpleRow> result = new List <MySimpleRow>(); CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>( row => result.Add(row) ); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Collection(result, d => Assert.True(d.Col1 == 1 && d.Col2 == "Test1"), d => Assert.True(d.Col1 == 2 && d.Col2 == "Test2"), d => Assert.True(d.Col1 == 3 && d.Col2 == "Test3") ); }
public void ColumnMapping(IConnectionManager connection) { //Arrange TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture(connection, "Source"); source2Columns.InsertTestData(); //Act DBSource <ColumnMapRow> source = new DBSource <ColumnMapRow>(connection, "Source"); CustomDestination <ColumnMapRow> dest = new CustomDestination <ColumnMapRow>( input => { //Assert Assert.InRange(input.Col1, 1, 3); Assert.StartsWith("Test", input.B); }); source.LinkTo(dest); source.Execute(); dest.Wait(); }
public void CreateJsonFile() { //Arrange TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("JSonSource"); source2Columns.InsertTestData(); DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(SqlConnection, "JSonSource"); List <MySimpleRow> rows = new List <MySimpleRow>(); //Act CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>( row => rows.Add(row) ); source.LinkTo(dest); source.Execute(); dest.Wait(); //Act string json = JsonConvert.SerializeObject(rows, Formatting.Indented); //Assert Assert.Equal(File.ReadAllText("res/CustomDestination/simpleJson_tobe.json"), json); }
public void SortSimpleDataDescending() { //Arrange TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("SortSourceNonGeneric"); source2Columns.InsertTestData(); DbSource <ExpandoObject> source = new DbSource <ExpandoObject>(Connection, "SortSourceNonGeneric"); //Act List <ExpandoObject> actual = new List <ExpandoObject>(); CustomDestination <ExpandoObject> dest = new CustomDestination <ExpandoObject>( row => actual.Add(row) ); Comparison <ExpandoObject> comp = new Comparison <ExpandoObject>( (x, y) => { dynamic xo = x as ExpandoObject; dynamic yo = y as ExpandoObject; return(yo.Col1 - xo.Col1); } ); Sort <ExpandoObject> block = new Sort <ExpandoObject>(comp); source.LinkTo(block); block.LinkTo(dest); source.Execute(); dest.Wait(); //Assert List <int> expected = new List <int>() { 3, 2, 1 }; Assert.Equal(expected, actual.Select(row => { dynamic r = row as ExpandoObject; return(r.Col1); }).Cast <int>().ToList()); }