public void CSVAttributeNoHeaderTest() { var stream = new System.IO.MemoryStream(); var text = "A,B\r\nD,E"; var bytes = System.Text.Encoding.UTF8.GetBytes(text); stream.Write(bytes, 0, bytes.Length); stream.Position = 0; var target = new CSVSource<TestClass>(stream, System.Text.Encoding.UTF8); var firstLine = target.ReadNext(); Assert.AreEqual("A", firstLine.Col1); Assert.AreEqual("B", firstLine.Col2); //Without Index stream = new System.IO.MemoryStream(); text = "A,B\r\nD,E"; bytes = System.Text.Encoding.UTF8.GetBytes(text); stream.Write(bytes, 0, bytes.Length); stream.Position = 0; var target2 = new CSVSource<TestClass2>(stream, System.Text.Encoding.UTF8); var firstLine2 = target2.ReadNext(); Assert.IsNull(firstLine2.Col1); Assert.IsNull(firstLine2.Col2); }
public void CSVAttributeWithSkipRowCountOnReadingEmptyTextTest() { var stream = new System.IO.MemoryStream(); var text = ""; var bytes = System.Text.Encoding.UTF8.GetBytes(text); stream.Write(bytes, 0, bytes.Length); stream.Position = 0; var target = new CSVSource<TestClass3>(stream, System.Text.Encoding.UTF8); var firstLine = target.ReadNext(); Assert.IsNull(firstLine); }
public void WithoutErrorLinking() { //Arrange MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>(); //Act CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumnsErrorLinking.csv"); //Assert Assert.Throws <CsvHelper.TypeConversion.TypeConverterException>(() => { source.LinkTo(dest); source.Execute(); dest.Wait(); }); }
public void MoreColumnsInSource() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVSourceDynamicColsInSource"); DBDestination <ExpandoObject> dest = new DBDestination <ExpandoObject>(SqlConnection, "CSVSourceDynamicColsInSource"); //Act CSVSource <ExpandoObject> source = new CSVSource <ExpandoObject>("res/CSVSource/FourColumnsForDynamic.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void IdentityAtTheEnd() { //Arrange FourColumnsTableFixture dest4Columns = new FourColumnsTableFixture("CSVDestination4Columns", identityColumnIndex: 3); DBDestination dest = new DBDestination(Connection, "CSVDestination4Columns"); //Act CSVSource source = new CSVSource("res/CSVSource/ThreeColumnsNoId.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest4Columns.AssertTestData(); }
public void MoreColumnsInCSV() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVDestination2Columns"); DBDestination dest = new DBDestination(Connection, "CSVDestination2Columns"); //Act CSVSource source = new CSVSource("res/CSVSource/ThreeColumns.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void SimpleFlowWithObject() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVDestination2Columns"); DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "CSVDestination2Columns"); //Act CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumns.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
private static void Load_CSV_DB(List <TableColumn> columnsInStageTable) { TableDefinition stagingTable = new TableDefinition("test.Staging", columnsInStageTable); stagingTable.CreateTable(); CSVSource source = new CSVSource("src/DataFlow/Simple_CSV2DB.csv"); DBDestination <string[]> dest = new DBDestination <string[]>() { DestinationTableDefinition = stagingTable }; source.LinkTo(dest); source.Execute(); dest.Wait(); Assert.AreEqual(3, RowCountTask.Count("test.Staging", "Col1 Like '%ValueRow%'")); }
public void MoreColumnsInDatabase() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVDestination2Columns"); DBDestination dest = new DBDestination(Connection, "CSVDestination2Columns"); //Act CSVSource source = new CSVSource("res/CSVSource/OneColumn.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(3, RowCountTask.Count(Connection, "CSVDestination2Columns", "Col1 IN (1,2,3)")); }
public void CSVSourceNoHeader() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVSourceNoHeader"); DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "CSVSourceNoHeader"); //Act CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumnsNoHeader.csv"); source.Configuration.HasHeaderRecord = false; source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void CSVGenericWithSkipRows_DB() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVSourceSkipRows"); DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "CSVSourceSkipRows"); //Act CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumnsSkipRows.csv"); source.SkipRows = 2; source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void SimpleFlowUsingClassMap() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVDestination2Columns"); DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "CSVDestination2Columns"); //Act CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumns.csv"); source.Configuration.RegisterClassMap <ModelClassMap>(); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void CSV_DB_WithOdbcConnection() { ControlFlow.CurrentDbConnection = new OdbcConnectionManager(new OdbcConnectionString(OdbcConnectionStringParameter)); CreateSchemaTask.Create("test"); TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>() { new TableColumn("ID", "int", allowNulls: false, isPrimaryKey: true, isIdentity: true), new TableColumn("Col1", "bit", allowNulls: true), new TableColumn("Col2", "decimal(10,5)", allowNulls: true), new TableColumn("Col3", "tinyint", allowNulls: true), new TableColumn("Col4", "int", allowNulls: true), new TableColumn("Col5", "uniqueidentifier", allowNulls: true), new TableColumn("Col6", "nvarchar(100)", allowNulls: true) }); stagingTable.CreateTable(); CSVSource source = new CSVSource("src/ConnectionManager/DatatypeCSV.csv") { Configuration = new CsvHelper.Configuration.Configuration() { Quote = '"' } }; RowTransformation <string[], string[]> trans = new RowTransformation <string[], string[]>("Set empty values to null", row => { for (int i = 0; i < row.Length; i++) { if (row[i] == String.Empty) { row[i] = null; } } return(row); }); DBDestination <string[]> dest = new DBDestination <string[]>(stagingTable, 2); source.LinkTo(trans); trans.LinkTo(dest); source.Execute(); dest.Wait(); Assert.AreEqual(3, RowCountTask.Count(stagingTable.Name)); }
public void CSV2ACCESS_ViaOdbc() { ControlFlow.CurrentDbConnection = new AccessOdbcConnectionManager(new OdbcConnectionString(AccessConnectionStringParameter)) { AlwaysUseSameConnection = false }; TableDefinition testTable = RecreateTestTable(); CSVSource source = new CSVSource("src/ConnectionManager/AccessData.csv"); DBDestination <string[]> dest = new DBDestination <string[]>(testTable, 2); source.LinkTo(dest); source.Execute(); dest.Wait(); Assert.AreEqual(6, RowCountTask.Count(testTable.Name)); }
public void Multicast_Into2Tables() { var tableDestination1 = this.CreateTable("test.Table1"); var tableDestination2 = this.CreateTable("test.Table2"); var row1 = new RowTransformation <TestPoco, TestEntity1>(input => { return(new TestEntity1 { Col1 = input.Value1, Col3 = input.Value3 }); }); var row2 = new RowTransformation <TestPoco, TestEntity2>(input => { return(new TestEntity2 { Col2 = input.Value2, Col4 = input.Value4 }); }); var source = new CSVSource <TestPoco>("src/DataFlowExamples/Issue5.csv") { Configuration = new CsvHelper.Configuration.Configuration() { Delimiter = ";" } }; var multicast = new Multicast <TestPoco>(); var destination1 = new DBDestination <TestEntity1>("test.Table1"); var destination2 = new DBDestination <TestEntity2>("test.Table2"); source.LinkTo(multicast); multicast.LinkTo(row1); multicast.LinkTo(row2); row1.LinkTo(destination1); row2.LinkTo(destination2); source.Execute(); destination1.Wait(); destination2.Wait(); Assert.AreEqual(2, RowCountTask.Count("test.Table1", "Col1 in ('one','five') and Col3 in ('three','seven')")); Assert.AreEqual(2, RowCountTask.Count("test.Table2", "Col2 in ('two','six') and Col4 in ('four','eight')")); }
public void CSV_RowTrans_DB_NonGeneric() { SqlTask.ExecuteNonQuery("Create source table", @"CREATE TABLE test.Staging (Col1 int null, Col2 nvarchar(100) null)"); CSVSource source = new CSVSource("src/DataFlow/Simple_CSV2DB.csv"); RowTransformation trans = new RowTransformation( csvdata => { return(new string[] { csvdata[1], csvdata[0] }); }); DBDestination dest = new DBDestination("test.Staging"); source.LinkTo(trans); trans.LinkTo(dest); source.Execute(); dest.Wait(); Assert.AreEqual(3, RowCountTask.Count("test.Staging")); }
public void SimpleLookupWithDynamicObject(IConnectionManager connection) { //Arrange TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture(connection, "SourceLookupDynamicObject"); source2Columns.InsertTestData(); FourColumnsTableFixture dest4Columns = new FourColumnsTableFixture(connection, "DestinationLookupDynamicObject", -1); DBSource <ExpandoObject> source = new DBSource <ExpandoObject>(connection, "SourceLookupDynamicObject"); DBDestination <ExpandoObject> dest = new DBDestination <ExpandoObject>(connection, "DestinationLookupDynamicObject"); //Act List <ExpandoObject> lookupList = new List <ExpandoObject>(); CSVSource <ExpandoObject> lookupSource = new CSVSource <ExpandoObject>("res/Lookup/LookupSource.csv"); var lookup = new ETLBox.DataFlow.Lookup <ExpandoObject, ExpandoObject>( row => { dynamic r = row as ExpandoObject; r.Col3 = lookupList .Where(lkupRow => { dynamic lk = lkupRow as dynamic; return(int.Parse(lk.Key) == r.Col1); }) .Select(lkupRow => { dynamic lk = lkupRow as dynamic; return(lk.Column3 == string.Empty ? null : Int64.Parse(lk.Column3)); }) .FirstOrDefault(); r.Col4 = lookupList .Where(lkupRow => { dynamic lk = lkupRow as dynamic; return(int.Parse(lk.Key) == r.Col1); }) .Select(lkupRow => { dynamic lk = lkupRow as dynamic; return(double.Parse(lk.Column4)); }) .FirstOrDefault(); return(row); }, lookupSource, lookupList ); source.LinkTo(lookup); lookup.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest4Columns.AssertTestData(); }
public void CSVIntoAccess() { //Arrange TableDefinition testTable = RecreateAccessTestTable(); //Act CSVSource source = new CSVSource("res/UseCases/AccessData.csv"); DBDestination <string[]> dest = new DBDestination <string[]>(testTable, batchSize: 2) { ConnectionManager = AccessOdbcConnection }; source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(3, RowCountTask.Count(AccessOdbcConnection, testTable.Name)); }
/// <summary> /// Gets the right reader/writer TextSource depending of the extension. /// </summary> /// <param name="extension">The extension.</param> /// <returns></returns> public static TextSource GetSource(string extension) { TextSource source = null; switch (extension.ToLower().Remove(0, 1)) { case "csv": source = new CSVSource(); break; case "xlsx": source = new XLSXSource(); break; default: break; } return(source); }
public void UsingClassMapAndNoHeader() { //Arrange FourColumnsTableFixture d4c = new FourColumnsTableFixture("CSVDestination4ColumnsClassMap"); DBDestination <MyExtendedRow> dest = new DBDestination <MyExtendedRow>(Connection, "CSVDestination4ColumnsClassMap"); //Act CSVSource <MyExtendedRow> source = new CSVSource <MyExtendedRow>("res/CSVSource/FourColumnsInvalidHeader.csv"); source.Configuration.RegisterClassMap <ExtendedClassMap>(); source.Configuration.HasHeaderRecord = false; source.Configuration.ShouldSkipRecord = ShouldSkipRecordDelegate; source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert d4c.AssertTestData(); }
public void BigData_CSV_DB(int numberOfRows) { Stopwatch watch = new Stopwatch(); TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>() { new TableColumn("Col1", "nchar(1000)", allowNulls: false), new TableColumn("Col2", "nchar(1000)", allowNulls: false), new TableColumn("Col3", "nchar(1000)", allowNulls: false), new TableColumn("Col4", "nchar(1000)", allowNulls: false), }); stagingTable.CreateTable(); string fileName = "src/DataFlow/BigData_CSV2DB.csv"; BigDataHelper bigData = new BigDataHelper() { FileName = fileName, NumberOfRows = numberOfRows, TableDefinition = stagingTable }; watch.Start(); LogTask.Info($"Create .csv file {fileName} with {numberOfRows} Rows"); bigData.CreateBigDataCSV(); LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to create .csv file"); watch.Reset(); CSVSource source = new CSVSource(fileName); DBDestination <string[]> dest = new DBDestination <string[]>(1000) { DestinationTableDefinition = stagingTable }; source.LinkTo(dest); watch.Start(); source.Execute(); LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to read everything into memory (while constantly writing)"); LogTask.Info($"Already {RowCountTask.Count("test.Staging", RowCountOptions.QuickQueryMode)} inserted into table"); dest.Wait(); //TODO Wait should be part of source LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to write everything into database"); Assert.AreEqual(numberOfRows, SqlTask.ExecuteScalar <int>("Check staging table", $"select count(*) from test.Staging")); }
public void ThrowExceptionWithoutHandling() { //Arrange CSVSource source = new CSVSource("res/RowTransformation/TwoColumns.csv"); MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>(); //Act RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>( csvdata => throw new InvalidOperationException("Test")); source.LinkTo(trans); trans.LinkTo(dest); //Assert Assert.Throws <AggregateException>(() => { source.Execute(); dest.Wait(); }); }
private TimeSpan GetETLBoxTime <T>(int numberOfRows, CSVSource <T> source, MemoryDestination <T> dest) { source.LinkTo(dest); var timeElapsedETLBox = BigDataHelper.LogExecutionTime($"Copying Csv into DB (non generic) with {numberOfRows} rows of data using ETLBox", () => { source.Execute(); dest.Wait(); } ); if (typeof(T) == typeof(string[])) { output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Non generic)."); } else { output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Generic)."); } return(timeElapsedETLBox); }
public void DuplicateCheckWithBlockTrans() { //Arrange CSVSource <Poco> source = CreateDuplicateCSVSource("res/UseCases/DuplicateCheck.csv"); List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source //Act BlockTransformation <Poco> blockTrans = new BlockTransformation <Poco>(inputList => { return(inputList.GroupBy(item => item.ID).Select(y => y.First()).ToList()); }); DBDestination <Poco> dest = CreateDestinationTable("dbo.DuplicateCheck"); source.LinkTo(blockTrans); blockTrans.LinkTo(dest); source.Execute(); dest.Wait(); //Assert AssertDataWithoutDuplicates(); }
public void CSV_DB() { TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>() { new TableColumn("Col1", "nvarchar(100)", allowNulls: false), new TableColumn("Col2", "int", allowNulls: true) }); stagingTable.CreateTable(); CSVSource source = new CSVSource("DataFlow/Simple_CSV2DB.csv"); DBDestination <string[]> dest = new DBDestination <string[]>() { DestinationTableDefinition = stagingTable }; source.LinkTo(dest); source.Execute(); dest.Wait(); //TODO Wait should be part of source Assert.AreEqual(3, SqlTask.ExecuteScalar <int>("Check staging table", $"select count(*) from test.Staging where Col1 Like '%ValueRow%' and Col2 <> 1")); }
public void CSVGeneric_DB() { TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>() { new TableColumn("Col1", "nvarchar(100)", allowNulls: false), new TableColumn("Col2", "int", allowNulls: true) }); stagingTable.CreateTable(); CSVSource <CSVData> source = new CSVSource <CSVData>("src/DataFlow/Simple_CSV2DB.csv"); DBDestination <CSVData> dest = new DBDestination <CSVData>() { DestinationTableDefinition = stagingTable }; source.LinkTo(dest); source.Execute(); dest.Wait(); Assert.AreEqual(3, RowCountTask.Count("test.Staging", "Col1 Like '%ValueRow%' and Col2 <> 1")); }
public void ThrowExceptionInFlow() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("RowTransExceptionTest"); CSVSource source = new CSVSource("res/RowTransformation/TwoColumns.csv"); DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(SqlConnection, "RowTransExceptionTest"); CreateErrorTableTask.DropAndCreate(SqlConnection, "errors"); DBDestination <ETLBoxError> errorDest = new DBDestination <ETLBoxError>(SqlConnection, "errors"); //Act RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>( csvdata => { int no = int.Parse(csvdata[0]); if (no == 2) { throw new Exception("Test"); } return(new MySimpleRow() { Col1 = no, Col2 = csvdata[1] }); }); source.LinkTo(trans); trans.LinkTo(dest); trans.LinkErrorTo(errorDest); source.Execute(); dest.Wait(); errorDest.Wait(); //Assert Assert.Equal(2, RowCountTask.Count(SqlConnection, "RowTransExceptionTest")); Assert.Equal(1, RowCountTask.Count(SqlConnection, "errors")); }
public void RearrangeSwappedData() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation"); CSVSource source = new CSVSource("res/RowTransformation/TwoColumnsSwapped.csv"); //Act RowTransformation trans = new RowTransformation( csvdata => { return(new string[] { csvdata[1], csvdata[0] }); }); DBDestination dest = new DBDestination(Connection, "DestinationRowTransformation"); source.LinkTo(trans); trans.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void DuplicateCheckInRowTrans() { //Arrange CSVSource <Poco> source = CreateDuplicateCSVSource("res/UseCases/DuplicateCheck.csv"); List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source //Act RowTransformation <Poco, Poco> rowTrans = new RowTransformation <Poco, Poco>(input => { if (IDs.Contains(input.ID)) { input.IsDuplicate = true; } else { IDs.Add(input.ID); } return(input); }); Multicast <Poco> multicast = new Multicast <Poco>(); DBDestination <Poco> dest = CreateDestinationTable("dbo.DuplicateCheck"); VoidDestination <Poco> trash = new VoidDestination <Poco>(); source.LinkTo(rowTrans); rowTrans.LinkTo(multicast); multicast.LinkTo(dest, input => input.IsDuplicate == false); multicast.LinkTo(trash, input => input.IsDuplicate == true); source.Execute(); dest.Wait(); trash.Wait(); //Assert AssertDataWithoutDuplicates(); }
private void button1_Click(object sender, EventArgs e) { DataLoader loader; DataSource source; GroupSource group_source; try { loader = new NetworkDownloader(textBox1.Text); loader.Load(); source = new CSVSource(loader.GetData()); source.Load(); group_source = new GroupSource(textBox2.Text); group_source.Load(); }catch (Exception ex) { MessageBox.Show("Unable to download data: " + ex.Message); return; } listView1.Items.Clear(); this.Groups = group_source.Groups; CovidDataRow total = new CovidDataRow("Total"); foreach (var group in group_source.Groups) { var last_date = dateTimePicker1.Value; var data = group.GetDataForDate(last_date); this.DisplayData(group.Name, data); total += data; } this.DisplayData("Total:", total); }
public void CSVAttributeWithSkipRowCountTest() { var stream = new System.IO.MemoryStream(); var text = "\r\n\r\nCol1,Col2\r\nA,B\r\nD,E"; var bytes = System.Text.Encoding.UTF8.GetBytes(text); stream.Write(bytes, 0, bytes.Length); stream.Position = 0; var target = new CSVSource<TestClass3>(stream, System.Text.Encoding.UTF8); var firstLine = target.ReadNext(); Assert.AreEqual("A", firstLine.Col1); Assert.AreEqual("B", firstLine.Col2); }
public void TestSampleDataflow() { using (TestDb) { TableColumn keyCol = new TableColumn("Key", SqlDbType.Int, false, true, true); TableColumn col1 = new TableColumn("Col1", SqlDbType.NVarChar, 100, false); TableColumn col2 = new TableColumn("Col2", SqlDbType.NVarChar, 50, true); TableColumn col3 = new TableColumn("Col3", SqlDbType.Int, false); string destSchema = "test"; string destTable1 = "Staging1"; string destObject1 = $"[{destSchema}].[{destTable1}]"; new DropAndCreateTableTask(TestDb.getNewSqlConnection()).Execute(destSchema, destTable1, new List <TableColumn>() { keyCol, col1, col2, col3 }); SqlDestination <string[]> destination1 = new SqlDestination <string[]>(); destination1.ObjectName = destObject1; destination1.FieldCount = 4; destination1.ObjectMappingMethod = WriterAdapter_SampleDataflow.Fill; destination1.SqlConnection = TestDb.getNewSqlConnection(); string destTable2 = "Staging2"; string destObject2 = $"[{destSchema}].[{destTable2}]"; new DropAndCreateTableTask(TestDb.getNewSqlConnection()).Execute(destSchema, destTable2, new List <TableColumn>() { keyCol, col1, col2, col3 }); SqlDestination <string[]> destination2 = new SqlDestination <string[]>(); destination2.ObjectName = destObject2; destination2.FieldCount = 4; destination2.ObjectMappingMethod = WriterAdapter_SampleDataflow.Fill; destination2.SqlConnection = TestDb.getNewSqlConnection(); CSVSource <string[]> CSVSource = new CSVSource <string[]>("DataFlow/InputData.csv"); Graph g = new Graph(); g.GetVertex(0, CSVSource); g.GetVertex(1, new RowTransformation <string[]>(RowTransformation1)); g.GetVertex(11, new RowTransformation <string[]>(RowTransformation2)); g.GetVertex(10, new BroadCast <string[]>(CloneTransformation1)); g.GetVertex(12, new RowTransformationMany <string[]>(RowTransformationMany)); g.GetVertex(20, new RowTransformation <string[]>(RowTransformation3)); g.GetVertex(100, destination1); g.GetVertex(110, destination2); Edge e1 = g.AddEdge(0, 1); // connect 0 to 1 Edge e2 = g.AddEdge(1, 10); // connect 1 to 10 Edge e3 = g.AddEdge(10, 20); Edge e4 = g.AddEdge(20, 100); Edge e5 = g.AddEdge(10, 11); Edge e6 = g.AddEdge(11, 12); Edge e7 = g.AddEdge(12, 110); DataFlowTask <string[]> .Execute("Test dataflow task", 1000, 1, g); e2.cost = counter_RowTransformation1; e6.cost = counter_RowTransformation2; e4.cost = counter_RowTransformation3; e7.cost = counter_RowTransformationMany; //TestHelper.VisualizeGraph(g); Assert.AreEqual(4, new ExecuteSQLTask(TestDb.getNewSqlConnection()).ExecuteScalar(string.Format("select count(*) from {0}", destObject1))); Assert.AreEqual(8, new ExecuteSQLTask(TestDb.getNewSqlConnection()).ExecuteScalar(string.Format("select count(*) from {0}", destObject2))); } }