public void CSVAttributeNoHeaderTest()
{
var stream = new System.IO.MemoryStream();
var text = "A,B\r\nD,E";
var bytes = System.Text.Encoding.UTF8.GetBytes(text);
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var target = new CSVSource<TestClass>(stream, System.Text.Encoding.UTF8);
var firstLine = target.ReadNext();
Assert.AreEqual("A", firstLine.Col1);
Assert.AreEqual("B", firstLine.Col2);
//Without Index
stream = new System.IO.MemoryStream();
text = "A,B\r\nD,E";
bytes = System.Text.Encoding.UTF8.GetBytes(text);
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var target2 = new CSVSource<TestClass2>(stream, System.Text.Encoding.UTF8);
var firstLine2 = target2.ReadNext();
Assert.IsNull(firstLine2.Col1);
Assert.IsNull(firstLine2.Col2);
}
public void CSVAttributeWithSkipRowCountOnReadingEmptyTextTest()
{
var stream = new System.IO.MemoryStream();
var text = "";
var bytes = System.Text.Encoding.UTF8.GetBytes(text);
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var target = new CSVSource<TestClass3>(stream, System.Text.Encoding.UTF8);
var firstLine = target.ReadNext();
Assert.IsNull(firstLine);
}
public void WithoutErrorLinking()
{
//Arrange
MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();
//Act
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumnsErrorLinking.csv");
//Assert
Assert.Throws <CsvHelper.TypeConversion.TypeConverterException>(() =>
{
source.LinkTo(dest);
source.Execute();
dest.Wait();
});
}
public void MoreColumnsInSource()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVSourceDynamicColsInSource");
DBDestination <ExpandoObject> dest = new DBDestination <ExpandoObject>(SqlConnection, "CSVSourceDynamicColsInSource");
//Act
CSVSource <ExpandoObject> source = new CSVSource <ExpandoObject>("res/CSVSource/FourColumnsForDynamic.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void IdentityAtTheEnd()
{
//Arrange
FourColumnsTableFixture dest4Columns = new FourColumnsTableFixture("CSVDestination4Columns", identityColumnIndex: 3);
DBDestination dest = new DBDestination(Connection, "CSVDestination4Columns");
//Act
CSVSource source = new CSVSource("res/CSVSource/ThreeColumnsNoId.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest4Columns.AssertTestData();
}
public void MoreColumnsInCSV()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVDestination2Columns");
DBDestination dest = new DBDestination(Connection, "CSVDestination2Columns");
//Act
CSVSource source = new CSVSource("res/CSVSource/ThreeColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void SimpleFlowWithObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVDestination2Columns");
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "CSVDestination2Columns");
//Act
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
private static void Load_CSV_DB(List <TableColumn> columnsInStageTable)
{
TableDefinition stagingTable = new TableDefinition("test.Staging", columnsInStageTable);
stagingTable.CreateTable();
CSVSource source = new CSVSource("src/DataFlow/Simple_CSV2DB.csv");
DBDestination <string[]> dest = new DBDestination <string[]>()
{
DestinationTableDefinition = stagingTable
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.AreEqual(3, RowCountTask.Count("test.Staging", "Col1 Like '%ValueRow%'"));
}
public void MoreColumnsInDatabase()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVDestination2Columns");
DBDestination dest = new DBDestination(Connection, "CSVDestination2Columns");
//Act
CSVSource source = new CSVSource("res/CSVSource/OneColumn.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(Connection, "CSVDestination2Columns", "Col1 IN (1,2,3)"));
}
public void CSVSourceNoHeader()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVSourceNoHeader");
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "CSVSourceNoHeader");
//Act
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumnsNoHeader.csv");
source.Configuration.HasHeaderRecord = false;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CSVGenericWithSkipRows_DB()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVSourceSkipRows");
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "CSVSourceSkipRows");
//Act
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumnsSkipRows.csv");
source.SkipRows = 2;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void SimpleFlowUsingClassMap()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVDestination2Columns");
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "CSVDestination2Columns");
//Act
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumns.csv");
source.Configuration.RegisterClassMap <ModelClassMap>();
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CSV_DB_WithOdbcConnection()
{
ControlFlow.CurrentDbConnection = new OdbcConnectionManager(new OdbcConnectionString(OdbcConnectionStringParameter));
CreateSchemaTask.Create("test");
TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>()
{
new TableColumn("ID", "int", allowNulls: false, isPrimaryKey: true, isIdentity: true),
new TableColumn("Col1", "bit", allowNulls: true),
new TableColumn("Col2", "decimal(10,5)", allowNulls: true),
new TableColumn("Col3", "tinyint", allowNulls: true),
new TableColumn("Col4", "int", allowNulls: true),
new TableColumn("Col5", "uniqueidentifier", allowNulls: true),
new TableColumn("Col6", "nvarchar(100)", allowNulls: true)
});
stagingTable.CreateTable();
CSVSource source = new CSVSource("src/ConnectionManager/DatatypeCSV.csv")
{
Configuration = new CsvHelper.Configuration.Configuration()
{
Quote = '"'
}
};
RowTransformation <string[], string[]> trans = new RowTransformation <string[], string[]>("Set empty values to null",
row => {
for (int i = 0; i < row.Length; i++)
{
if (row[i] == String.Empty)
{
row[i] = null;
}
}
return(row);
});
DBDestination <string[]> dest = new DBDestination <string[]>(stagingTable, 2);
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.AreEqual(3, RowCountTask.Count(stagingTable.Name));
}
public void CSV2ACCESS_ViaOdbc()
{
ControlFlow.CurrentDbConnection = new AccessOdbcConnectionManager(new OdbcConnectionString(AccessConnectionStringParameter))
{
AlwaysUseSameConnection = false
};
TableDefinition testTable = RecreateTestTable();
CSVSource source = new CSVSource("src/ConnectionManager/AccessData.csv");
DBDestination <string[]> dest = new DBDestination <string[]>(testTable, 2);
source.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.AreEqual(6, RowCountTask.Count(testTable.Name));
}
public void Multicast_Into2Tables()
{
var tableDestination1 = this.CreateTable("test.Table1");
var tableDestination2 = this.CreateTable("test.Table2");
var row1 = new RowTransformation <TestPoco, TestEntity1>(input => {
return(new TestEntity1
{
Col1 = input.Value1,
Col3 = input.Value3
});
});
var row2 = new RowTransformation <TestPoco, TestEntity2>(input => {
return(new TestEntity2
{
Col2 = input.Value2,
Col4 = input.Value4
});
});
var source = new CSVSource <TestPoco>("src/DataFlowExamples/Issue5.csv")
{
Configuration = new CsvHelper.Configuration.Configuration()
{
Delimiter = ";"
}
};
var multicast = new Multicast <TestPoco>();
var destination1 = new DBDestination <TestEntity1>("test.Table1");
var destination2 = new DBDestination <TestEntity2>("test.Table2");
source.LinkTo(multicast);
multicast.LinkTo(row1);
multicast.LinkTo(row2);
row1.LinkTo(destination1);
row2.LinkTo(destination2);
source.Execute();
destination1.Wait();
destination2.Wait();
Assert.AreEqual(2, RowCountTask.Count("test.Table1", "Col1 in ('one','five') and Col3 in ('three','seven')"));
Assert.AreEqual(2, RowCountTask.Count("test.Table2", "Col2 in ('two','six') and Col4 in ('four','eight')"));
}
public void CSV_RowTrans_DB_NonGeneric()
{
SqlTask.ExecuteNonQuery("Create source table", @"CREATE TABLE test.Staging
(Col1 int null, Col2 nvarchar(100) null)");
CSVSource source = new CSVSource("src/DataFlow/Simple_CSV2DB.csv");
RowTransformation trans = new RowTransformation(
csvdata => {
return(new string[] { csvdata[1], csvdata[0] });
});
DBDestination dest = new DBDestination("test.Staging");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.AreEqual(3, RowCountTask.Count("test.Staging"));
}
public void SimpleLookupWithDynamicObject(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture(connection, "SourceLookupDynamicObject");
source2Columns.InsertTestData();
FourColumnsTableFixture dest4Columns = new FourColumnsTableFixture(connection, "DestinationLookupDynamicObject", -1);
DBSource <ExpandoObject> source = new DBSource <ExpandoObject>(connection, "SourceLookupDynamicObject");
DBDestination <ExpandoObject> dest = new DBDestination <ExpandoObject>(connection, "DestinationLookupDynamicObject");
//Act
List <ExpandoObject> lookupList = new List <ExpandoObject>();
CSVSource <ExpandoObject> lookupSource = new CSVSource <ExpandoObject>("res/Lookup/LookupSource.csv");
var lookup = new ETLBox.DataFlow.Lookup <ExpandoObject, ExpandoObject>(
row =>
{
dynamic r = row as ExpandoObject;
r.Col3 = lookupList
.Where(lkupRow => { dynamic lk = lkupRow as dynamic; return(int.Parse(lk.Key) == r.Col1); })
.Select(lkupRow => { dynamic lk = lkupRow as dynamic;
return(lk.Column3 == string.Empty ? null : Int64.Parse(lk.Column3)); })
.FirstOrDefault();
r.Col4 = lookupList
.Where(lkupRow => { dynamic lk = lkupRow as dynamic; return(int.Parse(lk.Key) == r.Col1); })
.Select(lkupRow => { dynamic lk = lkupRow as dynamic; return(double.Parse(lk.Column4)); })
.FirstOrDefault();
return(row);
},
lookupSource,
lookupList
);
source.LinkTo(lookup);
lookup.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest4Columns.AssertTestData();
}
public void CSVIntoAccess()
{
//Arrange
TableDefinition testTable = RecreateAccessTestTable();
//Act
CSVSource source = new CSVSource("res/UseCases/AccessData.csv");
DBDestination <string[]> dest = new DBDestination <string[]>(testTable, batchSize: 2)
{
ConnectionManager = AccessOdbcConnection
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(AccessOdbcConnection, testTable.Name));
}
/// <summary>
/// Gets the right reader/writer TextSource depending of the extension.
/// </summary>
/// <param name="extension">The extension.</param>
/// <returns></returns>
public static TextSource GetSource(string extension)
{
TextSource source = null;
switch (extension.ToLower().Remove(0, 1))
{
case "csv":
source = new CSVSource();
break;
case "xlsx":
source = new XLSXSource();
break;
default:
break;
}
return(source);
}
public void UsingClassMapAndNoHeader()
{
//Arrange
FourColumnsTableFixture d4c = new FourColumnsTableFixture("CSVDestination4ColumnsClassMap");
DBDestination <MyExtendedRow> dest = new DBDestination <MyExtendedRow>(Connection, "CSVDestination4ColumnsClassMap");
//Act
CSVSource <MyExtendedRow> source = new CSVSource <MyExtendedRow>("res/CSVSource/FourColumnsInvalidHeader.csv");
source.Configuration.RegisterClassMap <ExtendedClassMap>();
source.Configuration.HasHeaderRecord = false;
source.Configuration.ShouldSkipRecord = ShouldSkipRecordDelegate;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
d4c.AssertTestData();
}
public void BigData_CSV_DB(int numberOfRows)
{
Stopwatch watch = new Stopwatch();
TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>()
{
new TableColumn("Col1", "nchar(1000)", allowNulls: false),
new TableColumn("Col2", "nchar(1000)", allowNulls: false),
new TableColumn("Col3", "nchar(1000)", allowNulls: false),
new TableColumn("Col4", "nchar(1000)", allowNulls: false),
});
stagingTable.CreateTable();
string fileName = "src/DataFlow/BigData_CSV2DB.csv";
BigDataHelper bigData = new BigDataHelper()
{
FileName = fileName,
NumberOfRows = numberOfRows,
TableDefinition = stagingTable
};
watch.Start();
LogTask.Info($"Create .csv file {fileName} with {numberOfRows} Rows");
bigData.CreateBigDataCSV();
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to create .csv file");
watch.Reset();
CSVSource source = new CSVSource(fileName);
DBDestination <string[]> dest = new DBDestination <string[]>(1000)
{
DestinationTableDefinition = stagingTable
};
source.LinkTo(dest);
watch.Start();
source.Execute();
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to read everything into memory (while constantly writing)");
LogTask.Info($"Already {RowCountTask.Count("test.Staging", RowCountOptions.QuickQueryMode)} inserted into table");
dest.Wait(); //TODO Wait should be part of source
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to write everything into database");
Assert.AreEqual(numberOfRows, SqlTask.ExecuteScalar <int>("Check staging table", $"select count(*) from test.Staging"));
}
public void ThrowExceptionWithoutHandling()
{
//Arrange
CSVSource source = new CSVSource("res/RowTransformation/TwoColumns.csv");
MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();
//Act
RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>(
csvdata => throw new InvalidOperationException("Test"));
source.LinkTo(trans);
trans.LinkTo(dest);
//Assert
Assert.Throws <AggregateException>(() =>
{
source.Execute();
dest.Wait();
});
}
private TimeSpan GetETLBoxTime <T>(int numberOfRows, CSVSource <T> source, MemoryDestination <T> dest)
{
source.LinkTo(dest);
var timeElapsedETLBox = BigDataHelper.LogExecutionTime($"Copying Csv into DB (non generic) with {numberOfRows} rows of data using ETLBox",
() =>
{
source.Execute();
dest.Wait();
}
);
if (typeof(T) == typeof(string[]))
{
output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Non generic).");
}
else
{
output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Generic).");
}
return(timeElapsedETLBox);
}
public void DuplicateCheckWithBlockTrans()
{
//Arrange
CSVSource <Poco> source = CreateDuplicateCSVSource("res/UseCases/DuplicateCheck.csv");
List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source
//Act
BlockTransformation <Poco> blockTrans = new BlockTransformation <Poco>(inputList =>
{
return(inputList.GroupBy(item => item.ID).Select(y => y.First()).ToList());
});
DBDestination <Poco> dest = CreateDestinationTable("dbo.DuplicateCheck");
source.LinkTo(blockTrans);
blockTrans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
AssertDataWithoutDuplicates();
}
public void CSV_DB()
{
TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>()
{
new TableColumn("Col1", "nvarchar(100)", allowNulls: false),
new TableColumn("Col2", "int", allowNulls: true)
});
stagingTable.CreateTable();
CSVSource source = new CSVSource("DataFlow/Simple_CSV2DB.csv");
DBDestination <string[]> dest = new DBDestination <string[]>()
{
DestinationTableDefinition = stagingTable
};
source.LinkTo(dest);
source.Execute();
dest.Wait(); //TODO Wait should be part of source
Assert.AreEqual(3, SqlTask.ExecuteScalar <int>("Check staging table", $"select count(*) from test.Staging where Col1 Like '%ValueRow%' and Col2 <> 1"));
}
public void CSVGeneric_DB()
{
TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>()
{
new TableColumn("Col1", "nvarchar(100)", allowNulls: false),
new TableColumn("Col2", "int", allowNulls: true)
});
stagingTable.CreateTable();
CSVSource <CSVData> source = new CSVSource <CSVData>("src/DataFlow/Simple_CSV2DB.csv");
DBDestination <CSVData> dest = new DBDestination <CSVData>()
{
DestinationTableDefinition = stagingTable
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.AreEqual(3, RowCountTask.Count("test.Staging", "Col1 Like '%ValueRow%' and Col2 <> 1"));
}
public void ThrowExceptionInFlow()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("RowTransExceptionTest");
CSVSource source = new CSVSource("res/RowTransformation/TwoColumns.csv");
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(SqlConnection, "RowTransExceptionTest");
CreateErrorTableTask.DropAndCreate(SqlConnection, "errors");
DBDestination <ETLBoxError> errorDest = new DBDestination <ETLBoxError>(SqlConnection, "errors");
//Act
RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>(
csvdata =>
{
int no = int.Parse(csvdata[0]);
if (no == 2)
{
throw new Exception("Test");
}
return(new MySimpleRow()
{
Col1 = no,
Col2 = csvdata[1]
});
});
source.LinkTo(trans);
trans.LinkTo(dest);
trans.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
Assert.Equal(2, RowCountTask.Count(SqlConnection, "RowTransExceptionTest"));
Assert.Equal(1, RowCountTask.Count(SqlConnection, "errors"));
}
public void RearrangeSwappedData()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation");
CSVSource source = new CSVSource("res/RowTransformation/TwoColumnsSwapped.csv");
//Act
RowTransformation trans = new RowTransformation(
csvdata =>
{
return(new string[] { csvdata[1], csvdata[0] });
});
DBDestination dest = new DBDestination(Connection, "DestinationRowTransformation");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void DuplicateCheckInRowTrans()
{
//Arrange
CSVSource <Poco> source = CreateDuplicateCSVSource("res/UseCases/DuplicateCheck.csv");
List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source
//Act
RowTransformation <Poco, Poco> rowTrans = new RowTransformation <Poco, Poco>(input =>
{
if (IDs.Contains(input.ID))
{
input.IsDuplicate = true;
}
else
{
IDs.Add(input.ID);
}
return(input);
});
Multicast <Poco> multicast = new Multicast <Poco>();
DBDestination <Poco> dest = CreateDestinationTable("dbo.DuplicateCheck");
VoidDestination <Poco> trash = new VoidDestination <Poco>();
source.LinkTo(rowTrans);
rowTrans.LinkTo(multicast);
multicast.LinkTo(dest, input => input.IsDuplicate == false);
multicast.LinkTo(trash, input => input.IsDuplicate == true);
source.Execute();
dest.Wait();
trash.Wait();
//Assert
AssertDataWithoutDuplicates();
}
private void button1_Click(object sender, EventArgs e)
{
DataLoader loader;
DataSource source;
GroupSource group_source;
try
{
loader = new NetworkDownloader(textBox1.Text);
loader.Load();
source = new CSVSource(loader.GetData());
source.Load();
group_source = new GroupSource(textBox2.Text);
group_source.Load();
}catch (Exception ex)
{
MessageBox.Show("Unable to download data: " + ex.Message);
return;
}
listView1.Items.Clear();
this.Groups = group_source.Groups;
CovidDataRow total = new CovidDataRow("Total");
foreach (var group in group_source.Groups)
{
var last_date = dateTimePicker1.Value;
var data = group.GetDataForDate(last_date);
this.DisplayData(group.Name, data);
total += data;
}
this.DisplayData("Total:", total);
}
public void CSVAttributeWithSkipRowCountTest()
{
var stream = new System.IO.MemoryStream();
var text = "\r\n\r\nCol1,Col2\r\nA,B\r\nD,E";
var bytes = System.Text.Encoding.UTF8.GetBytes(text);
stream.Write(bytes, 0, bytes.Length);
stream.Position = 0;
var target = new CSVSource<TestClass3>(stream, System.Text.Encoding.UTF8);
var firstLine = target.ReadNext();
Assert.AreEqual("A", firstLine.Col1);
Assert.AreEqual("B", firstLine.Col2);
}
public void TestSampleDataflow()
{
using (TestDb)
{
TableColumn keyCol = new TableColumn("Key", SqlDbType.Int, false, true, true);
TableColumn col1 = new TableColumn("Col1", SqlDbType.NVarChar, 100, false);
TableColumn col2 = new TableColumn("Col2", SqlDbType.NVarChar, 50, true);
TableColumn col3 = new TableColumn("Col3", SqlDbType.Int, false);
string destSchema = "test";
string destTable1 = "Staging1";
string destObject1 = $"[{destSchema}].[{destTable1}]";
new DropAndCreateTableTask(TestDb.getNewSqlConnection()).Execute(destSchema, destTable1, new List <TableColumn>()
{
keyCol, col1, col2, col3
});
SqlDestination <string[]> destination1 = new SqlDestination <string[]>();
destination1.ObjectName = destObject1;
destination1.FieldCount = 4;
destination1.ObjectMappingMethod = WriterAdapter_SampleDataflow.Fill;
destination1.SqlConnection = TestDb.getNewSqlConnection();
string destTable2 = "Staging2";
string destObject2 = $"[{destSchema}].[{destTable2}]";
new DropAndCreateTableTask(TestDb.getNewSqlConnection()).Execute(destSchema, destTable2, new List <TableColumn>()
{
keyCol, col1, col2, col3
});
SqlDestination <string[]> destination2 = new SqlDestination <string[]>();
destination2.ObjectName = destObject2;
destination2.FieldCount = 4;
destination2.ObjectMappingMethod = WriterAdapter_SampleDataflow.Fill;
destination2.SqlConnection = TestDb.getNewSqlConnection();
CSVSource <string[]> CSVSource =
new CSVSource <string[]>("DataFlow/InputData.csv");
Graph g = new Graph();
g.GetVertex(0, CSVSource);
g.GetVertex(1, new RowTransformation <string[]>(RowTransformation1));
g.GetVertex(11, new RowTransformation <string[]>(RowTransformation2));
g.GetVertex(10, new BroadCast <string[]>(CloneTransformation1));
g.GetVertex(12, new RowTransformationMany <string[]>(RowTransformationMany));
g.GetVertex(20, new RowTransformation <string[]>(RowTransformation3));
g.GetVertex(100, destination1);
g.GetVertex(110, destination2);
Edge e1 = g.AddEdge(0, 1); // connect 0 to 1
Edge e2 = g.AddEdge(1, 10); // connect 1 to 10
Edge e3 = g.AddEdge(10, 20);
Edge e4 = g.AddEdge(20, 100);
Edge e5 = g.AddEdge(10, 11);
Edge e6 = g.AddEdge(11, 12);
Edge e7 = g.AddEdge(12, 110);
DataFlowTask <string[]> .Execute("Test dataflow task", 1000, 1, g);
e2.cost = counter_RowTransformation1;
e6.cost = counter_RowTransformation2;
e4.cost = counter_RowTransformation3;
e7.cost = counter_RowTransformationMany;
//TestHelper.VisualizeGraph(g);
Assert.AreEqual(4, new ExecuteSQLTask(TestDb.getNewSqlConnection()).ExecuteScalar(string.Format("select count(*) from {0}", destObject1)));
Assert.AreEqual(8, new ExecuteSQLTask(TestDb.getNewSqlConnection()).ExecuteScalar(string.Format("select count(*) from {0}", destObject2)));
}
}
