public void WithBatchChanges(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DBDestinationBatchChanges");
DBDestination dest = new DBDestination(connection, "DBDestinationBatchChanges", batchSize: 2)
{
BeforeBatchWrite = rowArray =>
{
rowArray[0][1] = "NewValue";
return(rowArray);
}
};
//Act
CSVSource source = new CSVSource("res/CSVSource/TwoColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(connection, "DBDestinationBatchChanges"));
Assert.Equal(2, RowCountTask.Count(connection, "DBDestinationBatchChanges", $"{d2c.QB}Col2{d2c.QE}='NewValue'"));
Assert.Equal(1, RowCountTask.Count(connection, "DBDestinationBatchChanges", $"{d2c.QB}Col1{d2c.QE} = 2 AND {d2c.QB}Col2{d2c.QE}='Test2'"));
}
public void CSV_RowTrans_DB()
{
TableDefinition destinationTableDefinition = new TableDefinition("test.Staging", new List <TableColumn>()
{
new TableColumn("Col1", "nvarchar(100)", allowNulls: false),
new TableColumn("Col2", "int", allowNulls: true)
});
destinationTableDefinition.CreateTable();
CSVSource source = new CSVSource("src/DataFlow/Simple_CSV2DB.csv");
RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>(
csvdata => {
return(new MySimpleRow()
{
Col1 = csvdata[0],
Col2 = int.Parse(csvdata[1])
});
});
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>()
{
DestinationTableDefinition = destinationTableDefinition
};
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.AreEqual(3, RowCountTask.Count("test.Staging"));
}
public void CSV_DB_WithKey(int keyPosition)
{
List <TableColumn> columns = new List <TableColumn>()
{
new TableColumn("Col1", "nvarchar(100)", allowNulls: false),
new TableColumn("Col2", "int", allowNulls: true)
};
columns.Insert(keyPosition, new TableColumn("Key", "int", allowNulls: false, isPrimaryKey: true)
{
IsIdentity = true
});
TableDefinition stagingTable = new TableDefinition($"test.Staging{keyPosition}", columns);
stagingTable.CreateTable();
CSVSource source = new CSVSource("DataFlow/Simple_CSV2DB.csv");
DBDestination <string[]> dest = new DBDestination <string[]>()
{
DestinationTableDefinition = stagingTable
};
source.LinkTo(dest);
source.Execute();
dest.Wait(); //TODO Wait should be part of source
Assert.AreEqual(3, SqlTask.ExecuteScalar <int>("Check staging table", $"select count(*) from test.Staging{keyPosition} where Col1 Like '%ValueRow%' and Col2 <> 1"));
}
public void CSV_DB_WithBatchChanges()
{
TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>()
{
new TableColumn("Col1", "nvarchar(100)", allowNulls: false),
new TableColumn("Col2", "int", allowNulls: true)
});
stagingTable.CreateTable();
CSVSource source = new CSVSource("DataFlow/Simple_CSV2DB.csv");
DBDestination <string[]> dest = new DBDestination <string[]>(batchSize: 2)
{
DestinationTableDefinition = stagingTable,
BeforeBatchWrite =
rowArray => {
rowArray[0][0] = "NewValue";
return(rowArray);
}
};
source.LinkTo(dest);
source.Execute();
dest.Wait(); //TODO Wait should be part of source
Assert.AreEqual(1, SqlTask.ExecuteScalar <int>("Check staging table", $"select count(*) from test.Staging where Col1 Like '%ValueRow%' and Col2 <> 1"));
Assert.AreEqual(2, SqlTask.ExecuteScalar <int>("Check staging table", $"select count(*) from test.Staging where Col1 = 'NewValue'"));
}
public void Start()
{
CSVSource sourceOrderData = new CSVSource("src/DataFlow/DemoData.csv");
sourceOrderData.Configuration.Delimiter = ";";
RowTransformation <string[], Order> transIntoObject = new RowTransformation <string[], Order>(CSVIntoObject);
DBSource <Customer> sourceCustomerData = new DBSource <Customer>(CustomerTableDef);
LookupCustomerKey lookupCustKeyClass = new LookupCustomerKey();
Lookup <Order, Order, Customer> lookupCustomerKey = new Lookup <Order, Order, Customer>(
lookupCustKeyClass.FindKey, sourceCustomerData, lookupCustKeyClass.LookupData);
Multicast <Order> multiCast = new Multicast <Order>();
DBDestination <Order> destOrderTable = new DBDestination <Order>(OrderDataTableDef);
BlockTransformation <Order> blockOrders = new BlockTransformation <Order>(BlockTransformOrders);
DBDestination <Rating> destRating = new DBDestination <Rating>(CustomerRatingTableDef);
RowTransformation <Order, Rating> transOrderIntoCust = new RowTransformation <Order, Rating>(OrderIntoRating);
CustomDestination <Order> destSink = new CustomDestination <Order>(row => {; });
sourceOrderData.LinkTo(transIntoObject);
transIntoObject.LinkTo(lookupCustomerKey);
lookupCustomerKey.LinkTo(multiCast);
multiCast.LinkTo(destOrderTable);
multiCast.LinkTo(blockOrders);
blockOrders.LinkTo(transOrderIntoCust, ord => ord.Rating != null);
blockOrders.LinkTo(destSink, ord => ord.Rating == null);
transOrderIntoCust.LinkTo(destRating);
sourceOrderData.ExecuteAsync();
destOrderTable.Wait();
destRating.Wait();
}
public void WithObjectErrorLinking()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVSourceErrorLinking");
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(SqlConnection, "CSVSourceErrorLinking");
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumnsErrorLinking.csv");
source.LinkTo(dest);
source.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
dest2Columns.AssertTestData();
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public void ConvertIntoObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformationDynamic");
CSVSource <ExpandoObject> source = new CSVSource <ExpandoObject>("res/RowTransformation/TwoColumns.csv");
//Act
RowTransformation <ExpandoObject> trans = new RowTransformation <ExpandoObject>(
csvdata =>
{
dynamic c = csvdata as ExpandoObject;
c.Col1 = c.Header1;
c.Col2 = c.Header2;
return(c);
});
DBDestination <ExpandoObject> dest = new DBDestination <ExpandoObject>(Connection, "DestinationRowTransformationDynamic");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CSV_DB_WithBatchChanges()
{
TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>()
{
new TableColumn("Col1", "nvarchar(100)", allowNulls: false),
new TableColumn("Col2", "int", allowNulls: true)
});
stagingTable.CreateTable();
CSVSource source = new CSVSource("src/DataFlow/Simple_CSV2DB.csv");
DBDestination <string[]> dest = new DBDestination <string[]>(batchSize: 2)
{
DestinationTableDefinition = stagingTable,
BeforeBatchWrite =
rowArray => {
rowArray[0][0] = "NewValue";
return(rowArray);
}
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.AreEqual(1, RowCountTask.Count("test.Staging", "Col1 Like '%ValueRow%' and Col2 <> 1"));
Assert.AreEqual(2, RowCountTask.Count("test.Staging", "Col1 = 'NewValue'"));
}
//MemberData(nameof(Connections), 200000, 1000)]
public void UsingNonGeneric(IConnectionManager connection, int numberOfRows, int batchSize)
{
//Arrange
TableDefinition destinationTable = CreateDestinationTable(connection, "CSVDestination");
CreateCSVFile("res/Csv/TestData.csv", numberOfRows, destinationTable);
var source = new CSVSource("res/Csv/TestData.csv");
var dest = new DBDestination(batchSize)
{
DestinationTableDefinition = destinationTable,
ConnectionManager = connection
};
source.LinkTo(dest);
//Act
BigDataHelper.LogExecutionTime($"Copying Csv into DB (non generic) wswith {numberOfRows} rows of data",
() =>
{
source.Execute();
dest.Wait();
}
);
//Assert
Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CSVDestination"));
}
public void InitAction()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation");
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/RowTransformation/TwoColumnsIdMinus1.csv");
//Act
int IdOffset = 0;
RowTransformation <MySimpleRow, MySimpleRow> trans = new RowTransformation <MySimpleRow, MySimpleRow>(
"RowTransformation testing init Action",
row =>
{
row.Col1 += IdOffset;
return(row);
},
() => IdOffset += 1
);
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "DestinationRowTransformation");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CSV_DB_WithKey(int keyPosition)
{
List <TableColumn> columns = new List <TableColumn>()
{
new TableColumn("Col1", "nvarchar(100)", allowNulls: false),
new TableColumn("Col2", "int", allowNulls: true)
};
columns.Insert(keyPosition, new TableColumn("Key", "int", allowNulls: false, isPrimaryKey: true)
{
IsIdentity = true
});
TableDefinition stagingTable = new TableDefinition($"test.Staging{keyPosition}", columns);
stagingTable.CreateTable();
CSVSource source = new CSVSource("src/DataFlow/Simple_CSV2DB.csv");
DBDestination <string[]> dest = new DBDestination <string[]>()
{
DestinationTableDefinition = stagingTable
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.AreEqual(3, RowCountTask.Count($"test.Staging{keyPosition}", "Col1 Like '%ValueRow%' and Col2 <> 1"));
}
public void TestDuplicateCheckWithBlockTrans()
{
CSVSource <Poco> source = new CSVSource <Poco>("src/DataFlowExamples/Duplicate.csv");
source.Configuration.Delimiter = ";";
source.Configuration.TrimOptions = CsvHelper.Configuration.TrimOptions.Trim;
source.Configuration.MissingFieldFound = null;
List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source
BlockTransformation <Poco> blockTrans = new BlockTransformation <Poco>(inputList =>
{
return(inputList.GroupBy(item => item.ID).Select(y => y.First()).ToList());
});
var dest = new DBDestination <Poco>("dbo.Staging");
TableDefinition stagingTable = new TableDefinition("dbo.Staging", new List <TableColumn>()
{
new TableColumn("Key", "INT", allowNulls: false, isPrimaryKey: true, isIdentity: true),
new TableColumn("ID", "INT", allowNulls: false),
new TableColumn("Value", "NVARCHAR(100)", allowNulls: false),
new TableColumn("Name", "NVARCHAR(100)", allowNulls: false)
});
stagingTable.CreateTable();
source.LinkTo(blockTrans);
blockTrans.LinkTo(dest);
source.Execute();
dest.Wait();
}
public void MoreColumnsInDestination()
{
//Arrange
CreateTableTask.Create(SqlConnection, "CSVSourceDynamicColsInDest",
new List <TableColumn>()
{
new TableColumn("Col2", "VARCHAR(100)", allowNulls: true),
new TableColumn("Id", "INT", allowNulls: false, isPrimaryKey: true, isIdentity: true),
new TableColumn("Col1", "INT", allowNulls: true),
new TableColumn("ColX", "INT", allowNulls: true),
});
DBDestination <ExpandoObject> dest = new DBDestination <ExpandoObject>(SqlConnection, "CSVSourceDynamicColsInDest");
//Act
CSVSource <ExpandoObject> source = new CSVSource <ExpandoObject>("res/CSVSource/TwoColumnsForDynamic.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(SqlConnection, "CSVSourceDynamicColsInDest"));
Assert.Equal(1, RowCountTask.Count(SqlConnection, "CSVSourceDynamicColsInDest", $"Col1 = 1 AND Col2='Test1' AND Id > 0 AND ColX IS NULL"));
Assert.Equal(1, RowCountTask.Count(SqlConnection, "CSVSourceDynamicColsInDest", $"Col1 = 2 AND Col2='Test2' AND Id > 0 AND ColX IS NULL"));
Assert.Equal(1, RowCountTask.Count(SqlConnection, "CSVSourceDynamicColsInDest", $"Col1 = 3 AND Col2='Test3' AND Id > 0 AND ColX IS NULL"));
}
public void ConvertIntoObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation");
CSVSource source = new CSVSource("res/RowTransformation/TwoColumns.csv");
//Act
RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>(
csvdata =>
{
return(new MySimpleRow()
{
Col1 = int.Parse(csvdata[0]),
Col2 = csvdata[1]
});
});
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "DestinationRowTransformation");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void BigData_CSV_ACCESS_ViaOdbc()
{
int numberOfRows = 2000;
ControlFlow.CurrentDbConnection = new AccessOdbcConnectionManager(new OdbcConnectionString(AccessConnectionStringParameter))
{
AlwaysUseSameConnection = false
};
Stopwatch watch = new Stopwatch();
TableDefinition stagingTable = new TableDefinition("staging", new List <TableColumn>()
{
new TableColumn("Col1", "CHAR", allowNulls: true),
new TableColumn("Col2", "CHAR", allowNulls: true),
new TableColumn("Col3", "CHAR", allowNulls: false),
new TableColumn("Col4", "CHAR", allowNulls: false),
});
try {
SqlTask.ExecuteNonQuery("Try to drop table", $@"DROP TABLE {stagingTable.Name};");
} catch { }
new CreateTableTask(stagingTable)
{
ThrowErrorIfTableExists = true
}.Execute();
string fileName = "src/ConnectionManager/AccessBigData_CSV2DB.csv";
BigDataHelper bigData = new BigDataHelper()
{
FileName = fileName,
NumberOfRows = numberOfRows,
TableDefinition = stagingTable
};
watch.Start();
LogTask.Info($"Create .csv file {fileName} with {numberOfRows} Rows");
bigData.CreateBigDataCSV();
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to create .csv file");
watch.Reset();
CSVSource source = new CSVSource(fileName);
DBDestination <string[]> dest = new DBDestination <string[]>(30)
{
DestinationTableDefinition = stagingTable
};
source.LinkTo(dest);
watch.Start();
source.Execute();
dest.Wait();
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to write everything into database");
Assert.AreEqual(numberOfRows, RowCountTask.Count(stagingTable.Name));
}
public void SplitCSVSourceIn2Tables()
{
//Arrange
TwoColumnsTableFixture dest1Table = new TwoColumnsTableFixture("SplitDataDestination1");
FourColumnsTableFixture dest2Table = new FourColumnsTableFixture("SplitDataDestination2");
var source = new CSVSource <CSVPoco>("res/Multicast/CSVSourceToSplit.csv")
{
Configuration = new CsvHelper.Configuration.Configuration()
{
Delimiter = ";"
}
};
var multicast = new Multicast <CSVPoco>();
var row1 = new RowTransformation <CSVPoco, Entity1>(input => {
return(new Entity1
{
Col1 = input.CSVCol1,
Col2 = input.CSVCol2
});
});
var row2 = new RowTransformation <CSVPoco, Entity2>(input => {
return(new Entity2
{
Col2 = input.CSVCol2,
Col3 = input.CSVCol3,
Col4 = input.CSVCol4
});
});
var destination1 = new DBDestination <Entity1>(Connection, "SplitDataDestination1");
var destination2 = new DBDestination <Entity2>(Connection, "SplitDataDestination2");
//Act
source.LinkTo(multicast);
multicast.LinkTo(row1);
multicast.LinkTo(row2);
row1.LinkTo(destination1);
row2.LinkTo(destination2);
source.Execute();
destination1.Wait();
destination2.Wait();
//Assert
dest1Table.AssertTestData();
dest2Table.AssertTestData();
}
public void TestDuplicateCheckInRowTrans()
{
CreateLogTablesTask.CreateLog();
DataFlow.LoggingThresholdRows = 2;
CSVSource <Poco> source = new CSVSource <Poco>("src/DataFlowExamples/Duplicate.csv");
source.Configuration.Delimiter = ";";
source.Configuration.TrimOptions = CsvHelper.Configuration.TrimOptions.Trim;
source.Configuration.MissingFieldFound = null;
List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source
RowTransformation <Poco, Poco> rowTrans = new RowTransformation <Poco, Poco>(input =>
{
if (IDs.Contains(input.ID))
{
input.IsDuplicate = true;
}
else
{
IDs.Add(input.ID);
}
return(input);
});
var multicast = new Multicast <Poco>();
var dest = new DBDestination <Poco>("dbo.Staging");
TableDefinition stagingTable = new TableDefinition("dbo.Staging", new List <TableColumn>()
{
new TableColumn("Key", "INT", allowNulls: false, isPrimaryKey: true, isIdentity: true),
new TableColumn("ID", "INT", allowNulls: false),
new TableColumn("Value", "NVARCHAR(100)", allowNulls: false),
new TableColumn("Name", "NVARCHAR(100)", allowNulls: false)
});
stagingTable.CreateTable();
var trash = new VoidDestination <Poco>();
source.LinkTo(rowTrans);
rowTrans.LinkTo(multicast);
multicast.LinkTo(dest, input => input.IsDuplicate == false);
multicast.LinkTo(trash, input => input.IsDuplicate == true);
source.Execute();
dest.Wait();
trash.Wait();
}
private static void StartDefaultCSVLoad(Stopwatch watch, TableDefinition stagingTable, string fileName)
{
CSVSource source = new CSVSource(fileName);
DBDestination <string[]> dest = new DBDestination <string[]>(1000)
{
DestinationTableDefinition = stagingTable
};
source.LinkTo(dest);
watch.Start();
source.Execute();
LogTask.Info($"Needed {watch.Elapsed.TotalMinutes} to read everything into memory (while constantly writing)");
LogTask.Info($"Already {RowCountTask.Count("test.Staging", RowCountOptions.QuickQueryMode)} inserted into table");
dest.Wait();
}
public void IdentityAtTheEnd()
{
//Arrange
FourColumnsTableFixture dest4Columns = new FourColumnsTableFixture("CSVDestination4Columns", identityColumnIndex: 3);
DBDestination dest = new DBDestination(Connection, "CSVDestination4Columns");
//Act
CSVSource source = new CSVSource("res/CSVSource/ThreeColumnsNoId.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest4Columns.AssertTestData();
}
private static void Load_CSV_DB(List <TableColumn> columnsInStageTable)
{
TableDefinition stagingTable = new TableDefinition("test.Staging", columnsInStageTable);
stagingTable.CreateTable();
CSVSource source = new CSVSource("src/DataFlow/Simple_CSV2DB.csv");
DBDestination <string[]> dest = new DBDestination <string[]>()
{
DestinationTableDefinition = stagingTable
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.AreEqual(3, RowCountTask.Count("test.Staging", "Col1 Like '%ValueRow%'"));
}
public void SimpleFlowWithObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVDestination2Columns");
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "CSVDestination2Columns");
//Act
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void MoreColumnsInSource()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVSourceDynamicColsInSource");
DBDestination <ExpandoObject> dest = new DBDestination <ExpandoObject>(SqlConnection, "CSVSourceDynamicColsInSource");
//Act
CSVSource <ExpandoObject> source = new CSVSource <ExpandoObject>("res/CSVSource/FourColumnsForDynamic.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void MoreColumnsInDatabase()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVDestination2Columns");
DBDestination dest = new DBDestination(Connection, "CSVDestination2Columns");
//Act
CSVSource source = new CSVSource("res/CSVSource/OneColumn.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(Connection, "CSVDestination2Columns", "Col1 IN (1,2,3)"));
}
public void MoreColumnsInCSV()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVDestination2Columns");
DBDestination dest = new DBDestination(Connection, "CSVDestination2Columns");
//Act
CSVSource source = new CSVSource("res/CSVSource/ThreeColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void WithoutErrorLinking()
{
//Arrange
MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();
//Act
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumnsErrorLinking.csv");
//Assert
Assert.Throws <CsvHelper.TypeConversion.TypeConverterException>(() =>
{
source.LinkTo(dest);
source.Execute();
dest.Wait();
});
}
public void SimpleFlowUsingClassMap()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVDestination2Columns");
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "CSVDestination2Columns");
//Act
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumns.csv");
source.Configuration.RegisterClassMap <ModelClassMap>();
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CSVGenericWithSkipRows_DB()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVSourceSkipRows");
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "CSVSourceSkipRows");
//Act
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumnsSkipRows.csv");
source.SkipRows = 2;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CSVSourceNoHeader()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CSVSourceNoHeader");
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "CSVSourceNoHeader");
//Act
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/CSVSource/TwoColumnsNoHeader.csv");
source.Configuration.HasHeaderRecord = false;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CSV_DB_WithOdbcConnection()
{
ControlFlow.CurrentDbConnection = new OdbcConnectionManager(new OdbcConnectionString(OdbcConnectionStringParameter));
CreateSchemaTask.Create("test");
TableDefinition stagingTable = new TableDefinition("test.Staging", new List <TableColumn>()
{
new TableColumn("ID", "int", allowNulls: false, isPrimaryKey: true, isIdentity: true),
new TableColumn("Col1", "bit", allowNulls: true),
new TableColumn("Col2", "decimal(10,5)", allowNulls: true),
new TableColumn("Col3", "tinyint", allowNulls: true),
new TableColumn("Col4", "int", allowNulls: true),
new TableColumn("Col5", "uniqueidentifier", allowNulls: true),
new TableColumn("Col6", "nvarchar(100)", allowNulls: true)
});
stagingTable.CreateTable();
CSVSource source = new CSVSource("src/ConnectionManager/DatatypeCSV.csv")
{
Configuration = new CsvHelper.Configuration.Configuration()
{
Quote = '"'
}
};
RowTransformation <string[], string[]> trans = new RowTransformation <string[], string[]>("Set empty values to null",
row => {
for (int i = 0; i < row.Length; i++)
{
if (row[i] == String.Empty)
{
row[i] = null;
}
}
return(row);
});
DBDestination <string[]> dest = new DBDestination <string[]>(stagingTable, 2);
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.AreEqual(3, RowCountTask.Count(stagingTable.Name));
}
public void CSV2ACCESS_ViaOdbc()
{
ControlFlow.CurrentDbConnection = new AccessOdbcConnectionManager(new OdbcConnectionString(AccessConnectionStringParameter))
{
AlwaysUseSameConnection = false
};
TableDefinition testTable = RecreateTestTable();
CSVSource source = new CSVSource("src/ConnectionManager/AccessData.csv");
DBDestination <string[]> dest = new DBDestination <string[]>(testTable, 2);
source.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.AreEqual(6, RowCountTask.Count(testTable.Name));
}
