public void MoreColumnsInDestination()
{
//Arrange
CreateTableTask.Create(SqlConnection, "CsvSourceDynamicColsInDest",
new List <TableColumn>()
{
new TableColumn("Col2", "VARCHAR(100)", allowNulls: true),
new TableColumn("Id", "INT", allowNulls: false, isPrimaryKey: true, isIdentity: true),
new TableColumn("Col1", "INT", allowNulls: true),
new TableColumn("ColX", "INT", allowNulls: true),
});
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(SqlConnection, "CsvSourceDynamicColsInDest");
//Act
CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/CsvSource/TwoColumnsForDynamic.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest"));
Assert.Equal(1, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest", $"Col1 = 1 AND Col2='Test1' AND Id > 0 AND ColX IS NULL"));
Assert.Equal(1, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest", $"Col1 = 2 AND Col2='Test2' AND Id > 0 AND ColX IS NULL"));
Assert.Equal(1, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest", $"Col1 = 3 AND Col2='Test3' AND Id > 0 AND ColX IS NULL"));
}
public void AfterBatchWrite(IConnectionManager connection)
{
//Arrange
bool wasExecuted = false;
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DbDestinationBatchChanges");
DbDestination <string[]> dest = new DbDestination <string[]>(connection, "DbDestinationBatchChanges", batchSize: 1)
{
AfterBatchWrite = rowArray =>
{
Assert.True(rowArray.Length == 1);
wasExecuted = true;
}
};
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/TwoColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(connection, "DbDestinationBatchChanges"));
Assert.True(wasExecuted);
}
public void ConvertIntoObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformationDynamic");
CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/RowTransformation/TwoColumns.csv");
//Act
RowTransformation <ExpandoObject> trans = new RowTransformation <ExpandoObject>(
csvdata =>
{
dynamic c = csvdata as ExpandoObject;
c.Col1 = c.Header1;
c.Col2 = c.Header2;
return(c);
});
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(Connection, "DestinationRowTransformationDynamic");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CheckMemoryUsage(IConnectionManager connection, int numberOfRows, int batchSize, double deviation)
{
//Arrange
BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
ReCreateDestinationTable(connection, "CsvDestinationWithTransformation");
var sourceExpando = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var trans = new RowTransformation <ExpandoObject, CSVData>(
row =>
{
dynamic r = row as ExpandoObject;
return(new CSVData()
{
Col1 = r.Col1,
Col2 = r.Col2,
Col3 = r.Col3,
Col4 = r.Col4
});
});
var destGeneric = new DbDestination <CSVData>(connection, "CsvDestinationWithTransformation", batchSize);
sourceExpando.LinkTo(trans);
trans.LinkTo(destGeneric);
//Act
long memAfter = 0;
long memBefore = 0;
bool startCheck = true;
int count = 1;
destGeneric.AfterBatchWrite = data =>
{
if (count++ % 50 == 0)
{
using Process proc = Process.GetCurrentProcess();
memAfter = proc.WorkingSet64;
if (startCheck)
{
memBefore = memAfter;
startCheck = false;
}
Assert.True(memAfter < (memBefore + (memBefore * deviation)));
}
};
var timeElapsedETLBox = BigDataHelper.LogExecutionTime($"Copying Csv into DB (non generic) with {numberOfRows} rows of data using ETLBox",
() =>
{
sourceExpando.Execute();
destGeneric.Wait();
}
);
output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Expando to object transformation).");
//Assert
Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationWithTransformation"));
//10.000.000 rows, batch size 10.000: ~8 min
//10.000.000 rows, batch size 1.000: ~10 min 10 sec
}
public void InitAction()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation");
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/RowTransformation/TwoColumnsIdMinus1.csv");
//Act
int IdOffset = 0;
RowTransformation <MySimpleRow, MySimpleRow> trans = new RowTransformation <MySimpleRow, MySimpleRow>()
{
TransformationFunc = row =>
{
row.Col1 += IdOffset;
return(row);
},
InitAction = () => IdOffset += 1
};
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(Connection, "DestinationRowTransformation");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void ConvertIntoObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation");
CsvSource <string[]> source = new CsvSource <string[]>("res/RowTransformation/TwoColumns.csv");
//Act
RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>(
csvdata =>
{
return(new MySimpleRow()
{
Col1 = int.Parse(csvdata[0]),
Col2 = csvdata[1]
});
});
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(Connection, "DestinationRowTransformation");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void WithBatchChanges(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DbDestinationBatchChanges");
DbDestination <string[]> dest = new DbDestination <string[]>(connection, "DbDestinationBatchChanges", batchSize: 2)
{
BeforeBatchWrite = rowArray =>
{
rowArray[0][1] = "NewValue";
return(rowArray);
}
};
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/TwoColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(connection, "DbDestinationBatchChanges"));
Assert.Equal(2, RowCountTask.Count(connection, "DbDestinationBatchChanges", $"{d2c.QB}Col2{d2c.QE}='NewValue'"));
Assert.Equal(1, RowCountTask.Count(connection, "DbDestinationBatchChanges", $"{d2c.QB}Col1{d2c.QE} = 2 AND {d2c.QB}Col2{d2c.QE}='Test2'"));
}
static void Main(string[] args)
{
Preparation();
SqlConnectionManager connMan = new SqlConnectionManager("Data Source=.;Initial Catalog=demo;Integrated Security=false;User=sa;password=reallyStrongPwd123");
CsvSource <string[]> source = new CsvSource <string[]>("demodata.csv");
RowTransformation <string[], Order> rowTrans = new RowTransformation <string[], Order>(
row => new Order()
{
Id = int.Parse(row[0]),
Item = row[1],
Quantity = int.Parse(row[2]) + int.Parse(row[3]),
Price = double.Parse(row[4]) * 100
});
DbDestination <Order> dest = new DbDestination <Order>(connMan, "OrderTable");
source.LinkTo(rowTrans);
rowTrans.LinkTo(dest);
source.Execute();
dest.Wait();
Console.WriteLine("Press any key to continue...");
Console.ReadLine();
}
static void Main(string[] args)
{
//Set up database
Preparation();
//Define the source
CsvSource <string[]> source = new CsvSource <string[]>("demodata.csv");
//Define the transformation
RowTransformation <string[], Order> rowTrans = new RowTransformation <string[], Order>(
row => new Order()
{
Item = row[1],
Quantity = int.Parse(row[2]) + int.Parse(row[3]),
Price = int.Parse(row[4]) / 100
});
//DbDestination needs a connection manager pointing to the right database
SqlConnectionManager connMan = new SqlConnectionManager("Data Source=localhost;User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=demo;");
//Define the destination
DbDestination <Order> dest = new DbDestination <Order>(connMan, "OrderTable");
//Link & run flow
source.LinkTo(rowTrans);
rowTrans.LinkTo(dest);
source.Execute();
dest.Wait();
Console.WriteLine("Press any key to continue...");
Console.ReadLine();
}
public void WithObjectErrorLinking()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceErrorLinking");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("CsvSourceErrorLinking", SqlConnection);
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsErrorLinking.csv");
source.LinkTo(dest);
source.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
dest2Columns.AssertTestData();
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public static void Run()
{
Console.WriteLine("Running data flow");
//Read data from csv file
CsvSource sourceOrderData = new CsvSource("DemoData.csv");
sourceOrderData.Configuration.Delimiter = ";";
//Transform into Order object
RowTransformation <ExpandoObject, Order> transIntoObject = new RowTransformation <ExpandoObject, Order>(
csvLine =>
{
dynamic order = csvLine as dynamic;
return(new Order()
{
//Header in Csv: OrderNumber;OrderItem;OrderAmount;CustomerName
Number = order.OrderNumber,
Item = order.OrderItem,
Amount = decimal.Parse(order.OrderAmount.ToString().Replace("€", ""), CultureInfo.GetCultureInfo("en-US")),
CustomerName = order.CustomerName
});
});
sourceOrderData.LinkTo(transIntoObject);
//Find corresponding customer id if customer exists in Customer table
DbSource <Customer> sourceCustomerData = new DbSource <Customer>("customer");
LookupTransformation <Order, Customer> lookupCustomerKey = new LookupTransformation <Order, Customer>(sourceCustomerData);
transIntoObject.LinkTo(lookupCustomerKey);
//Split data
Multicast <Order> multiCast = new Multicast <Order>();
lookupCustomerKey.LinkTo(multiCast);
//Store Order in Orders table
DbDestination <Order> destOrderTable = new DbDestination <Order>("orders");
multiCast.LinkTo(destOrderTable);
//Create rating for existing customers based total of order amount
Aggregation <Order, Rating> aggregation = new Aggregation <Order, Rating>();
multiCast.LinkTo(aggregation);
//Store the rating in the customer rating table
DbDestination <Rating> destRating = new DbDestination <Rating>("customer_rating");
aggregation.LinkTo(destRating);
//Execute the data flow synchronously
sourceOrderData.Execute();
destOrderTable.Wait();
destRating.Wait();
}
public void SplitCsvSourceIn2Tables()
{
//Arrange
TwoColumnsTableFixture dest1Table = new TwoColumnsTableFixture("SplitDataDestination1");
FourColumnsTableFixture dest2Table = new FourColumnsTableFixture("SplitDataDestination2");
var source = new CsvSource <CSVPoco>("res/Multicast/CsvSourceToSplit.csv")
{
Configuration = new CsvHelper.Configuration.Configuration()
{
Delimiter = ";"
}
};
var multicast = new Multicast <CSVPoco>();
var row1 = new RowTransformation <CSVPoco, Entity1>(input => {
return(new Entity1
{
Col1 = input.CSVCol1,
Col2 = input.CSVCol2
});
});
var row2 = new RowTransformation <CSVPoco, Entity2>(input => {
return(new Entity2
{
Col2 = input.CSVCol2,
Col3 = input.CSVCol3,
Col4 = input.CSVCol4
});
});
var destination1 = new DbDestination <Entity1>(Connection, "SplitDataDestination1");
var destination2 = new DbDestination <Entity2>(Connection, "SplitDataDestination2");
//Act
source.LinkTo(multicast);
multicast.LinkTo(row1);
multicast.LinkTo(row2);
row1.LinkTo(destination1);
row2.LinkTo(destination2);
source.Execute();
destination1.Wait();
destination2.Wait();
//Assert
dest1Table.AssertTestData();
dest2Table.AssertTestData();
}
public void MoreColumnsInDatabase()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvDestination2Columns");
DbDestination <string[]> dest = new DbDestination <string[]>(Connection, "CsvDestination2Columns");
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/OneColumn.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(Connection, "CsvDestination2Columns", "Col1 IN (1,2,3)"));
}
public void IdentityAtTheEnd()
{
//Arrange
FourColumnsTableFixture dest4Columns = new FourColumnsTableFixture("CsvDestination4Columns", identityColumnIndex: 3);
DbDestination <string[]> dest = new DbDestination <string[]>(Connection, "CsvDestination4Columns");
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/ThreeColumnsNoId.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest4Columns.AssertTestData();
}
public void SimpleFlowWithDynamicObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceDynamic");
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>("CsvSourceDynamic", SqlConnection);
//Act
CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/CsvSource/TwoColumnsForDynamic.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void MoreColumnsInCSV()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvDestination2Columns");
DbDestination <string[]> dest = new DbDestination <string[]>(Connection, "CsvDestination2Columns");
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/ThreeColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void WithoutErrorLinking()
{
//Arrange
MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();
//Act
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsErrorLinking.csv");
//Assert
Assert.Throws <CsvHelper.TypeConversion.TypeConverterException>(() =>
{
source.LinkTo(dest);
source.Execute();
dest.Wait();
});
}
public void SimpleFlowWithObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSource2Cols");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("CsvSource2Cols", Connection);
//Act
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void MoreColumnsInSource()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceDynamicColsInSource");
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(SqlConnection, "CsvSourceDynamicColsInSource");
//Act
CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/CsvSource/FourColumnsForDynamic.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CsvSourceNoHeader()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceNoHeader");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(Connection, "CsvSourceNoHeader");
//Act
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsNoHeader.csv");
source.Configuration.HasHeaderRecord = false;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
static void Main(string[] args)
{
//Set up the connection manager to master
var masterConnection = new SqlConnectionManager("Data Source=localhost;User Id=sa;Password=YourStrong@Passw0rd;");
//Recreate database
DropDatabaseTask.DropIfExists(masterConnection, "demo");
CreateDatabaseTask.Create(masterConnection, "demo");
//Get connection manager to previously create database
var dbConnection = new SqlConnectionManager("Data Source=localhost;User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=demo;");
//Create destination table
CreateTableTask.Create(dbConnection, "Table1", new List <TableColumn>()
{
new TableColumn("ID", "int", allowNulls: false, isPrimaryKey: true, isIdentity: true),
new TableColumn("Col1", "nvarchar(100)", allowNulls: true),
new TableColumn("Col2", "smallint", allowNulls: true)
});
//Create dataflow for loading data from csv into table
CsvSource <string[]> source = new CsvSource <string[]>("input.csv");
RowTransformation <string[], MyData> row = new RowTransformation <string[], MyData>(
input =>
new MyData()
{
Col1 = input[0], Col2 = input[1]
}
);
DbDestination <MyData> dest = new DbDestination <MyData>(dbConnection, "Table1");
//Link components & run data flow
source.LinkTo(row);
row.LinkTo(dest);
source.Execute();
dest.Wait();
//Check if data exists in destination
SqlTask.ExecuteReader(dbConnection, "Read all data from table1",
"select Col1, Col2 from Table1",
col1 => Console.WriteLine(col1.ToString() + ","),
col2 => Console.WriteLine(col2.ToString()));
Console.WriteLine("Press any key to continue...");
Console.ReadLine();
}
public void SimpleFlowUsingClassMap()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvDestination2ColumnsClassMap");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("CsvDestination2ColumnsClassMap", Connection);
//Act
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumns.csv");
source.Configuration.RegisterClassMap <ModelClassMap>();
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CSVGenericWithSkipRows_DB()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceSkipRows");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("CsvSourceSkipRows", Connection);
//Act
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsSkipRows.csv");
source.SkipRows = 2;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void CSVIntoAccess()
{
//Arrange
TableDefinition testTable = RecreateAccessTestTable();
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/UseCases/AccessData.csv");
DbDestination <string[]> dest = new DbDestination <string[]>(testTable, AccessOdbcConnection, batchSize: 2);
var linkTo = source.LinkTo(dest);
using (linkTo.link)
{
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(AccessOdbcConnection, testTable.Name));
}
}
public void UsingClassMapAndNoHeader()
{
//Arrange
FourColumnsTableFixture d4c = new FourColumnsTableFixture("CsvDestination4ColumnsClassMap");
DbDestination <MyExtendedRow> dest = new DbDestination <MyExtendedRow>("CsvDestination4ColumnsClassMap", Connection);
//Act
CsvSource <MyExtendedRow> source = new CsvSource <MyExtendedRow>("res/CsvSource/FourColumnsInvalidHeader.csv");
source.Configuration.RegisterClassMap <ExtendedClassMap>();
source.Configuration.HasHeaderRecord = false;
source.Configuration.ShouldSkipRecord = ShouldSkipRecordDelegate;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
d4c.AssertTestData();
}
public void Run()
{
PostgresConnectionManager conMan = new PostgresConnectionManager(PostgresConnectionString);
//Import CSV
CsvSource sourceCSV = new CsvSource("NameList.csv");
DbDestination importDest = new DbDestination(conMan, "NameTable");
sourceCSV.LinkTo(importDest);
sourceCSV.Execute();
importDest.Wait();
//Export again
DbSource <NameListElement> sourceTable = new DbSource <NameListElement>(conMan, "NameTable");
CsvDestination <NameListElement> destCSV = new CsvDestination <NameListElement>("Export.csv");
destCSV.Configuration.Delimiter = ";";
sourceTable.LinkTo(destCSV);
sourceTable.Execute();
destCSV.Wait();
}
public void CSVIntoAccess()
{
//Arrange
TableDefinition testTable = RecreateAccessTestTable();
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/Access/AccessData.csv");
DbDestination <string[]> dest = new DbDestination <string[]>(batchSize: 2)
{
DestinationTableDefinition = testTable,
ConnectionManager = AccessOdbcConnection
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(AccessOdbcConnection, testTable.Name));
}
public void ThrowExceptionWithoutHandling()
{
//Arrange
CsvSource <string[]> source = new CsvSource <string[]>("res/RowTransformation/TwoColumns.csv");
MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();
//Act
RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>(
csvdata => throw new InvalidOperationException("Test"));
source.LinkTo(trans);
trans.LinkTo(dest);
//Assert
Assert.Throws <AggregateException>(() =>
{
source.Execute();
dest.Wait();
});
}
private TimeSpan GetETLBoxTime <T>(int numberOfRows, CsvSource <T> source, DbDestination <T> dest)
{
source.LinkTo(dest);
var timeElapsedETLBox = BigDataHelper.LogExecutionTime($"Copying Csv into DB (non generic) with {numberOfRows} rows of data using ETLBox",
() =>
{
source.Execute();
dest.Wait();
}
);
if (typeof(T) == typeof(string[]))
{
output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Non generic).");
}
else
{
output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Generic).");
}
return(timeElapsedETLBox);
}
public void DuplicateCheckWithBlockTrans()
{
//Arrange
CsvSource <Poco> source = CreateDuplicateCsvSource("res/UseCases/DuplicateCheck.csv");
List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source
//Act
BlockTransformation <Poco> blockTrans = new BlockTransformation <Poco>(inputList =>
{
return(inputList.GroupBy(item => item.ID).Select(y => y.First()).ToList());
});
DbDestination <Poco> dest = CreateDestinationTable("dbo.DuplicateCheck");
source.LinkTo(blockTrans);
blockTrans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
AssertDataWithoutDuplicates();
}
