public void MoreColumnsInDestination() { //Arrange CreateTableTask.Create(SqlConnection, "CsvSourceDynamicColsInDest", new List <TableColumn>() { new TableColumn("Col2", "VARCHAR(100)", allowNulls: true), new TableColumn("Id", "INT", allowNulls: false, isPrimaryKey: true, isIdentity: true), new TableColumn("Col1", "INT", allowNulls: true), new TableColumn("ColX", "INT", allowNulls: true), }); DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(SqlConnection, "CsvSourceDynamicColsInDest"); //Act CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/CsvSource/TwoColumnsForDynamic.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(3, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest")); Assert.Equal(1, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest", $"Col1 = 1 AND Col2='Test1' AND Id > 0 AND ColX IS NULL")); Assert.Equal(1, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest", $"Col1 = 2 AND Col2='Test2' AND Id > 0 AND ColX IS NULL")); Assert.Equal(1, RowCountTask.Count(SqlConnection, "CsvSourceDynamicColsInDest", $"Col1 = 3 AND Col2='Test3' AND Id > 0 AND ColX IS NULL")); }
public void AfterBatchWrite(IConnectionManager connection) { //Arrange bool wasExecuted = false; TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DbDestinationBatchChanges"); DbDestination <string[]> dest = new DbDestination <string[]>(connection, "DbDestinationBatchChanges", batchSize: 1) { AfterBatchWrite = rowArray => { Assert.True(rowArray.Length == 1); wasExecuted = true; } }; //Act CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/TwoColumns.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(3, RowCountTask.Count(connection, "DbDestinationBatchChanges")); Assert.True(wasExecuted); }
public void ConvertIntoObject() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformationDynamic"); CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/RowTransformation/TwoColumns.csv"); //Act RowTransformation <ExpandoObject> trans = new RowTransformation <ExpandoObject>( csvdata => { dynamic c = csvdata as ExpandoObject; c.Col1 = c.Header1; c.Col2 = c.Header2; return(c); }); DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(Connection, "DestinationRowTransformationDynamic"); source.LinkTo(trans); trans.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void CheckMemoryUsage(IConnectionManager connection, int numberOfRows, int batchSize, double deviation) { //Arrange BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows); ReCreateDestinationTable(connection, "CsvDestinationWithTransformation"); var sourceExpando = new CsvSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows)); var trans = new RowTransformation <ExpandoObject, CSVData>( row => { dynamic r = row as ExpandoObject; return(new CSVData() { Col1 = r.Col1, Col2 = r.Col2, Col3 = r.Col3, Col4 = r.Col4 }); }); var destGeneric = new DbDestination <CSVData>(connection, "CsvDestinationWithTransformation", batchSize); sourceExpando.LinkTo(trans); trans.LinkTo(destGeneric); //Act long memAfter = 0; long memBefore = 0; bool startCheck = true; int count = 1; destGeneric.AfterBatchWrite = data => { if (count++ % 50 == 0) { using Process proc = Process.GetCurrentProcess(); memAfter = proc.WorkingSet64; if (startCheck) { memBefore = memAfter; startCheck = false; } Assert.True(memAfter < (memBefore + (memBefore * deviation))); } }; var timeElapsedETLBox = BigDataHelper.LogExecutionTime($"Copying Csv into DB (non generic) with {numberOfRows} rows of data using ETLBox", () => { sourceExpando.Execute(); destGeneric.Wait(); } ); output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Expando to object transformation)."); //Assert Assert.Equal(numberOfRows, RowCountTask.Count(connection, "CsvDestinationWithTransformation")); //10.000.000 rows, batch size 10.000: ~8 min //10.000.000 rows, batch size 1.000: ~10 min 10 sec }
public void InitAction() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation"); CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/RowTransformation/TwoColumnsIdMinus1.csv"); //Act int IdOffset = 0; RowTransformation <MySimpleRow, MySimpleRow> trans = new RowTransformation <MySimpleRow, MySimpleRow>() { TransformationFunc = row => { row.Col1 += IdOffset; return(row); }, InitAction = () => IdOffset += 1 }; DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(Connection, "DestinationRowTransformation"); source.LinkTo(trans); trans.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void ConvertIntoObject() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation"); CsvSource <string[]> source = new CsvSource <string[]>("res/RowTransformation/TwoColumns.csv"); //Act RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>( csvdata => { return(new MySimpleRow() { Col1 = int.Parse(csvdata[0]), Col2 = csvdata[1] }); }); DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(Connection, "DestinationRowTransformation"); source.LinkTo(trans); trans.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void WithBatchChanges(IConnectionManager connection) { //Arrange TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DbDestinationBatchChanges"); DbDestination <string[]> dest = new DbDestination <string[]>(connection, "DbDestinationBatchChanges", batchSize: 2) { BeforeBatchWrite = rowArray => { rowArray[0][1] = "NewValue"; return(rowArray); } }; //Act CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/TwoColumns.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(3, RowCountTask.Count(connection, "DbDestinationBatchChanges")); Assert.Equal(2, RowCountTask.Count(connection, "DbDestinationBatchChanges", $"{d2c.QB}Col2{d2c.QE}='NewValue'")); Assert.Equal(1, RowCountTask.Count(connection, "DbDestinationBatchChanges", $"{d2c.QB}Col1{d2c.QE} = 2 AND {d2c.QB}Col2{d2c.QE}='Test2'")); }
static void Main(string[] args) { Preparation(); SqlConnectionManager connMan = new SqlConnectionManager("Data Source=.;Initial Catalog=demo;Integrated Security=false;User=sa;password=reallyStrongPwd123"); CsvSource <string[]> source = new CsvSource <string[]>("demodata.csv"); RowTransformation <string[], Order> rowTrans = new RowTransformation <string[], Order>( row => new Order() { Id = int.Parse(row[0]), Item = row[1], Quantity = int.Parse(row[2]) + int.Parse(row[3]), Price = double.Parse(row[4]) * 100 }); DbDestination <Order> dest = new DbDestination <Order>(connMan, "OrderTable"); source.LinkTo(rowTrans); rowTrans.LinkTo(dest); source.Execute(); dest.Wait(); Console.WriteLine("Press any key to continue..."); Console.ReadLine(); }
static void Main(string[] args) { //Set up database Preparation(); //Define the source CsvSource <string[]> source = new CsvSource <string[]>("demodata.csv"); //Define the transformation RowTransformation <string[], Order> rowTrans = new RowTransformation <string[], Order>( row => new Order() { Item = row[1], Quantity = int.Parse(row[2]) + int.Parse(row[3]), Price = int.Parse(row[4]) / 100 }); //DbDestination needs a connection manager pointing to the right database SqlConnectionManager connMan = new SqlConnectionManager("Data Source=localhost;User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=demo;"); //Define the destination DbDestination <Order> dest = new DbDestination <Order>(connMan, "OrderTable"); //Link & run flow source.LinkTo(rowTrans); rowTrans.LinkTo(dest); source.Execute(); dest.Wait(); Console.WriteLine("Press any key to continue..."); Console.ReadLine(); }
public void WithObjectErrorLinking() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceErrorLinking"); DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("CsvSourceErrorLinking", SqlConnection); MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>(); //Act CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsErrorLinking.csv"); source.LinkTo(dest); source.LinkErrorTo(errorDest); source.Execute(); dest.Wait(); errorDest.Wait(); //Assert dest2Columns.AssertTestData(); Assert.Collection <ETLBoxError>(errorDest.Data, d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)), d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)), d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)), d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)) ); }
public static void Run() { Console.WriteLine("Running data flow"); //Read data from csv file CsvSource sourceOrderData = new CsvSource("DemoData.csv"); sourceOrderData.Configuration.Delimiter = ";"; //Transform into Order object RowTransformation <ExpandoObject, Order> transIntoObject = new RowTransformation <ExpandoObject, Order>( csvLine => { dynamic order = csvLine as dynamic; return(new Order() { //Header in Csv: OrderNumber;OrderItem;OrderAmount;CustomerName Number = order.OrderNumber, Item = order.OrderItem, Amount = decimal.Parse(order.OrderAmount.ToString().Replace("€", ""), CultureInfo.GetCultureInfo("en-US")), CustomerName = order.CustomerName }); }); sourceOrderData.LinkTo(transIntoObject); //Find corresponding customer id if customer exists in Customer table DbSource <Customer> sourceCustomerData = new DbSource <Customer>("customer"); LookupTransformation <Order, Customer> lookupCustomerKey = new LookupTransformation <Order, Customer>(sourceCustomerData); transIntoObject.LinkTo(lookupCustomerKey); //Split data Multicast <Order> multiCast = new Multicast <Order>(); lookupCustomerKey.LinkTo(multiCast); //Store Order in Orders table DbDestination <Order> destOrderTable = new DbDestination <Order>("orders"); multiCast.LinkTo(destOrderTable); //Create rating for existing customers based total of order amount Aggregation <Order, Rating> aggregation = new Aggregation <Order, Rating>(); multiCast.LinkTo(aggregation); //Store the rating in the customer rating table DbDestination <Rating> destRating = new DbDestination <Rating>("customer_rating"); aggregation.LinkTo(destRating); //Execute the data flow synchronously sourceOrderData.Execute(); destOrderTable.Wait(); destRating.Wait(); }
public void SplitCsvSourceIn2Tables() { //Arrange TwoColumnsTableFixture dest1Table = new TwoColumnsTableFixture("SplitDataDestination1"); FourColumnsTableFixture dest2Table = new FourColumnsTableFixture("SplitDataDestination2"); var source = new CsvSource <CSVPoco>("res/Multicast/CsvSourceToSplit.csv") { Configuration = new CsvHelper.Configuration.Configuration() { Delimiter = ";" } }; var multicast = new Multicast <CSVPoco>(); var row1 = new RowTransformation <CSVPoco, Entity1>(input => { return(new Entity1 { Col1 = input.CSVCol1, Col2 = input.CSVCol2 }); }); var row2 = new RowTransformation <CSVPoco, Entity2>(input => { return(new Entity2 { Col2 = input.CSVCol2, Col3 = input.CSVCol3, Col4 = input.CSVCol4 }); }); var destination1 = new DbDestination <Entity1>(Connection, "SplitDataDestination1"); var destination2 = new DbDestination <Entity2>(Connection, "SplitDataDestination2"); //Act source.LinkTo(multicast); multicast.LinkTo(row1); multicast.LinkTo(row2); row1.LinkTo(destination1); row2.LinkTo(destination2); source.Execute(); destination1.Wait(); destination2.Wait(); //Assert dest1Table.AssertTestData(); dest2Table.AssertTestData(); }
public void MoreColumnsInDatabase() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvDestination2Columns"); DbDestination <string[]> dest = new DbDestination <string[]>(Connection, "CsvDestination2Columns"); //Act CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/OneColumn.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(3, RowCountTask.Count(Connection, "CsvDestination2Columns", "Col1 IN (1,2,3)")); }
public void IdentityAtTheEnd() { //Arrange FourColumnsTableFixture dest4Columns = new FourColumnsTableFixture("CsvDestination4Columns", identityColumnIndex: 3); DbDestination <string[]> dest = new DbDestination <string[]>(Connection, "CsvDestination4Columns"); //Act CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/ThreeColumnsNoId.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest4Columns.AssertTestData(); }
public void SimpleFlowWithDynamicObject() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceDynamic"); DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>("CsvSourceDynamic", SqlConnection); //Act CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/CsvSource/TwoColumnsForDynamic.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void MoreColumnsInCSV() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvDestination2Columns"); DbDestination <string[]> dest = new DbDestination <string[]>(Connection, "CsvDestination2Columns"); //Act CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/ThreeColumns.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void WithoutErrorLinking() { //Arrange MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>(); //Act CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsErrorLinking.csv"); //Assert Assert.Throws <CsvHelper.TypeConversion.TypeConverterException>(() => { source.LinkTo(dest); source.Execute(); dest.Wait(); }); }
public void SimpleFlowWithObject() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSource2Cols"); DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("CsvSource2Cols", Connection); //Act CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumns.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void MoreColumnsInSource() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceDynamicColsInSource"); DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(SqlConnection, "CsvSourceDynamicColsInSource"); //Act CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/CsvSource/FourColumnsForDynamic.csv"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void CsvSourceNoHeader() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceNoHeader"); DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(Connection, "CsvSourceNoHeader"); //Act CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsNoHeader.csv"); source.Configuration.HasHeaderRecord = false; source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
static void Main(string[] args) { //Set up the connection manager to master var masterConnection = new SqlConnectionManager("Data Source=localhost;User Id=sa;Password=YourStrong@Passw0rd;"); //Recreate database DropDatabaseTask.DropIfExists(masterConnection, "demo"); CreateDatabaseTask.Create(masterConnection, "demo"); //Get connection manager to previously create database var dbConnection = new SqlConnectionManager("Data Source=localhost;User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=demo;"); //Create destination table CreateTableTask.Create(dbConnection, "Table1", new List <TableColumn>() { new TableColumn("ID", "int", allowNulls: false, isPrimaryKey: true, isIdentity: true), new TableColumn("Col1", "nvarchar(100)", allowNulls: true), new TableColumn("Col2", "smallint", allowNulls: true) }); //Create dataflow for loading data from csv into table CsvSource <string[]> source = new CsvSource <string[]>("input.csv"); RowTransformation <string[], MyData> row = new RowTransformation <string[], MyData>( input => new MyData() { Col1 = input[0], Col2 = input[1] } ); DbDestination <MyData> dest = new DbDestination <MyData>(dbConnection, "Table1"); //Link components & run data flow source.LinkTo(row); row.LinkTo(dest); source.Execute(); dest.Wait(); //Check if data exists in destination SqlTask.ExecuteReader(dbConnection, "Read all data from table1", "select Col1, Col2 from Table1", col1 => Console.WriteLine(col1.ToString() + ","), col2 => Console.WriteLine(col2.ToString())); Console.WriteLine("Press any key to continue..."); Console.ReadLine(); }
public void SimpleFlowUsingClassMap() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvDestination2ColumnsClassMap"); DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("CsvDestination2ColumnsClassMap", Connection); //Act CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumns.csv"); source.Configuration.RegisterClassMap <ModelClassMap>(); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void CSVGenericWithSkipRows_DB() { //Arrange TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceSkipRows"); DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("CsvSourceSkipRows", Connection); //Act CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsSkipRows.csv"); source.SkipRows = 2; source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert dest2Columns.AssertTestData(); }
public void CSVIntoAccess() { //Arrange TableDefinition testTable = RecreateAccessTestTable(); //Act CsvSource <string[]> source = new CsvSource <string[]>("res/UseCases/AccessData.csv"); DbDestination <string[]> dest = new DbDestination <string[]>(testTable, AccessOdbcConnection, batchSize: 2); var linkTo = source.LinkTo(dest); using (linkTo.link) { source.Execute(); dest.Wait(); //Assert Assert.Equal(3, RowCountTask.Count(AccessOdbcConnection, testTable.Name)); } }
public void UsingClassMapAndNoHeader() { //Arrange FourColumnsTableFixture d4c = new FourColumnsTableFixture("CsvDestination4ColumnsClassMap"); DbDestination <MyExtendedRow> dest = new DbDestination <MyExtendedRow>("CsvDestination4ColumnsClassMap", Connection); //Act CsvSource <MyExtendedRow> source = new CsvSource <MyExtendedRow>("res/CsvSource/FourColumnsInvalidHeader.csv"); source.Configuration.RegisterClassMap <ExtendedClassMap>(); source.Configuration.HasHeaderRecord = false; source.Configuration.ShouldSkipRecord = ShouldSkipRecordDelegate; source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert d4c.AssertTestData(); }
public void Run() { PostgresConnectionManager conMan = new PostgresConnectionManager(PostgresConnectionString); //Import CSV CsvSource sourceCSV = new CsvSource("NameList.csv"); DbDestination importDest = new DbDestination(conMan, "NameTable"); sourceCSV.LinkTo(importDest); sourceCSV.Execute(); importDest.Wait(); //Export again DbSource <NameListElement> sourceTable = new DbSource <NameListElement>(conMan, "NameTable"); CsvDestination <NameListElement> destCSV = new CsvDestination <NameListElement>("Export.csv"); destCSV.Configuration.Delimiter = ";"; sourceTable.LinkTo(destCSV); sourceTable.Execute(); destCSV.Wait(); }
public void CSVIntoAccess() { //Arrange TableDefinition testTable = RecreateAccessTestTable(); //Act CsvSource <string[]> source = new CsvSource <string[]>("res/Access/AccessData.csv"); DbDestination <string[]> dest = new DbDestination <string[]>(batchSize: 2) { DestinationTableDefinition = testTable, ConnectionManager = AccessOdbcConnection }; source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(3, RowCountTask.Count(AccessOdbcConnection, testTable.Name)); }
public void ThrowExceptionWithoutHandling() { //Arrange CsvSource <string[]> source = new CsvSource <string[]>("res/RowTransformation/TwoColumns.csv"); MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>(); //Act RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>( csvdata => throw new InvalidOperationException("Test")); source.LinkTo(trans); trans.LinkTo(dest); //Assert Assert.Throws <AggregateException>(() => { source.Execute(); dest.Wait(); }); }
private TimeSpan GetETLBoxTime <T>(int numberOfRows, CsvSource <T> source, DbDestination <T> dest) { source.LinkTo(dest); var timeElapsedETLBox = BigDataHelper.LogExecutionTime($"Copying Csv into DB (non generic) with {numberOfRows} rows of data using ETLBox", () => { source.Execute(); dest.Wait(); } ); if (typeof(T) == typeof(string[])) { output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Non generic)."); } else { output.WriteLine("Elapsed " + timeElapsedETLBox.TotalSeconds + " seconds for ETLBox (Generic)."); } return(timeElapsedETLBox); }
public void DuplicateCheckWithBlockTrans() { //Arrange CsvSource <Poco> source = CreateDuplicateCsvSource("res/UseCases/DuplicateCheck.csv"); List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source //Act BlockTransformation <Poco> blockTrans = new BlockTransformation <Poco>(inputList => { return(inputList.GroupBy(item => item.ID).Select(y => y.First()).ToList()); }); DbDestination <Poco> dest = CreateDestinationTable("dbo.DuplicateCheck"); source.LinkTo(blockTrans); blockTrans.LinkTo(dest); source.Execute(); dest.Wait(); //Assert AssertDataWithoutDuplicates(); }