public void AfterBatchWrite(IConnectionManager connection)
{
//Arrange
bool wasExecuted = false;
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DbDestinationBatchChanges");
DbDestination <string[]> dest = new DbDestination <string[]>(connection, "DbDestinationBatchChanges", batchSize: 1)
{
AfterBatchWrite = rowArray =>
{
Assert.True(rowArray.Length == 1);
wasExecuted = true;
}
};
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/TwoColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(connection, "DbDestinationBatchChanges"));
Assert.True(wasExecuted);
}
public void UsingTableDefinitions(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture(connection, "SourceTableDef");
source2Columns.InsertTestData();
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture(connection, "DestinationTableDef");
//Act
DBSource source = new DBSource()
{
SourceTableDefinition = source2Columns.TableDefinition,
ConnectionManager = connection
};
DBDestination dest = new DBDestination()
{
DestinationTableDefinition = dest2Columns.TableDefinition,
ConnectionManager = connection
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void NoIdColumn(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture s2c = new TwoColumnsTableFixture(connection, "DBMergeSource");
s2c.InsertTestData();
s2c.InsertTestDataSet2();
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DBMergeDestination");
d2c.InsertTestDataSet3();
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>("DBMergeSource", connection);
//Act
DbMerge <MySimpleRow> dest = new DbMerge <MySimpleRow>(MergeMode.Full, "DBMergeDestination", connection);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.True(dest.UseTruncateMethod == true);
Assert.Equal(6, RowCountTask.Count(connection, "DBMergeDestination", $"{d2c.QB}Col1{d2c.QE} BETWEEN 1 AND 7 AND {d2c.QB}Col2{d2c.QE} LIKE 'Test%'"));
Assert.True(dest.DeltaTable.Count == 7);
Assert.True(dest.DeltaTable.Where(row => row.ChangeAction == ChangeAction.Update).Count() == 3);
Assert.True(dest.DeltaTable.Where(row => row.ChangeAction == ChangeAction.Delete && row.Key == 10).Count() == 1);
Assert.True(dest.DeltaTable.Where(row => row.ChangeAction == ChangeAction.Insert).Count() == 3);
}
public void ColumnMapping(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture(connection, "SpecialCharacterSource");
InsertTestData(connection, "SpecialCharacterSource");
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture(connection, "SpecialCharacterDestination");
//Act
DBSource source = new DBSource()
{
ConnectionManager = connection,
SourceTableDefinition = source2Columns.TableDefinition
};
DBDestination dest = new DBDestination()
{
ConnectionManager = connection,
DestinationTableDefinition = dest2Columns.TableDefinition
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(4, RowCountTask.Count(connection, "SpecialCharacterDestination"));
}
public void UsingDifferentObjectTypes()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("SourceMultipleLinks");
source2Columns.InsertTestData();
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationMultipleLinks");
DBSource <MySimpleRow> source = new DBSource <MySimpleRow>(SqlConnection, "SourceMultipleLinks");
DBDestination <MyOtherRow> dest = new DBDestination <MyOtherRow>(SqlConnection, "DestinationMultipleLinks");
RowTransformation <MySimpleRow, MyOtherRow> trans1 = new RowTransformation <MySimpleRow, MyOtherRow>
(row =>
{
return(new MyOtherRow()
{
ColA = row.Col1,
ColB = row.Col2
});
}
);
//Act
source.LinkTo <MyOtherRow>(trans1).LinkTo(dest);
//Assert
source.Execute();
dest.Wait();
dest2Columns.AssertTestData();
}
public void SplitInto2Tables()
{
//Arrange
TwoColumnsTableFixture sourceTable = new TwoColumnsTableFixture("Source");
sourceTable.InsertTestData();
TwoColumnsTableFixture dest1Table = new TwoColumnsTableFixture("Destination1");
TwoColumnsTableFixture dest2Table = new TwoColumnsTableFixture("Destination2");
DbSource <string[]> source = new DbSource <string[]>("Source", Connection);
DbDestination <string[]> dest1 = new DbDestination <string[]>("Destination1", Connection);
DbDestination <string[]> dest2 = new DbDestination <string[]>("Destination2", Connection);
//Act
Multicast <string[]> multicast = new Multicast <string[]>();
source.LinkTo(multicast);
multicast.LinkTo(dest1);
multicast.LinkTo(dest2);
source.Execute();
dest1.Wait();
dest2.Wait();
//Assert
dest1Table.AssertTestData();
dest2Table.AssertTestData();
}
public void WithPredicate()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("RowDuplicationSource");
source2Columns.InsertTestData();
DbSource source = new DbSource(SqlConnection, "RowDuplicationSource");
RowDuplication duplication = new RowDuplication(
row =>
{
dynamic r = row as dynamic;
return(r.Col1 == 1 || r.Col2 == "Test3");
});
MemoryDestination dest = new MemoryDestination();
//Act
source.LinkTo(duplication);
duplication.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection(dest.Data,
row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 1 && d.Col2 == "Test1"); },
row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 1 && d.Col2 == "Test1"); },
row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 2 && d.Col2 == "Test2"); },
row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 3 && d.Col2 == "Test3"); },
row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 3 && d.Col2 == "Test3"); }
);
}
public void CommitTransaction(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture s2c = new TwoColumnsTableFixture(connection, "TransactionSource");
s2c.InsertTestData();
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "TransactionDest");
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(connection, "TransactionSource");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(connection, "TransactionDest", batchSize: 2);
//Act & Assert
connection.BeginTransaction(System.Data.IsolationLevel.ReadCommitted);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
if (connection.GetType() == typeof(SqlConnectionManager))
{
Assert.Equal(3, RowCountTask.Count(connection.Clone(), "TransactionDest", RowCountOptions.NoLock));
}
connection.CommitTransaction();
Assert.Equal(3, RowCountTask.Count(connection, "TransactionDest"));
Assert.Equal(3, RowCountTask.Count(connection.Clone(), "TransactionDest"));
//Assert Connections are closed
Assert.True(dest.BulkInsertConnectionManager.State == null);
Assert.True(connection.State == null);
}
public void DynamicObjectJoin()
{
//Arrange
TwoColumnsTableFixture table1 = new TwoColumnsTableFixture(SqlConnection, "CrossJoinSource1");
table1.InsertTestData();
TwoColumnsTableFixture table2 = new TwoColumnsTableFixture(SqlConnection, "CrossJoinSource2");
table2.InsertTestData();
DbSource <ExpandoObject> source1 = new DbSource <ExpandoObject>(SqlConnection, "CrossJoinSource1");
DbSource <ExpandoObject> source2 = new DbSource <ExpandoObject>(SqlConnection, "CrossJoinSource2");
MemoryDestination dest = new MemoryDestination();
CrossJoin crossJoin = new CrossJoin(
(data1, data2) =>
{
dynamic d1 = data1 as dynamic;
dynamic d2 = data1 as dynamic;
dynamic res = new ExpandoObject();
res.Val = d1.Col1 + d2.Col2;
return(res);
}
);
//Act
source1.LinkTo(crossJoin.InMemoryTarget);
source2.LinkTo(crossJoin.PassingTarget);
crossJoin.LinkTo(dest);
source1.Execute();
source2.Execute();
dest.Wait();
//Assert
Assert.Equal(9, dest.Data.Count);
}
public void TwoTransactionsAndParallelWriting(IConnectionManager connection)
{
if (connection.ConnectionManagerType == ConnectionManagerType.SQLite)
{
return;
}
//Arrange
var concopy = connection.Clone();
TwoColumnsTableFixture s2c = new TwoColumnsTableFixture(connection, "TransactionSourceParallelWrite");
s2c.InsertTestData();
TwoColumnsTableFixture d2c1 = new TwoColumnsTableFixture(connection, "TransactionDest1");
TwoColumnsTableFixture d2c2 = new TwoColumnsTableFixture(connection, "TransactionDest2");
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(connection, "TransactionSourceParallelWrite");
DbDestination <MySimpleRow> dest1 = new DbDestination <MySimpleRow>(connection, "TransactionDest1", batchSize: 2);
DbDestination <MySimpleRow> dest2 = new DbDestination <MySimpleRow>(concopy, "TransactionDest2", batchSize: 2);
Multicast <MySimpleRow> multicast = new Multicast <MySimpleRow>();
//Act & Assert
connection.BeginTransaction(System.Data.IsolationLevel.ReadCommitted);
concopy.BeginTransaction(System.Data.IsolationLevel.ReadCommitted);
source.LinkTo(multicast);
multicast.LinkTo(dest1);
multicast.LinkTo(dest2);
source.Execute();
dest1.Wait();
dest2.Wait();
connection.CommitTransaction();
concopy.CommitTransaction();
Assert.Equal(3, RowCountTask.Count(connection, "TransactionDest1"));
Assert.Equal(3, RowCountTask.Count(connection, "TransactionDest2"));
}
public void OneTransactionAndParallelWritingWithMARS()
{
//Arrange
TwoColumnsTableFixture s2c = new TwoColumnsTableFixture(SqlConnection, "TransactionSourceParallelWrite");
s2c.InsertTestData();
TwoColumnsTableFixture d2c1 = new TwoColumnsTableFixture(SqlConnection, "TransactionDest1");
TwoColumnsTableFixture d2c2 = new TwoColumnsTableFixture(SqlConnection, "TransactionDest2");
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(SqlConnection, "TransactionSourceParallelWrite");
string constring = $"{Config.SqlConnection.RawConnectionString("DataFlow")};MultipleActiveResultSets=True;";
var marscon = new SqlConnectionManager(constring);
DbDestination <MySimpleRow> dest1 = new DbDestination <MySimpleRow>(marscon, "TransactionDest1", batchSize: 2);
DbDestination <MySimpleRow> dest2 = new DbDestination <MySimpleRow>(marscon, "TransactionDest2", batchSize: 2);
Multicast <MySimpleRow> multicast = new Multicast <MySimpleRow>();
//Act & Assert
marscon.BeginTransaction(System.Data.IsolationLevel.ReadCommitted);
source.LinkTo(multicast);
multicast.LinkTo(dest1);
multicast.LinkTo(dest2);
source.Execute();
dest1.Wait();
dest2.Wait();
marscon.CommitTransaction();
d2c1.AssertTestData();
d2c1.AssertTestData();
}
public void LeaveOpen(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture s2c = new TwoColumnsTableFixture(connection, "TransactionSource");
s2c.InsertTestData();
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "TransactionDest");
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(connection, "TransactionSource");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(connection, "TransactionDest", batchSize: 2);
//Act
connection.LeaveOpen = true;
connection.BeginTransaction(System.Data.IsolationLevel.ReadCommitted);
source.LinkTo(dest);
source.Execute();
dest.Wait();
connection.CommitTransaction();
//Assert
Assert.Equal(connection, dest.BulkInsertConnectionManager);
//Assert Connections are closed
Assert.True(dest.BulkInsertConnectionManager.State == System.Data.ConnectionState.Open);
Assert.True(connection.State == System.Data.ConnectionState.Open);
Assert.Equal(3, RowCountTask.Count(connection, "TransactionDest"));
}
public void SimpleFlow()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("Destination4CustomSource");
List <string> Data = new List <string>()
{
"Test1", "Test2", "Test3"
};
int _readIndex = 0;
Func <MySimpleRow> ReadData = () =>
{
var result = new MySimpleRow()
{
Col1 = _readIndex + 1,
Col2 = Data[_readIndex]
};
_readIndex++;
return(result);
};
Func <bool> EndOfData = () => _readIndex >= Data.Count;
//Act
CustomSource <MySimpleRow> source = new CustomSource <MySimpleRow>(ReadData, EndOfData);
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "Destination4CustomSource");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void DataIsFromList()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("MemoryDestination");
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(SqlConnection, "MemoryDestination");
//Act
source.Data = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
},
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
}
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void SqlWithNamedColumns(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture s2c = new TwoColumnsTableFixture(connection, "SourceSql");
s2c.InsertTestData();
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DestinationSql");
//Act
DBSource <MySimpleRow> source = new DBSource <MySimpleRow>()
{
Sql = $@"SELECT CASE WHEN {s2c.QB}Col1{s2c.QE} IS NOT NULL THEN {s2c.QB}Col1{s2c.QE} ELSE {s2c.QB}Col1{s2c.QE} END AS {s2c.QB}Col1{s2c.QE},
{s2c.QB}Col2{s2c.QE}
FROM {s2c.QB}SourceSql{s2c.QE}",
ConnectionManager = connection
};
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(connection, "DestinationSql");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
d2c.AssertTestData();
}
public void SimpleMerge(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture s2c = new TwoColumnsTableFixture(connection, "DBMergeSource");
s2c.InsertTestData();
s2c.InsertTestDataSet2();
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DBMergeDestination");
d2c.InsertTestDataSet3();
DBSource <MyMergeRow> source = new DBSource <MyMergeRow>(connection, "DBMergeSource");
//Act
DBMerge <MyMergeRow> dest = new DBMerge <MyMergeRow>(connection, "DBMergeDestination");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(6, RowCountTask.Count(connection, "DBMergeDestination", $"{d2c.QB}Col1{d2c.QE} BETWEEN 1 AND 7 AND {d2c.QB}Col2{d2c.QE} LIKE 'Test%'"));
Assert.True(dest.DeltaTable.Count == 7);
Assert.True(dest.DeltaTable.Where(row => row.ChangeAction == "U").Count() == 2);
Assert.True(dest.DeltaTable.Where(row => row.ChangeAction == "D" && row.Key == 10).Count() == 1);
Assert.True(dest.DeltaTable.Where(row => row.ChangeAction == "I").Count() == 3);
Assert.True(dest.DeltaTable.Where(row => row.ChangeAction == "E" && row.Key == 1).Count() == 1);
}
public void JsonPathInEpandoObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("JsonSourceNestedDynamic");
RowTransformation <ExpandoObject> trans = new RowTransformation <ExpandoObject>(
row =>
{
dynamic r = row as ExpandoObject;
r.Col1 = r.Column1;
return(r);
});
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(SqlConnection, "JsonSourceNestedDynamic");
//Act
JsonSource <ExpandoObject> source = new JsonSource <ExpandoObject>("res/JsonSource/NestedData.json", ResourceType.File);
List <JsonProperty2JsonPath> pathLookups = new List <JsonProperty2JsonPath>()
{
new JsonProperty2JsonPath()
{
JsonPropertyName = "Column2",
JsonPath = "Value",
NewPropertyName = "Col2"
}
};
source.JsonSerializer.Converters.Add(new ExpandoJsonPathConverter(pathLookups));
source.LinkTo(trans).LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void ConvertIntoObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation");
CSVSource source = new CSVSource("res/RowTransformation/TwoColumns.csv");
//Act
RowTransformation <string[], MySimpleRow> trans = new RowTransformation <string[], MySimpleRow>(
csvdata =>
{
return(new MySimpleRow()
{
Col1 = int.Parse(csvdata[0]),
Col2 = csvdata[1]
});
});
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "DestinationRowTransformation");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void InsertIntoTable()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("Source");
source2Columns.InsertTestData();
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CustomDestination");
//Act
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(SqlConnection, "Source");
CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>(
row =>
{
SqlTask.ExecuteNonQuery(SqlConnection, "Insert row",
$"INSERT INTO dbo.CustomDestination VALUES({row.Col1},'{row.Col2}')");
}
);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void InitAction()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformation");
CSVSource <MySimpleRow> source = new CSVSource <MySimpleRow>("res/RowTransformation/TwoColumnsIdMinus1.csv");
//Act
int IdOffset = 0;
RowTransformation <MySimpleRow, MySimpleRow> trans = new RowTransformation <MySimpleRow, MySimpleRow>(
"RowTransformation testing init Action",
row =>
{
row.Col1 += IdOffset;
return(row);
},
() => IdOffset += 1
);
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "DestinationRowTransformation");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void PredicateFilteringWithInteger()
{
//Arrange
TwoColumnsTableFixture sourceTable = new TwoColumnsTableFixture("Source");
sourceTable.InsertTestData();
TwoColumnsTableFixture dest1Table = new TwoColumnsTableFixture("Destination1");
TwoColumnsTableFixture dest2Table = new TwoColumnsTableFixture("Destination2");
DBSource <MySimpleRow> source = new DBSource <MySimpleRow>(Connection, "Source");
DBDestination <MySimpleRow> dest1 = new DBDestination <MySimpleRow>(Connection, "Destination1");
DBDestination <MySimpleRow> dest2 = new DBDestination <MySimpleRow>(Connection, "Destination2");
//Act
Multicast <MySimpleRow> multicast = new Multicast <MySimpleRow>();
source.LinkTo(multicast);
multicast.LinkTo(dest1, row => row.Col1 <= 2);
multicast.LinkTo(dest2, row => row.Col1 > 2);
source.Execute();
dest1.Wait();
dest2.Wait();
//Assert
Assert.Equal(1, RowCountTask.Count(Connection, "Destination1", "Col1 = 1 AND Col2='Test1'"));
Assert.Equal(1, RowCountTask.Count(Connection, "Destination1", "Col1 = 2 AND Col2='Test2'"));
Assert.Equal(1, RowCountTask.Count(Connection, "Destination2", "Col1 = 3 AND Col2='Test3'"));
}
public void WithSqlNotMatchingColumns(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture s2c = new TwoColumnsTableFixture(connection, "SourceNotMatchingCols");
s2c.InsertTestData();
SqlTask.ExecuteNonQuery(connection, "Create destination table",
$@"CREATE TABLE destination_notmatchingcols
( col3 VARCHAR(100) NULL
, col4 VARCHAR(100) NULL
, {s2c.QB}Col1{s2c.QE} VARCHAR(100) NULL)");
//Act
DBSource source = new DBSource()
{
Sql = $"SELECT {s2c.QB}Col1{s2c.QE}, {s2c.QB}Col2{s2c.QE} FROM {s2c.QB}SourceNotMatchingCols{s2c.QE}",
ConnectionManager = connection
};
DBDestination dest = new DBDestination(connection, "destination_notmatchingcols");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(connection, "destination_notmatchingcols"));
Assert.Equal(1, RowCountTask.Count(connection, "destination_notmatchingcols", $"col3 = '1' AND col4='Test1'"));
Assert.Equal(1, RowCountTask.Count(connection, "destination_notmatchingcols", $"col3 = '2' AND col4='Test2'"));
Assert.Equal(1, RowCountTask.Count(connection, "destination_notmatchingcols", $"col3 = '3' AND col4='Test3'"));
}
public void SimpleDataNoHeader()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("ExcelDestinationDynamic");
//Act
ExcelSource source = new ExcelSource("res/Excel/TwoColumnShiftedData.xlsx")
{
Range = new ExcelRange(3, 4),
HasNoHeader = true
};
RowTransformation trans = new RowTransformation(row =>
{
dynamic r = row as dynamic;
r.Col1 = r.Column1;
r.Col2 = r.Column2;
return(r);
});
DbDestination dest = new DbDestination(Connection, "ExcelDestinationDynamic");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void DataIsInList()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("RowDuplicationStringArraySource");
source2Columns.InsertTestData();
DbSource <string[]> source = new DbSource <string[]>(SqlConnection, "RowDuplicationStringArraySource");
RowDuplication <string[]> duplication = new RowDuplication <string[]>();
MemoryDestination <string[]> dest = new MemoryDestination <string[]>();
//Act
source.LinkTo(duplication);
duplication.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection(dest.Data,
d => Assert.True(d[0] == "1" && d[1] == "Test1"),
d => Assert.True(d[0] == "1" && d[1] == "Test1"),
d => Assert.True(d[0] == "2" && d[1] == "Test2"),
d => Assert.True(d[0] == "2" && d[1] == "Test2"),
d => Assert.True(d[0] == "3" && d[1] == "Test3"),
d => Assert.True(d[0] == "3" && d[1] == "Test3")
);
}
public void ConvertIntoObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("DestinationRowTransformationDynamic");
CsvSource <ExpandoObject> source = new CsvSource <ExpandoObject>("res/RowTransformation/TwoColumns.csv");
//Act
RowTransformation <ExpandoObject> trans = new RowTransformation <ExpandoObject>(
csvdata =>
{
dynamic c = csvdata as ExpandoObject;
c.Col1 = c.Header1;
c.Col2 = c.Header2;
return(c);
});
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(Connection, "DestinationRowTransformationDynamic");
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void RedirectErrorWithObject(IConnectionManager connection)
{
if (connection.GetType() == typeof(SQLiteConnectionManager))
{
Task.Delay(100).Wait(); //Database was locked and needs to recover after exception
}
//Arrange
CreateSourceTable(connection, "DbSourceErrorLinking");
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture(connection, "DbDestinationErrorLinking");
//Act
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(connection, "DbSourceErrorLinking");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(connection, "DbDestinationErrorLinking");
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
source.LinkTo(dest);
source.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
dest2Columns.AssertTestData();
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public void SortSimpleDataDescending()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("SortSource");
source2Columns.InsertTestData();
DBSource <MySimpleRow> source = new DBSource <MySimpleRow>(Connection, "SortSource");
//Act
List <MySimpleRow> actual = new List <MySimpleRow>();
CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>(
row => actual.Add(row)
);
Comparison <MySimpleRow> comp = new Comparison <MySimpleRow>(
(x, y) => y.Col1 - x.Col1
);
Sort <MySimpleRow> block = new Sort <MySimpleRow>(comp);
source.LinkTo(block);
block.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
List <int> expected = new List <int>()
{
3, 2, 1
};
Assert.Equal(expected, actual.Select(row => row.Col1).ToList());
}
public void SimpleFlow()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("Destination4CustomSourceDynamic");
List <string> Data = new List <string>()
{
"Test1", "Test2", "Test3"
};
int _readIndex = 0;
Func <ExpandoObject> ReadData = () =>
{
dynamic result = new ExpandoObject();
result.Col1 = (_readIndex + 1).ToString();
result.Col2 = Data[_readIndex];
_readIndex++;
return(result);
};
Func <bool> EndOfData = () => _readIndex >= Data.Count;
//Act
CustomSource <ExpandoObject> source = new CustomSource <ExpandoObject>(ReadData, EndOfData);
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>("Destination4CustomSourceDynamic", Connection);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void WithDeltaDestination(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture s2c = new TwoColumnsTableFixture(connection, "DBMergeSource");
s2c.InsertTestData();
s2c.InsertTestDataSet2();
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DBMergeDestination");
d2c.InsertTestDataSet3();
TwoColumnsDeltaTableFixture delta2Columns = new TwoColumnsDeltaTableFixture(connection, "DBMergeDelta");
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>("DBMergeSource", connection);
//Act
DbMerge <MySimpleRow> merge = new DbMerge <MySimpleRow>(MergeMode.Full, "DBMergeDestination", connection);
DbDestination <MySimpleRow> delta = new DbDestination <MySimpleRow>("DBMergeDelta", connection);
source.LinkTo(merge);
merge.LinkTo(delta);
source.Execute();
merge.Wait();
delta.Wait();
//Assert
Assert.True(merge.UseTruncateMethod == true);
Assert.Equal(6, RowCountTask.Count(connection, "DBMergeDestination", $"{d2c.QB}Col1{d2c.QE} BETWEEN 1 AND 7 AND {d2c.QB}Col2{d2c.QE} LIKE 'Test%'"));
Assert.Equal(7, RowCountTask.Count(connection, "DBMergeDelta", $"{d2c.QB}Col1{d2c.QE} BETWEEN 1 AND 10 AND {d2c.QB}Col2{d2c.QE} LIKE 'Test%'"));
Assert.Equal(1, RowCountTask.Count(connection, "DBMergeDelta", $"{d2c.QB}ChangeAction{d2c.QE} = 'D' AND {d2c.QB}Col1{d2c.QE} = 10"));
Assert.Equal(3, RowCountTask.Count(connection, "DBMergeDelta", $"{d2c.QB}ChangeAction{d2c.QE} = 'U' AND {d2c.QB}Col1{d2c.QE} IN (1,2,4)"));
Assert.Equal(3, RowCountTask.Count(connection, "DBMergeDelta", $"{d2c.QB}ChangeAction{d2c.QE} = 'I' AND {d2c.QB}Col1{d2c.QE} IN (3,5,6)"));
}
public void WithBatchChanges(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DbDestinationBatchChanges");
DbDestination <string[]> dest = new DbDestination <string[]>(connection, "DbDestinationBatchChanges", batchSize: 2)
{
BeforeBatchWrite = rowArray =>
{
rowArray[0][1] = "NewValue";
return(rowArray);
}
};
//Act
CsvSource <string[]> source = new CsvSource <string[]>("res/CsvSource/TwoColumns.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(connection, "DbDestinationBatchChanges"));
Assert.Equal(2, RowCountTask.Count(connection, "DbDestinationBatchChanges", $"{d2c.QB}Col2{d2c.QE}='NewValue'"));
Assert.Equal(1, RowCountTask.Count(connection, "DbDestinationBatchChanges", $"{d2c.QB}Col1{d2c.QE} = 2 AND {d2c.QB}Col2{d2c.QE}='Test2'"));
}