private void TransferTestDataIntoDestination(List <MyMergeRow> knownGuids)
{
MemorySource <MyMergeRow> source = new MemorySource <MyMergeRow>();
source.Data = knownGuids;
DbDestination <MyMergeRow> dest = new DbDestination <MyMergeRow>("MergeDestination", SqlConnection);
source.LinkTo(dest);
source.Execute();
dest.Wait();
}
private static void StoreLastSyncKey(SyncData syncData)
{
var memsource = new MemorySource <SyncData>();
memsource.DisableLogging = true;
memsource.DataAsList.Add(syncData);
var syncdest = new JsonDestination <SyncData>("LastSyncId.json");
syncdest.DisableLogging = true;
memsource.LinkTo(syncdest);
Network.Execute(memsource);
}
public void WithGrouping()
{
//Arrange
MemorySource <MyRow> source = new MemorySource <MyRow>();
source.DataAsList = new List <MyRow>()
{
new MyRow {
Id = 1, ClassName = "Class1", DetailValue = 3.5
},
new MyRow {
Id = 2, ClassName = "Class1", DetailValue = 6.5
},
new MyRow {
Id = 3, ClassName = "Class2", DetailValue = 1.2
},
new MyRow {
Id = 4, ClassName = "Class2", DetailValue = 2.3
},
new MyRow {
Id = 5, ClassName = "Class2", DetailValue = 16.5
},
new MyRow {
Id = 6, ClassName = "Class3", DetailValue = 30.0
},
new MyRow {
Id = 6, ClassName = null, DetailValue = 14.5
},
new MyRow {
Id = 6, ClassName = null, DetailValue = 15.5
},
};
Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>();
MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();
//Act
source.LinkTo(agg);
agg.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MyAggRow>(dest.Data,
ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class1"),
ar => Assert.True(ar.AggValue == 20 && ar.GroupName == "Class2"),
ar => Assert.True(ar.AggValue == 30 && ar.GroupName == "Class3"),
ar => Assert.True(ar.AggValue == 30 && ar.GroupName == null)
);
}
public void WithNullable()
{
//Arrange
MemorySource <MyRowNullable> source = new MemorySource <MyRowNullable>();
source.DataAsList = new List <MyRowNullable>()
{
new MyRowNullable {
Id = 1, ClassId = 1, DetailValue = 3.5
},
new MyRowNullable {
Id = 2, ClassId = 1, DetailValue = 6.5
},
new MyRowNullable {
Id = 3, ClassId = 2, DetailValue = 1.2
},
new MyRowNullable {
Id = 4, ClassId = 2, DetailValue = 2.3
},
new MyRowNullable {
Id = 5, ClassId = 2, DetailValue = 16.5
},
new MyRowNullable {
Id = 6, ClassId = 3, DetailValue = 30.0
},
new MyRowNullable {
Id = 6, ClassId = null, DetailValue = 14.5
},
new MyRowNullable {
Id = 6, ClassId = null, DetailValue = 15.5
},
};
Aggregation <MyRowNullable, MyAggRowNullable> agg = new Aggregation <MyRowNullable, MyAggRowNullable>();
MemoryDestination <MyAggRowNullable> dest = new MemoryDestination <MyAggRowNullable>();
//Act
source.LinkTo(agg);
agg.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MyAggRowNullable>(dest.Data,
ar => Assert.True(ar.AggValue == 10 && ar.GroupId == 1),
ar => Assert.True(ar.AggValue == 20 && ar.GroupId == 2),
ar => Assert.True(ar.AggValue == 30 && ar.GroupId == 3),
ar => Assert.True(ar.AggValue == 30 && ar.GroupId == null)
);
}
public ReferenceDataFlow(int key, string value)
{
var x = new MyData()
{
Key = key, Value = value
};
_source = new MemorySource <MyData>();
_source.DataAsList.Add(x);
_destination = new CustomDestination <MyData>(x => Data.Add(x.Key, x));
_source.LinkTo(_destination);
_source.ExecuteAsync();
}
public void UsingBatchTransformation()
{
var orderSource = new MemorySource <Order>();
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 815, CustomerName = "John"
});
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 4711, CustomerName = "Jim"
});
var batchTrans = new BatchTransformation <Order>()
{
BatchSize = 100,
BatchTransformationFunc =
batch =>
{
var names = string.Join(",", batch.Select(cust => $"'{cust.CustomerName}'"));
string curName = ""; int curId = 0;
Dictionary <string, int> idByName = new Dictionary <string, int>();
var sql = new SqlTask()
{
ConnectionManager = SqlConnection,
Sql = $"SELECT DISTINCT Name, Id FROM CustomerTable WHERE Name IN ({names})",
AfterRowReadAction = () => { idByName.Add(curName, curId); },
Actions = new List <Action <object> >()
{
name => curName = (string)name,
id => curId = (int)id
}
};
sql.ExecuteReader();
foreach (var row in batch)
{
row.CustomerId = idByName[row.CustomerName];
}
return(batch);
}
};
var dest = new MemoryDestination <Order>();
orderSource.LinkTo(batchTrans).LinkTo(dest);
Network.Execute(orderSource);
foreach (var result in dest.Data)
{
Console.WriteLine($"Customer {result.CustomerName} has id {result.CustomerId}");
}
}
public void TestBoundedCapacityIsWorking()
{
var connection = SqlConnection;
connection.FireTriggers = true;
SqlTask.ExecuteNonQuery(connection, "Create test table",
"CREATE TABLE test ( id INT NOT NULL );");
SqlTask.ExecuteNonQuery(connection, "Add wait trigger",
$@"CREATE TRIGGER testtrigger ON test
AFTER INSERT
AS
WAITFOR DELAY '00:00:0.500';");
var source = new MemorySource <string[]>()
{
MaxBufferSize = 2
};
for (int i = 0; i < 8; i++)
{
source.DataAsList.Add(new string[] { i.ToString() });
}
var dest = new DbDestination <string[]>(connection, "test", batchSize: 1)
{
MaxBufferSize = 1
};
source.LinkTo(dest);
var s = source.ExecuteAsync();
var d = dest.Completion;
int count = 1;
while (!d.IsCompleted)
{
Task.Delay(500).Wait();
if (count > 1 && count < 7)
{
Assert.True(source.ProgressCount > dest.ProgressCount);
}
if (count == 1)
{
Assert.True(source.ProgressCount <= 6);
}
count++;
}
Assert.Equal(8, dest.ProgressCount);
}
public void IgnoreNullValues()
{
//Arrange
MemorySource <string> source1 = new MemorySource <string>();
source1.DataAsList = new List <string>()
{
"A", null, "B", "C"
};
MemorySource <int?> source2 = new MemorySource <int?>();
source2.DataAsList = new List <int?>()
{
1, null, 2, null, 3
};
CrossJoin <string, int?, string> crossJoin = new CrossJoin <string, int?, string>(
(data1, data2) =>
{
if (data1 == "C")
{
return(null);
}
else
{
return(data1 + data2?.ToString());
}
}
);
MemoryDestination <string> dest = new MemoryDestination <string>();
//Act
source1.LinkTo(crossJoin.InMemoryTarget);
source2.LinkTo(crossJoin.PassingTarget);
crossJoin.LinkTo(dest);
source1.Execute();
source2.Execute();
dest.Wait();
//Assert
Assert.Equal(6, dest.Data.Count);
Assert.Collection <string>(dest.Data,
s => Assert.Equal("A1", s),
s => Assert.Equal("B1", s),
s => Assert.Equal("A2", s),
s => Assert.Equal("B2", s),
s => Assert.Equal("A3", s),
s => Assert.Equal("B3", s)
);
}
public void IgnoreWithObject()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
null,
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
null,
new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
},
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
},
null
};
//Act
RowMultiplication <MySimpleRow> multiplication = new RowMultiplication <MySimpleRow>(
row => new List <MySimpleRow>()
{
row, row
}
);
MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();
source.LinkTo(multiplication);
multiplication.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection(dest.Data,
d => Assert.True(d.Col1 == 1 && d.Col2 == "Test1"),
d => Assert.True(d.Col1 == 1 && d.Col2 == "Test1"),
d => Assert.True(d.Col1 == 2 && d.Col2 == "Test2"),
d => Assert.True(d.Col1 == 2 && d.Col2 == "Test2"),
d => Assert.True(d.Col1 == 3 && d.Col2 == "Test3"),
d => Assert.True(d.Col1 == 3 && d.Col2 == "Test3")
);
}
public void ClassificationAndKeepingKey()
{
//Arrange
MemorySource <MyRow> source = new MemorySource <MyRow>();
source.Data = new List <MyRow>()
{
new MyRow {
Id = 1, ClassName = "Class1", DetailValue = 3.5
},
new MyRow {
Id = 2, ClassName = "Class1", DetailValue = 6.5
},
new MyRow {
Id = 3, ClassName = "Class2", DetailValue = 1.2
},
new MyRow {
Id = 4, ClassName = "Class2", DetailValue = 2.3
},
new MyRow {
Id = 5, ClassName = "Class2", DetailValue = 16.5
},
new MyRow {
Id = 6, ClassName = "Class3", DetailValue = 30.0
},
};
Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
(row, aggValue) => aggValue.AggValue += row.DetailValue,
row => row.ClassName,
(key, agg) => agg.ClassName = (string)key
);
MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();
//Act
source.LinkTo(agg);
agg.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MyAggRow>(dest.Data,
ar => Assert.True(ar.AggValue == 10 && ar.ClassName == "Class1"),
ar => Assert.True(ar.AggValue == 20 && ar.ClassName == "Class2"),
ar => Assert.True(ar.AggValue == 30 && ar.ClassName == "Class3")
);
}
public void AttributesWithDynamic()
{
Prepare();
var orderSource = new MemorySource();
dynamic sourceRow1 = new ExpandoObject();
sourceRow1.OrderNumber = 815;
sourceRow1.CustomerName = "John";
orderSource.DataAsList.Add(sourceRow1);
dynamic sourceRow2 = new ExpandoObject();
sourceRow2.OrderNumber = 4711;
sourceRow2.CustomerName = "Jim";
orderSource.DataAsList.Add(sourceRow2);
var lookupSource = new DbSource(SqlConnection, "CustomerTable");
var lookup = new LookupTransformation();
lookup.MatchColumns = new[] {
new MatchColumn()
{
LookupSourcePropertyName = "Name"
, InputPropertyName = "CustomerName"
}
};
lookup.RetrieveColumns = new[] {
new RetrieveColumn()
{
LookupSourcePropertyName = "Id",
InputPropertyName = "CustomerId"
}
};
lookup.Source = lookupSource;
var dest = new MemoryDestination();
orderSource.LinkTo(lookup).LinkTo(dest);
Network.Execute(orderSource);
foreach (dynamic row in dest.Data)
{
Console.WriteLine($"Order:{row.OrderNumber} Name:{row.CustomerName} Id:{row.CustomerId}");
}
//Output
//Order:815 Name:John Id:1
//Order:4711 Name:Jim Id:2
}
public void DataIsFromList()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("MemoryDestination");
MemorySource <ExpandoObject> source = new MemorySource <ExpandoObject>();
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>("MemoryDestination", SqlConnection);
AddObjectsToSource(source);
//Act
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
private MemoryDestination <T> CreateFlow <T>(List <MyInputRow> sourceData)
{
MemorySource <MyInputRow> source = new MemorySource <MyInputRow>();
source.DataAsList = sourceData;
Aggregation <MyInputRow, T> agg = new Aggregation <MyInputRow, T>();
MemoryDestination <T> dest = new MemoryDestination <T>();
//Act
source.LinkTo(agg);
agg.LinkTo(dest);
source.Execute();
dest.Wait();
return(dest);
}
public void RedirectSingleRecordWithObject()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = "X"
},
new MySimpleRow()
{
Col1 = "1"
},
new MySimpleRow()
{
Col1 = "2"
},
new MySimpleRow()
{
Col1 = null
},
new MySimpleRow()
{
Col1 = "3"
},
};
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("ErrorFile.csv");
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
source.LinkTo(dest);
dest.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./ErrorFile.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsErrorLinking.csv"));
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public void NoErrorLinking()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture(SqlConnection, "LookupErrorLinkingDest");
CreateSourceTable(SqlConnection, "LookupErrorLinkingSource");
DbSource <MyLookupRow> lookupSource = new DbSource <MyLookupRow>(SqlConnection, "LookupErrorLinkingSource");
MemorySource <MyInputDataRow> source = new MemorySource <MyInputDataRow>();
source.DataAsList = new List <MyInputDataRow>()
{
new MyInputDataRow()
{
Col1 = 1
},
new MyInputDataRow()
{
Col1 = 2
},
new MyInputDataRow()
{
Col1 = 3
}
};
//Act & Assert
Assert.ThrowsAny <Exception>(() =>
{
List <MyLookupRow> LookupTableData = new List <MyLookupRow>();
LookupTransformation <MyInputDataRow, MyLookupRow> lookup = new LookupTransformation <MyInputDataRow, MyLookupRow>(
lookupSource,
row =>
{
row.Col2 = LookupTableData.Where(ld => ld.Key == row.Col1).Select(ld => ld.LookupValue).FirstOrDefault();
return(row);
}
, LookupTableData
);
DbDestination <MyInputDataRow> dest = new DbDestination <MyInputDataRow>(SqlConnection, "LookupErrorLinkingDest");
source.LinkTo(lookup);
lookup.LinkTo(dest);
source.Execute();
dest.Wait();
});
}
public void TestErrorLink()
{
//Arrange
MemorySource <string> source1 = new MemorySource <string>();
source1.DataAsList = new List <string>()
{
"A", "B"
};
MemorySource <int> source2 = new MemorySource <int>();
source2.DataAsList = new List <int>()
{
1, 2, 3
};
CrossJoin <string, int, string> crossJoin = new CrossJoin <string, int, string>(
(data1, data2) =>
{
if (data1 == "A")
{
throw new Exception("Invalid record");
}
return(data1 + data2.ToString());
}
);
MemoryDestination <string> dest = new MemoryDestination <string>();
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
source1.LinkTo(crossJoin.InMemoryTarget);
source2.LinkTo(crossJoin.PassingTarget);
crossJoin.LinkTo(dest);
crossJoin.LinkErrorTo(errorDest);
source1.Execute();
source2.Execute();
dest.Wait();
errorDest.Wait();
//Assert
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public void CheckMemoryUsageDbDestination(IConnectionManager connection, int numberOfRows, int batchSize, double deviation)
{
//Arrange
ReCreateDestinationTable(connection, "MemoryDestination");
var source = new MemorySource <CSVData>();
source.Data = GenerateWithYield(numberOfRows);
var dest = new DbDestination <CSVData>(connection, "MemoryDestination", batchSize);
//Act
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(numberOfRows, RowCountTask.Count(connection, "MemoryDestination"));
}
public void DeltaLoadWithDeletion(IConnectionManager connection)
{
//Arrange
MemorySource <MyMergeRow> source = new MemorySource <MyMergeRow>();
source.DataAsList.Add(new MyMergeRow()
{
Key = 2, Value = "Test2"
});
source.DataAsList.Add(new MyMergeRow()
{
Key = 3, Value = "Test3"
});
source.DataAsList.Add(new MyMergeRow()
{
Key = 4, DeleteThisRow = true
});
source.DataAsList.Add(new MyMergeRow()
{
Key = 10, DeleteThisRow = true
});
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DBMergeDeltaDestination");
d2c.InsertTestDataSet3();
//Act
DbMerge <MyMergeRow> dest = new DbMerge <MyMergeRow>(connection, "DBMergeDeltaDestination")
{
DeltaMode = MergeMode.Delta
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.True(dest.UseTruncateMethod == false);
d2c.AssertTestData();
Assert.True(dest.DeltaTable.Count == 4);
Assert.True(dest.DeltaTable.Where(row => row.ChangeAction == ChangeAction.Update && row.Key == 2).Count() == 1);
Assert.True(dest.DeltaTable.Where(row => row.ChangeAction == ChangeAction.Insert && row.Key == 3).Count() == 1);
Assert.True(dest.DeltaTable.Where(row => row.ChangeAction == ChangeAction.Delete && row.Key == 4).Count() == 1);
Assert.True(dest.DeltaTable.Where(row => row.ChangeAction == ChangeAction.Delete && row.Key == 10).Count() == 1);
}
public void UsingLookupWithRetrievalByKeyFunc()
{
Prepare();
var orderSource = new MemorySource <Order>();
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 815, CustomerName = "John"
});
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 4711, CustomerName = "Jim"
});
var lookupSource = new DbSource <Customer>(SqlConnection, "CustomerTable");
var lookup = new LookupTransformation <Order, Customer>();
lookup.Source = lookupSource;
lookup.GetInputRecordKeyFunc = inputrow => inputrow.CustomerName;
lookup.GetSourceRecordKeyFunc = sourcerow => sourcerow.Name;
lookup.RetrievalByKeyFunc = (inputrow, dict) => {
if (dict.ContainsKey(inputrow.CustomerName))
{
inputrow.CustomerId = dict[inputrow.CustomerName].Id;
}
return(inputrow);
};
var dest = new MemoryDestination <Order>();
orderSource.LinkTo(lookup).LinkTo(dest);
Network.Execute(orderSource);
foreach (var row in dest.Data)
{
Console.WriteLine($"Order:{row.OrderNumber} Name:{row.CustomerName} Id:{row.CustomerId}");
}
//Output
//Order:815 Name:John Id:1
//Order:4711 Name:Jim Id:2
}
public void TestErrorLink()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
new MySimpleRow()
{
Col1 = 2, Col2 = "ErrorRecord"
},
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
},
};
CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>(
row =>
{
if (row.Col1 == 2)
{
throw new Exception("Error record!");
}
}
);
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
source.LinkTo(dest);
dest.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public void WithMultipleGroupingAndAggregation()
{
//Arrange
MemorySource <MyRowMultiple> source = new MemorySource <MyRowMultiple>();
source.DataAsList = new List <MyRowMultiple>()
{
new MyRowMultiple {
Id = 1, Class1Name = "Class", Class2Name = "1", DetailValue1 = 4
},
new MyRowMultiple {
Id = 2, Class1Name = "Class", Class2Name = "1", DetailValue1 = 6
},
new MyRowMultiple {
Id = 3, Class1Name = "Class2", Class2Name = null, DetailValue1 = 3
},
new MyRowMultiple {
Id = 4, Class1Name = "Class2", Class2Name = null, DetailValue1 = 7
},
new MyRowMultiple {
Id = 5, Class1Name = "Class", Class2Name = "3", DetailValue1 = 10
},
};
Aggregation <MyRowMultiple, MyAggRowMultiple> agg = new Aggregation <MyRowMultiple, MyAggRowMultiple>();
MemoryDestination <MyAggRowMultiple> dest = new MemoryDestination <MyAggRowMultiple>();
//Act
source.LinkTo(agg);
agg.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MyAggRowMultiple>(dest.Data,
ar => Assert.True(ar.AggValue1 == 10 && ar.AggValue2 == 2 && ar.Group1Name == "Class" && ar.Group2Name == "1"),
ar => Assert.True(ar.AggValue1 == 10 && ar.AggValue2 == 2 && ar.Group1Name == "Class2" && ar.Group2Name == null),
ar => Assert.True(ar.AggValue1 == 10 && ar.AggValue2 == 1 && ar.Group1Name == "Class" && ar.Group2Name == "3")
);
}
public void UsingLookup()
{
Prepare();
var orderSource = new MemorySource <Order>();
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 815, CustomerName = "John"
});
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 4711, CustomerName = "Jim"
});
var lookupSource = new DbSource <Customer>(SqlConnection, "CustomerTable");
var lookup = new LookupTransformation <Order, Customer>();
lookup.Source = lookupSource;
lookup.RetrievalFunc =
(row, cache) => {
row.CustomerId = cache.Where(cust => cust.Name == row.CustomerName)
.Select(cust => cust.Id)
.FirstOrDefault();
return(row);
};
var dest = new MemoryDestination <Order>();
orderSource.LinkTo(lookup).LinkTo(dest);
Network.Execute(orderSource);
foreach (var row in dest.Data)
{
Console.WriteLine($"Order:{row.OrderNumber} Name:{row.CustomerName} Id:{row.CustomerId}");
}
//Output
//Order:815 Name:John Id:1
//Order:4711 Name:Jim Id:2
}
public void AggregateWithNull()
{
//Arrange
MemorySource <MyRowNullable> source = new MemorySource <MyRowNullable>();
source.DataAsList = new List <MyRowNullable>()
{
new MyRowNullable {
Id = 1, DetailValue = 3.5
},
new MyRowNullable {
Id = 0, DetailValue = null
},
new MyRowNullable {
Id = 2, DetailValue = 4.5
},
new MyRowNullable {
Id = 3, DetailValue = 2.0
},
new MyRowNullable {
Id = 4, DetailValue = null
},
};
Aggregation <MyRowNullable, MyAggRowNullable> agg = new Aggregation <MyRowNullable, MyAggRowNullable>(
(row, aggRow) => aggRow.AggValue += row.DetailValue ?? 0
);
MemoryDestination <MyAggRowNullable> dest = new MemoryDestination <MyAggRowNullable>();
//Act
source.LinkTo(agg);
agg.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MyAggRowNullable>(dest.Data,
ar => Assert.True(ar.AggValue == 10)
);
}
public void IgnoreWithObject()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.Data = new List <MySimpleRow>()
{
null,
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
null,
new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
},
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
},
null
};
//Act
List <MySimpleRow> result = new List <MySimpleRow>();
CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>(
row => result.Add(row)
);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection(result,
d => Assert.True(d.Col1 == 1 && d.Col2 == "Test1"),
d => Assert.True(d.Col1 == 2 && d.Col2 == "Test2"),
d => Assert.True(d.Col1 == 3 && d.Col2 == "Test3")
);
}
public void DataIsFromList()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("MemoryDestinationNonGeneric");
MemorySource <string[]> source = new MemorySource <string[]>();
DbDestination <string[]> dest = new DbDestination <string[]>(SqlConnection, "MemoryDestinationNonGeneric");
//Act
source.Data = new List <string[]>()
{
new string[] { "1", "Test1" },
new string[] { "2", "Test2" },
new string[] { "3", "Test3" },
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void WriteWithArray()
{
//Arrange
var source = new MemorySource <string[]>();
source.DataAsList.Add(new string[] { "Line 1" });
source.DataAsList.Add(new string[] { "Line 2" });
source.DataAsList.Add(new string[] { "Line 3" });
//Act
TextDestination <string[]> dest = new TextDestination <string[]>("res/TextDestination/TestFileArray.txt"
, tr => tr[0]);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.True(File.Exists("res/TextDestination/TestFileArray.txt"));
Assert.Equal(File.ReadAllText("res/TextDestination/ToBeTestFile.txt"), File.ReadAllText("res/TextDestination/TestFileArray.txt"));
}
public void NullValuesFirst(IConnectionManager connection)
{
CreateTestTable(connection, "datatypedestinationdynamicmixed");
//Arrange
MemorySource source = new MemorySource();
dynamic d1 = new ExpandoObject();
d1.DateTimeCol = new DateTime(2010, 1, 1, 10, 10, 10);
d1.IntCol = 2;
d1.LongCol = -1;
d1.DoubleCol = 5.4;
d1.DateCol = new DateTime(2020, 1, 1);
source.DataAsList.Add(d1);
dynamic d2 = new ExpandoObject();
d2.IntCol = 1;
d2.LongCol = -1;
d2.DecimalCol = 2.3M;
d2.DoubleCol = 5.4;
d2.DateTimeCol = new DateTime(2010, 1, 1, 10, 10, 10);
d2.DateCol = new DateTime(2020, 1, 1);
d2.StringCol = "Test";
d2.CharCol = 'T';
d2.DecimalStringCol = "13.4566";
d2.NullCol = null;
d2.EnumCol = EnumType.Value2;
source.DataAsList.Add(d2);
//Act
DbDestination dest = new DbDestination(connection, "datatypedestinationdynamicmixed");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
AssertFirstRow(connection, "datatypedestinationdynamicmixed");
}
public void PartialDbCacheWithSql()
{
Prepare();
var orderSource = new MemorySource <Order>();
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 815, CustomerName = "John"
});
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 4711, CustomerName = "Jim"
});
var lookupSource = new DbSource <CustomerWithAttr>(SqlConnection, "CustomerTable");
var lookup = new LookupTransformation <Order, CustomerWithAttr>();
lookup.Source = lookupSource;
lookup.CacheMode = CacheMode.Partial;
lookup.PartialCacheSettings.LoadBatchSize = 1;
lookup.PartialCacheSettings.LoadCacheSql = batch =>
$@"SELECT Id, Name
FROM CustomerTable
WHERE Name in ({string.Join(",", batch.Select(r => $"'{r.CustomerName}'"))})";
var dest = new MemoryDestination <Order>();
orderSource.LinkTo(lookup).LinkTo(dest);
Network.Execute(orderSource);
foreach (var row in dest.Data)
{
Console.WriteLine($"Order:{row.OrderNumber} Name:{row.CustomerName} Id:{row.CustomerId}");
}
//Output
//Order:815 Name:John Id:1
//Order:4711 Name:Jim Id:2
}
public void CrossJoinStringWithInt()
{
//Arrange
MemorySource <string> source1 = new MemorySource <string>();
source1.DataAsList = new List <string>()
{
"A", "B"
};
MemorySource <int> source2 = new MemorySource <int>();
source2.DataAsList = new List <int>()
{
1, 2, 3
};
CrossJoin <string, int, string> crossJoin = new CrossJoin <string, int, string>(
(data1, data2) => data1 + data2.ToString()
);
MemoryDestination <string> dest = new MemoryDestination <string>();
//Act
source1.LinkTo(crossJoin.InMemoryTarget);
source2.LinkTo(crossJoin.PassingTarget);
crossJoin.LinkTo(dest);
source1.Execute();
source2.Execute();
dest.Wait();
//Assert
Assert.Equal(6, dest.Data.Count);
Assert.Collection <string>(dest.Data,
s => Assert.Equal("A1", s),
s => Assert.Equal("B1", s),
s => Assert.Equal("A2", s),
s => Assert.Equal("B2", s),
s => Assert.Equal("A3", s),
s => Assert.Equal("B3", s)
);
}
public void TwoMemSourcesIntoDB(IConnectionManager connection)
{
//Arrange
MemorySource <MySimpleRow> source1 = new MemorySource <MySimpleRow>();
MemorySource <MySimpleRow> source2 = new MemorySource <MySimpleRow>();
//Act
source1.DataAsList = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
},
};
source2.DataAsList = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
}
};
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture(connection, "DBMultipleDestination");
//Act
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(connection, "DBMultipleDestination");
source1.LinkTo(dest);
source2.LinkTo(dest);
source2.Execute();
source1.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
