public void WithObjectErrorLinking()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CsvSourceErrorLinking");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("CsvSourceErrorLinking", SqlConnection);
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
CsvSource <MySimpleRow> source = new CsvSource <MySimpleRow>("res/CsvSource/TwoColumnsErrorLinking.csv");
source.LinkTo(dest);
source.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
dest2Columns.AssertTestData();
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public void ExceptionInStoreKeyFunction()
{
//Arrange
MemorySource <MyRow> source = new MemorySource <MyRow>();
source.DataAsList = new List <MyRow>()
{
new MyRow {
Id = 1, ClassName = "Class1", DetailValue = 3.5
}
};
//Act
Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
(row, aggValue) => aggValue.AggValue += row.DetailValue,
row => row.ClassName,
(key, agg) => throw new Exception("Test")
);
MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();
source.LinkTo(agg);
agg.LinkTo(dest);
//Assert
Assert.Throws <AggregateException>(() =>
{
source.Execute();
dest.Wait();
});
}
public void BatchSize()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("MemoryDestinationBatchSizeSource");
source2Columns.InsertTestData();
DBSource <MySimpleRow> source = new DBSource <MySimpleRow>(SqlConnection, "MemoryDestinationBatchSizeSource");
MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>()
{
BatchSize = 2
};
//Act
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection(dest.Data,
d => Assert.True(d.Col1 == 1 && d.Col2 == "Test1"),
d => Assert.True(d.Col1 == 2 && d.Col2 == "Test2"),
d => Assert.True(d.Col1 == 3 && d.Col2 == "Test3")
);
}
public void ExceptionInAggregationFunction()
{
//Arrange
MemorySource <MyRow> source = new MemorySource <MyRow>();
source.DataAsList = new List <MyRow>()
{
new MyRow {
Id = 1, DetailValue = 3.5
},
};
//Act
Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
(row, aggRow) => throw new Exception("Test")
);
MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();
//Assert
source.LinkTo(agg);
agg.LinkTo(dest);
Assert.Throws <AggregateException>(() =>
{
source.Execute();
dest.Wait();
});
}
public void UsingRowTransformation()
{
Prepare();
var orderSource = new MemorySource <Order>();
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 815, CustomerName = "John"
});
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 4711, CustomerName = "Jim"
});
var rowTrans = new RowTransformation <Order>(
row => {
int?id = SqlTask.ExecuteScalar <int>(SqlConnection,
sql: $"SELECT Id FROM CustomerTable WHERE Name='{row.CustomerName}'");
row.CustomerId = id;
return(row);
});
/* Delete below here */
var dest = new MemoryDestination <Order>();
orderSource.LinkTo(rowTrans).LinkTo(dest);
orderSource.Execute();
}
public void UsingLookupWithAttributes()
{
Prepare();
var orderSource = new MemorySource <Order>();
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 815, CustomerName = "John"
});
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 4711, CustomerName = "Jim"
});
var lookupSource = new DbSource <CustomerWithAttr>(SqlConnection, "CustomerTable");
var lookup = new LookupTransformation <Order, CustomerWithAttr>();
lookup.Source = lookupSource;
var dest = new MemoryDestination <Order>();
orderSource.LinkTo(lookup).LinkTo(dest);
Network.Execute(orderSource);
foreach (var row in dest.Data)
{
Console.WriteLine($"Order:{row.OrderNumber} Name:{row.CustomerName} Id:{row.CustomerId}");
}
//Output
//Order:815 Name:John Id:1
//Order:4711 Name:Jim Id:2
}
public void AggregateSumWithNullable()
{
//Arrange
List <MyInputRow> sourceData = new List <MyInputRow>()
{
new MyInputRow {
Id = 1, DetailValue = 3.5
},
new MyInputRow {
Id = 2, DetailValue = 4.5
},
new MyInputRow {
Id = 3, DetailValue = 2.0
},
new MyInputRow {
Id = 4, DetailValue = null
},
};
MemoryDestination <MySumRowNullable> dest = CreateFlow <MySumRowNullable>(sourceData);
//Assert
Assert.Collection(dest.Data,
ar => Assert.True(ar.AggValue == 10)
);
}
public void WithPredicate()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("RowDuplicationSource");
source2Columns.InsertTestData();
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(SqlConnection, "RowDuplicationSource");
RowDuplication <MySimpleRow> duplication = new RowDuplication <MySimpleRow>(
row => row.Col1 == 1 || row.Col2 == "Test3"
);
MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();
//Act
source.LinkTo(duplication);
duplication.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection(dest.Data,
d => Assert.True(d.Col1 == 1 && d.Col2 == "Test1"),
d => Assert.True(d.Col1 == 1 && d.Col2 == "Test1"),
d => Assert.True(d.Col1 == 2 && d.Col2 == "Test2"),
d => Assert.True(d.Col1 == 3 && d.Col2 == "Test3"),
d => Assert.True(d.Col1 == 3 && d.Col2 == "Test3")
);
}
public void WithPredicate()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("RowDuplicationSource");
source2Columns.InsertTestData();
DbSource source = new DbSource(SqlConnection, "RowDuplicationSource");
RowDuplication duplication = new RowDuplication(
row =>
{
dynamic r = row as dynamic;
return(r.Col1 == 1 || r.Col2 == "Test3");
});
MemoryDestination dest = new MemoryDestination();
//Act
source.LinkTo(duplication);
duplication.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection(dest.Data,
row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 1 && d.Col2 == "Test1"); },
row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 1 && d.Col2 == "Test1"); },
row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 2 && d.Col2 == "Test2"); },
row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 3 && d.Col2 == "Test3"); },
row => { dynamic d = row as dynamic; Assert.True(d.Col1 == 3 && d.Col2 == "Test3"); }
);
}
public void NoLookupSource()
{
//Arrange
MemorySource <MyDataRow> source = new MemorySource <MyDataRow>();
source.DataAsList.Add(new MyDataRow()
{
Col1 = 1, Col2 = "Test1"
});
//Act
var lookup = new LookupTransformation <MyDataRow, MyLookupRow>();
MemoryDestination <MyDataRow> dest = new MemoryDestination <MyDataRow>();
//Assert
Assert.Throws <ETLBoxException>(() =>
{
try
{
source.LinkTo(lookup);
lookup.LinkTo(dest);
source.Execute();
dest.Wait();
}
catch (AggregateException e) { throw e.InnerException; }
});
//Assert
}
public void ValidateSchemaForArray()
{
//Arrange
var source = new MemorySource <string[]>();
source.DataAsList.Add(new string[] { _validXml });
source.DataAsList.Add(new string[] { _invalidXml });
source.DataAsList.Add(new string[] { _validXml });
MemoryDestination <string[]> dest = new MemoryDestination <string[]>();
MemoryDestination <ETLBoxError> error = new MemoryDestination <ETLBoxError>();
//Act
XmlSchemaValidation <string[]> schemaValidation = new XmlSchemaValidation <string[]>();
schemaValidation.XmlSelector = row => row[0];
schemaValidation.XmlSchema = xsdMarkup;
source.LinkTo(schemaValidation);
schemaValidation.LinkTo(dest);
schemaValidation.LinkErrorTo(error);
source.Execute();
dest.Wait();
error.Wait();
//Assert
Assert.True(dest.Data.Count == 2);
Assert.True(error.Data.Count == 1);
}
public void ThrowExceptionWithoutHandling()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("RowMultiplicationSource");
source2Columns.InsertTestData();
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(SqlConnection, "RowMultiplicationSource");
RowMultiplication <MySimpleRow> multiplication = new RowMultiplication <MySimpleRow>(
row =>
{
List <MySimpleRow> result = new List <MySimpleRow>();
result.Add(row);
if (row.Col1 == 2)
{
throw new Exception("Error in Flow!");
}
return(result);
});
MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();
//Act & Assert
source.LinkTo(multiplication);
multiplication.LinkTo(dest);
Assert.Throws <AggregateException>(() =>
{
source.Execute();
dest.Wait();
});
}
public void ReadingIntoDynamicObject()
{
//Arrange
var dest = new MemoryDestination();
//Act
TextSource source = new TextSource();
source.Uri = "res/TextSource/Test.txt";
source.WriteLineIntoObject = (line, dynob) =>
{
dynamic o = dynob as ExpandoObject;
o.Text = line;
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.True(dest.Data.Count == 3);
Assert.Collection <ExpandoObject>(dest.Data,
row => { dynamic r = row as ExpandoObject; Assert.Equal("Line 1", r.Text); },
row => { dynamic r = row as ExpandoObject; Assert.Equal("Line 2", r.Text); },
row => { dynamic r = row as ExpandoObject; Assert.Equal("Line 3", r.Text); }
);
}
public void CompareFlowWithBulkInsert(int numberOfRows, double deviation)
{
//Arrange
BigDataCsvSource.CreateCSVFileIfNeeded(numberOfRows);
var sourceNonGeneric = new CSVSource(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destNonGeneric = new MemoryDestination();
var sourceGeneric = new CSVSource <CSVData>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destGeneric = new MemoryDestination <CSVData>();
var sourceDynamic = new CSVSource <ExpandoObject>(BigDataCsvSource.GetCompleteFilePath(numberOfRows));
var destDynamic = new MemoryDestination <ExpandoObject>();
//Act
var teNonGeneric = GetETLBoxTime(numberOfRows, sourceNonGeneric, destNonGeneric);
var teGeneric = GetETLBoxTime(numberOfRows, sourceGeneric, destGeneric);
var teDynamic = GetETLBoxTime(numberOfRows, sourceDynamic, destDynamic);
//Assert
Assert.Equal(numberOfRows, destNonGeneric.Data.Count);
Assert.Equal(numberOfRows, destGeneric.Data.Count);
Assert.Equal(numberOfRows, destDynamic.Data.Count);
Assert.True(new [] { teGeneric.TotalMilliseconds, teNonGeneric.TotalMilliseconds, teDynamic.TotalMilliseconds }.Max() <
new [] { teGeneric.TotalMilliseconds, teNonGeneric.TotalMilliseconds, teDynamic.TotalMilliseconds }.Max() *(deviation + 1));
}
public void GroupingUsingStringArray()
{
//Arrange
MemorySource <string[]> source = new MemorySource <string[]>();
source.DataAsList.Add(new string[] { "Class1", "3.5" });
source.DataAsList.Add(new string[] { "Class1", "6.5" });
source.DataAsList.Add(new string[] { "Class2", "10" });
Aggregation <string[], MyAggRow> agg = new Aggregation <string[], MyAggRow>(
(row, aggValue) => aggValue.AggValue += Convert.ToDouble(row[1]),
row => row[0],
(key, agg) => agg.GroupName = (string)key
);
MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();
//Act
source.LinkTo(agg);
agg.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MyAggRow>(dest.Data,
ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class1"),
ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class2")
);
}
public void ErrorInSql()
{
//Arrange
DbSource source = new DbSource(SqlConnection)
{
Sql = "SELECT XYZ FROM ABC"
};
MemoryDestination dest = new MemoryDestination();
source.LinkTo(dest);
//Act & Assert
Assert.Throws <SqlException>(() =>
{
try
{
Task s = source.ExecuteAsync();
Task c = dest.Completion;
Task.WaitAll(c, s);
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
public void DataIsInList()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("RowDuplicationStringArraySource");
source2Columns.InsertTestData();
DbSource <string[]> source = new DbSource <string[]>(SqlConnection, "RowDuplicationStringArraySource");
RowDuplication <string[]> duplication = new RowDuplication <string[]>();
MemoryDestination <string[]> dest = new MemoryDestination <string[]>();
//Act
source.LinkTo(duplication);
duplication.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection(dest.Data,
d => Assert.True(d[0] == "1" && d[1] == "Test1"),
d => Assert.True(d[0] == "1" && d[1] == "Test1"),
d => Assert.True(d[0] == "2" && d[1] == "Test2"),
d => Assert.True(d[0] == "2" && d[1] == "Test2"),
d => Assert.True(d[0] == "3" && d[1] == "Test3"),
d => Assert.True(d[0] == "3" && d[1] == "Test3")
);
}
public void RedirectErrorWithObject(IConnectionManager connection)
{
if (connection.GetType() == typeof(SQLiteConnectionManager))
{
Task.Delay(100).Wait(); //Database was locked and needs to recover after exception
}
//Arrange
CreateSourceTable(connection, "DbSourceErrorLinking");
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture(connection, "DbDestinationErrorLinking");
//Act
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(connection, "DbSourceErrorLinking");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(connection, "DbDestinationErrorLinking");
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
source.LinkTo(dest);
source.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
dest2Columns.AssertTestData();
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
private static void RunExceptionFlowWithType <T>()
{
//Arrange
MemorySource <InputDataRow> source = new MemorySource <InputDataRow>();
source.DataAsList.Add(new InputDataRow()
{
LookupId = 1
});
MemorySource <T> lookupSource = new MemorySource <T>();
var lookup = new LookupTransformation <InputDataRow, T>(lookupSource);
MemoryDestination <InputDataRow> dest = new MemoryDestination <InputDataRow>();
source.LinkTo(lookup);
lookup.LinkTo(dest);
//Act && Assert
Assert.Throws <ETLBoxException>(() =>
{
try
{
source.Execute();
dest.Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
public void AggregateSimple()
{
//Arrange
MemorySource <MyRow> source = new MemorySource <MyRow>();
source.DataAsList = new List <MyRow>()
{
new MyRow {
Id = 1, DetailValue = 3.5
},
new MyRow {
Id = 2, DetailValue = 4.5
},
new MyRow {
Id = 3, DetailValue = 2.0
},
};
Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
(row, aggRow) => aggRow.AggValue += row.DetailValue
);
MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();
//Act
source.LinkTo(agg);
agg.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MyAggRow>(dest.Data,
ar => Assert.True(ar.AggValue == 10)
);
}
public void DynamicObjectJoin()
{
//Arrange
TwoColumnsTableFixture table1 = new TwoColumnsTableFixture(SqlConnection, "CrossJoinSource1");
table1.InsertTestData();
TwoColumnsTableFixture table2 = new TwoColumnsTableFixture(SqlConnection, "CrossJoinSource2");
table2.InsertTestData();
DbSource <ExpandoObject> source1 = new DbSource <ExpandoObject>(SqlConnection, "CrossJoinSource1");
DbSource <ExpandoObject> source2 = new DbSource <ExpandoObject>(SqlConnection, "CrossJoinSource2");
MemoryDestination dest = new MemoryDestination();
CrossJoin crossJoin = new CrossJoin(
(data1, data2) =>
{
dynamic d1 = data1 as dynamic;
dynamic d2 = data1 as dynamic;
dynamic res = new ExpandoObject();
res.Val = d1.Col1 + d2.Col2;
return(res);
}
);
//Act
source1.LinkTo(crossJoin.InMemoryTarget);
source2.LinkTo(crossJoin.PassingTarget);
crossJoin.LinkTo(dest);
source1.Execute();
source2.Execute();
dest.Wait();
//Assert
Assert.Equal(9, dest.Data.Count);
}
public void ValidateSchemaForDynamicObject()
{
//Arrange
var source = new MemorySource();
dynamic n1 = new ExpandoObject(); n1.Xml = _validXml; source.DataAsList.Add(n1);
dynamic n2 = new ExpandoObject(); n2.Xml = _validXml; source.DataAsList.Add(n2);
dynamic n3 = new ExpandoObject(); n3.Xml = _invalidXml; source.DataAsList.Add(n3);
MemoryDestination dest = new MemoryDestination();
MemoryDestination <ETLBoxError> error = new MemoryDestination <ETLBoxError>();
//Act
XmlSchemaValidation schemaValidation = new XmlSchemaValidation();
schemaValidation.XmlSelector = row =>
{
dynamic r = row as ExpandoObject;
return(r.Xml);
};
schemaValidation.XmlSchema = xsdMarkup;
source.LinkTo(schemaValidation);
schemaValidation.LinkTo(dest);
schemaValidation.LinkErrorTo(error);
source.Execute();
dest.Wait();
error.Wait();
//Assert
Assert.True(dest.Data.Count == 2);
Assert.True(error.Data.Count == 1);
}
public void GroupingAndKeepingKey()
{
//Arrange
MemorySource <MyRow> source = new MemorySource <MyRow>();
source.Data = new List <MyRow>()
{
new MyRow {
Id = 1, ClassName = "Class1", DetailValue = 3.5
},
new MyRow {
Id = 2, ClassName = "Class1", DetailValue = 6.5
},
new MyRow {
Id = 3, ClassName = "Class2", DetailValue = 1.2
},
new MyRow {
Id = 4, ClassName = "Class2", DetailValue = 2.3
},
new MyRow {
Id = 5, ClassName = "Class2", DetailValue = 16.5
},
new MyRow {
Id = 6, ClassName = "Class3", DetailValue = 30.0
},
new MyRow {
Id = 6, ClassName = null, DetailValue = 14.5
},
new MyRow {
Id = 6, ClassName = null, DetailValue = 15.5
},
};
Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
(row, aggValue) => aggValue.AggValue += row.DetailValue,
row => row.ClassName,
(key, agg) => agg.GroupName = (string)key
);
MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();
//Act
source.LinkTo(agg);
agg.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MyAggRow>(dest.Data,
ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class1"),
ar => Assert.True(ar.AggValue == 20 && ar.GroupName == "Class2"),
ar => Assert.True(ar.AggValue == 30 && ar.GroupName == "Class3"),
ar => Assert.True(ar.AggValue == 30 && ar.GroupName == string.Empty)
);
}
protected override void InitBufferObjects()
{
InMemoryTarget = new MemoryDestination <TInput1>(this);
PassingTarget = new CustomDestination <TInput2>(this, CrossJoinData);
if (MaxBufferSize > 0)
{
PassingTarget.MaxBufferSize = this.MaxBufferSize;
}
PassingTarget.OnCompletion = () => Buffer.Complete();
}
public void WaitForLongRunningProcedure()
{
//Arrange
CreateProcedureTask.CreateOrAlter(SqlConnection, new ProcedureDefinition()
{
Name = "longrunning",
Definition = @"
WAITFOR DELAY '00:00:01';
SELECT @input + 10;",
Parameter = new List <ProcedureParameter>()
{
new ProcedureParameter()
{
Name = "input", DataType = "INT"
}
// new ProcedureParameter() { Name ="result", Out = true, DataType="INT"}
}
});
var source = new MemorySource <MySimpleRow>();
source.DataAsList.Add(new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
});
source.DataAsList.Add(new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
});
source.DataAsList.Add(new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
});
var dest = new MemoryDestination <MySimpleRow>();
var trans = new RowTransformation <MySimpleRow>(
row =>
{
row.Col1 = SqlTask.ExecuteScalar <int>(SqlConnection, "Read from procedure",
$"EXEC longrunning @input = {row.Col1}") ?? 0;
return(row);
}
);
//Act
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MySimpleRow>(dest.Data,
row => Assert.Equal(11, row.Col1),
row => Assert.Equal(12, row.Col1),
row => Assert.Equal(13, row.Col1)
);
}
public void OneMatchOneRetrieveColumn()
{
//Arrange
MemorySource <InputDataRow> source = new MemorySource <InputDataRow>();
source.DataAsList.Add(new InputDataRow()
{
LookupId = 1
});
source.DataAsList.Add(new InputDataRow()
{
LookupId = 2
});
source.DataAsList.Add(new InputDataRow()
{
LookupId = 4
});
source.DataAsList.Add(new InputDataRow()
{
LookupId = 3
});
MemorySource <LookupData> lookupSource = new MemorySource <LookupData>();
lookupSource.DataAsList.Add(new LookupData()
{
Id = 1, Value = "Test1"
});
lookupSource.DataAsList.Add(new LookupData()
{
Id = 2, Value = "Test2"
});
lookupSource.DataAsList.Add(new LookupData()
{
Id = 3, Value = "Test3"
});
var lookup = new LookupTransformation <InputDataRow, LookupData>();
lookup.Source = lookupSource;
MemoryDestination <InputDataRow> dest = new MemoryDestination <InputDataRow>();
source.LinkTo(lookup);
lookup.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <InputDataRow>(dest.Data,
r => Assert.True(r.LookupId == 1 && r.LookupValue == "Test1"),
r => Assert.True(r.LookupId == 2 && r.LookupValue == "Test2"),
r => Assert.True(r.LookupId == 4 && r.LookupValue == null),
r => Assert.True(r.LookupId == 3 && r.LookupValue == "Test3")
);
}
public void JsonFromWebService()
{
JsonSource source = new JsonSource("res/JsonSource/EmptyObject.json", ResourceType.File);
MemoryDestination dest = new MemoryDestination();
source.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.True(dest.Data.Count == 0);
}
public void RedirectBatch(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DestLinkError");
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.Data = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = null, Col2 = "ErrorRecord"
},
new MySimpleRow()
{
Col1 = "X2", Col2 = "ErrorRecord"
},
new MySimpleRow()
{
Col1 = "1", Col2 = "Test1"
},
new MySimpleRow()
{
Col1 = "2", Col2 = "Test2"
},
new MySimpleRow()
{
Col1 = "3", Col2 = "Test3 - good, but in error batch"
},
new MySimpleRow()
{
Col1 = null, Col2 = "ErrorRecord"
},
new MySimpleRow()
{
Col1 = "3", Col2 = "Test3"
},
};
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(connection, "DestLinkError", batchSize: 2);
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
source.LinkTo(dest);
dest.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
d2c.AssertTestData();
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public void MultipleMatchAndRetrieveColumns()
{
//Arrange
MemorySource <InputDataMultiple> source = new MemorySource <InputDataMultiple>();
source.DataAsList.Add(new InputDataMultiple()
{
LookupId1 = 1, LookupId2 = "T1"
});
source.DataAsList.Add(new InputDataMultiple()
{
LookupId1 = 2, LookupId2 = "TX"
});
source.DataAsList.Add(new InputDataMultiple()
{
LookupId1 = 4, LookupId2 = "T2"
});
source.DataAsList.Add(new InputDataMultiple()
{
LookupId1 = 3, LookupId2 = "T3"
});
MemorySource <LookupDataMultiple> lookupSource = new MemorySource <LookupDataMultiple>();
lookupSource.DataAsList.Add(new LookupDataMultiple()
{
Id1 = 1, Id2 = "T1", Value1 = "Test1", Value2 = 100
});
lookupSource.DataAsList.Add(new LookupDataMultiple()
{
Id1 = 2, Value1 = "Test2", Value2 = 200
});
lookupSource.DataAsList.Add(new LookupDataMultiple()
{
Id1 = 3, Id2 = "T3", Value2 = 300
});
var lookup = new LookupTransformation <InputDataMultiple, LookupDataMultiple>();
lookup.Source = lookupSource;
MemoryDestination <InputDataMultiple> dest = new MemoryDestination <InputDataMultiple>();
source.LinkTo(lookup);
lookup.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <InputDataMultiple>(dest.Data,
r => Assert.True(r.LookupId1 == 1 && r.LookupValue1 == "Test1" && r.LookupValue2 == 100),
r => Assert.True(r.LookupId1 == 2 && r.LookupValue1 == null),
r => Assert.True(r.LookupId1 == 4 && r.LookupValue1 == null),
r => Assert.True(r.LookupId1 == 3 && r.LookupValue1 == null && r.LookupValue2 == 300)
);
}
public void ReadEmptyArray()
{
JsonSource source = new JsonSource("res/JsonSource/EmptyArray.json", ResourceType.File);
MemoryDestination dest = new MemoryDestination();
source.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.True(dest.Data.Count == 0);
}
