public void ExceptionInAggregationFunction()
{
//Arrange
MemorySource <MyRow> source = new MemorySource <MyRow>();
source.DataAsList = new List <MyRow>()
{
new MyRow {
Id = 1, DetailValue = 3.5
},
};
//Act
Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
(row, aggRow) => throw new Exception("Test")
);
MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();
//Assert
source.LinkTo(agg);
agg.LinkTo(dest);
Assert.Throws <AggregateException>(() =>
{
source.Execute();
dest.Wait();
});
}
public void DbDestination()
{
//Arrange
string[] data = { "1", "2" };
MemorySource <string[]> source = new MemorySource <string[]>();
source.DataAsList.Add(data);
DbDestination <string[]> dest = new DbDestination <string[]>("test");
source.LinkTo(dest);
//Act & Assert
Assert.Throws <ETLBoxException>(() =>
{
try
{
source.Execute();
dest.Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
public void IgnoreWithStringArray()
{
//Arrange
MemorySource source = new MemorySource();
source.Data = new List <string[]>()
{
null,
new string[] { "1", "Test1" },
null,
new string[] { "2", "Test2" },
new string[] { "3", "Test3" },
null
};
//Act
JsonDestination dest = new JsonDestination("./IgnoreNullValuesStringArray.json");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./IgnoreNullValuesStringArray.json"),
File.ReadAllText("res/JsonDestination/TwoColumnsStringArray.json"));
}
public void UsingRowTransformation()
{
Prepare();
var orderSource = new MemorySource <Order>();
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 815, CustomerName = "John"
});
orderSource.DataAsList.Add(new Order()
{
OrderNumber = 4711, CustomerName = "Jim"
});
var rowTrans = new RowTransformation <Order>(
row => {
int?id = SqlTask.ExecuteScalar <int>(SqlConnection,
sql: $"SELECT Id FROM CustomerTable WHERE Name='{row.CustomerName}'");
row.CustomerId = id;
return(row);
});
/* Delete below here */
var dest = new MemoryDestination <Order>();
orderSource.LinkTo(rowTrans).LinkTo(dest);
orderSource.Execute();
}
public void UnknownTableViaTableDefinition()
{
//Arrange
TableDefinition def = new TableDefinition("UnknownTable",
new List <TableColumn>()
{
new TableColumn("id", "INT")
});
//Arrange
string[] data = { "1", "2" };
MemorySource <string[]> source = new MemorySource <string[]>();
source.Data.Add(data);
DbDestination <string[]> dest = new DbDestination <string[]>(def, SqlConnection);
source.LinkTo(dest);
//Act & Assert
Assert.Throws <System.InvalidOperationException>(() =>
{
try
{
source.Execute();
dest.Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
public void ValidateSchemaForDynamicObject()
{
//Arrange
var source = new MemorySource();
dynamic n1 = new ExpandoObject(); n1.Xml = _validXml; source.DataAsList.Add(n1);
dynamic n2 = new ExpandoObject(); n2.Xml = _validXml; source.DataAsList.Add(n2);
dynamic n3 = new ExpandoObject(); n3.Xml = _invalidXml; source.DataAsList.Add(n3);
MemoryDestination dest = new MemoryDestination();
MemoryDestination <ETLBoxError> error = new MemoryDestination <ETLBoxError>();
//Act
XmlSchemaValidation schemaValidation = new XmlSchemaValidation();
schemaValidation.XmlSelector = row =>
{
dynamic r = row as ExpandoObject;
return(r.Xml);
};
schemaValidation.XmlSchema = xsdMarkup;
source.LinkTo(schemaValidation);
schemaValidation.LinkTo(dest);
schemaValidation.LinkErrorTo(error);
source.Execute();
dest.Wait();
error.Wait();
//Assert
Assert.True(dest.Data.Count == 2);
Assert.True(error.Data.Count == 1);
}
public void ExceptionInStoreKeyFunction()
{
//Arrange
MemorySource <MyRow> source = new MemorySource <MyRow>();
source.DataAsList = new List <MyRow>()
{
new MyRow {
Id = 1, ClassName = "Class1", DetailValue = 3.5
}
};
//Act
Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
(row, aggValue) => aggValue.AggValue += row.DetailValue,
row => row.ClassName,
(key, agg) => throw new Exception("Test")
);
MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();
source.LinkTo(agg);
agg.LinkTo(dest);
//Assert
Assert.Throws <AggregateException>(() =>
{
source.Execute();
dest.Wait();
});
}
public void NoErrorHandling()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = "X"
},
new MySimpleRow()
{
Col1 = "1"
},
new MySimpleRow()
{
Col1 = null
}
};
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("ErrorFileNoError.csv");
//Act
//Assert
Assert.ThrowsAny <Exception>(() =>
{
source.LinkTo(dest);
source.Execute();
dest.Wait();
});
}
public void WriteWithObject()
{
//Arrange
var source = new MemorySource <MyTextRow>();
source.DataAsList.Add(new MyTextRow()
{
Text = "Line 1"
});
source.DataAsList.Add(new MyTextRow()
{
Text = "Line 2"
});
source.DataAsList.Add(new MyTextRow()
{
Text = "Line 3"
});
//Act
TextDestination <MyTextRow> dest = new TextDestination <MyTextRow>("res/TextDestination/TestFile.txt"
, tr => tr.Text);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.True(File.Exists("res/TextDestination/TestFile.txt"));
Assert.Equal(File.ReadAllText("res/TextDestination/ToBeTestFile.txt"), File.ReadAllText("res/TextDestination/TestFile.txt"));
}
private static void RunExceptionFlowWithType <T>()
{
//Arrange
MemorySource <InputDataRow> source = new MemorySource <InputDataRow>();
source.DataAsList.Add(new InputDataRow()
{
LookupId = 1
});
MemorySource <T> lookupSource = new MemorySource <T>();
var lookup = new LookupTransformation <InputDataRow, T>(lookupSource);
MemoryDestination <InputDataRow> dest = new MemoryDestination <InputDataRow>();
source.LinkTo(lookup);
lookup.LinkTo(dest);
//Act && Assert
Assert.Throws <ETLBoxException>(() =>
{
try
{
source.Execute();
dest.Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
public void AggregateSimple()
{
//Arrange
MemorySource <MyRow> source = new MemorySource <MyRow>();
source.DataAsList = new List <MyRow>()
{
new MyRow {
Id = 1, DetailValue = 3.5
},
new MyRow {
Id = 2, DetailValue = 4.5
},
new MyRow {
Id = 3, DetailValue = 2.0
},
};
Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
(row, aggRow) => aggRow.AggValue += row.DetailValue
);
MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();
//Act
source.LinkTo(agg);
agg.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MyAggRow>(dest.Data,
ar => Assert.True(ar.AggValue == 10)
);
}
public void NoLookupSource()
{
//Arrange
MemorySource <MyDataRow> source = new MemorySource <MyDataRow>();
source.DataAsList.Add(new MyDataRow()
{
Col1 = 1, Col2 = "Test1"
});
//Act
var lookup = new LookupTransformation <MyDataRow, MyLookupRow>();
MemoryDestination <MyDataRow> dest = new MemoryDestination <MyDataRow>();
//Assert
Assert.Throws <ETLBoxException>(() =>
{
try
{
source.LinkTo(lookup);
lookup.LinkTo(dest);
source.Execute();
dest.Wait();
}
catch (AggregateException e) { throw e.InnerException; }
});
//Assert
}
public void DataIsFromList()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("MemoryDestination");
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(SqlConnection, "MemoryDestination");
//Act
source.Data = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
},
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
}
};
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void GroupingUsingStringArray()
{
//Arrange
MemorySource <string[]> source = new MemorySource <string[]>();
source.DataAsList.Add(new string[] { "Class1", "3.5" });
source.DataAsList.Add(new string[] { "Class1", "6.5" });
source.DataAsList.Add(new string[] { "Class2", "10" });
Aggregation <string[], MyAggRow> agg = new Aggregation <string[], MyAggRow>(
(row, aggValue) => aggValue.AggValue += Convert.ToDouble(row[1]),
row => row[0],
(key, agg) => agg.GroupName = (string)key
);
MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();
//Act
source.LinkTo(agg);
agg.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MyAggRow>(dest.Data,
ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class1"),
ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class2")
);
}
public void IgnoreWithObject()
{
//Arrange
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(SqlConnection, "DestIgnoreNullValues");
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
null,
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
null,
new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
},
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
},
null
};
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(SqlConnection, "DestIgnoreNullValues");
//Act
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
d2c.AssertTestData();
}
public void WriteIntoHttpClient()
{
//Arrange
Mock<HttpMessageHandler> handlerMock = CreateHandlerMoq();
HttpClient httpClient = CreateHttpClient(handlerMock);
MemorySource<MySimpleRow> source = new MemorySource<MySimpleRow>();
source.DataAsList.Add(new MySimpleRow() { Col1 = 1, Col2 = "Test1" });
//Act
JsonDestination<MySimpleRow> dest = new JsonDestination<MySimpleRow>("http://test.test", ResourceType.Http);
dest.HttpClient = httpClient;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
handlerMock.Protected().Verify(
"SendAsync",
Times.Exactly(1),
ItExpr.Is<HttpRequestMessage>(req =>
req.Method == HttpMethod.Get
&& req.RequestUri.Equals(new Uri("http://test.test"))
),
ItExpr.IsAny<CancellationToken>()
);
}
public void IgnoreWithObject()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
null,
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
null,
new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
},
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
},
null
};
//Act
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("./IgnoreNullValues.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./IgnoreNullValues.csv"),
File.ReadAllText("res/CsvDestination/TwoColumns.csv"));
}
public void IgnoreWithStringArray()
{
//Arrange
MemorySource <string[]> source = new MemorySource <string[]>();
source.DataAsList = new List <string[]>()
{
null,
new string[] { "1", "Test1" },
null,
new string[] { "2", "Test2" },
new string[] { "3", "Test3" },
null
};
//Act
CsvDestination <string[]> dest = new CsvDestination <string[]>("./IgnoreNullValuesStringArray.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./IgnoreNullValuesStringArray.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsNoHeader.csv"));
}
public void DeltaLoadWithDeletion()
{
//Arrange
MemorySource source = new MemorySource();
source.DataAsList.Add(CreateDynamicRow(2, "Test2"));
source.DataAsList.Add(CreateDynamicRow(3, "Test3"));
source.DataAsList.Add(CreateDynamicRow(4, delete: true));
source.DataAsList.Add(CreateDynamicRow(10, delete: true));
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(SqlConnection, "DBMergeDynamicDeltaDestination");
d2c.InsertTestDataSet3();
//Act
DbMerge dest = new DbMerge(SqlConnection, "DBMergeDynamicDeltaDestination")
{
DeltaMode = MergeMode.Delta
};
dest.MergeProperties.IdPropertyNames.Add("Col1");
dest.MergeProperties.ComparePropertyNames.Add("Col2");
dest.MergeProperties.DeletionProperties.Add("Delete", true);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
d2c.AssertTestData();
Assert.Collection <ExpandoObject>(dest.DeltaTable,
row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Update && r.Col1 == 2); },
row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Insert && r.Col1 == 3); },
row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Delete && r.Col1 == 4); },
row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Delete && r.Col1 == 10); }
);
}
public void ValidateSchemaForArray()
{
//Arrange
var source = new MemorySource <string[]>();
source.DataAsList.Add(new string[] { _validXml });
source.DataAsList.Add(new string[] { _invalidXml });
source.DataAsList.Add(new string[] { _validXml });
MemoryDestination <string[]> dest = new MemoryDestination <string[]>();
MemoryDestination <ETLBoxError> error = new MemoryDestination <ETLBoxError>();
//Act
XmlSchemaValidation <string[]> schemaValidation = new XmlSchemaValidation <string[]>();
schemaValidation.XmlSelector = row => row[0];
schemaValidation.XmlSchema = xsdMarkup;
source.LinkTo(schemaValidation);
schemaValidation.LinkTo(dest);
schemaValidation.LinkErrorTo(error);
source.Execute();
dest.Wait();
error.Wait();
//Assert
Assert.True(dest.Data.Count == 2);
Assert.True(error.Data.Count == 1);
}
public void SimpleMergeWithDynamic()
{
//Arrange
MemorySource source = new MemorySource();
source.DataAsList.Add(CreateDynamicRow(1, "Test1"));
source.DataAsList.Add(CreateDynamicRow(2, "Test2"));
source.DataAsList.Add(CreateDynamicRow(3, "Test3"));
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(SqlConnection, "DBMergeDynamicDestination");
d2c.InsertTestDataSet3();
//Act
DbMerge dest = new DbMerge(SqlConnection, "DBMergeDynamicDestination");
dest.MergeProperties.IdPropertyNames.Add("Col1");
dest.MergeProperties.ComparePropertyNames.Add("Col2");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(3, RowCountTask.Count(SqlConnection, "DBMergeDynamicDestination", $"{d2c.QB}Col1{d2c.QE} BETWEEN 1 AND 7 AND {d2c.QB}Col2{d2c.QE} LIKE 'Test%'"));
d2c.AssertTestData();
Assert.Collection <ExpandoObject>(dest.DeltaTable,
row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Exists && r.Col1 == 1); },
row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Update && r.Col1 == 2); },
row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Insert && r.Col1 == 3); },
row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Delete && r.Col1 == 4); },
row => { dynamic r = row as ExpandoObject; Assert.True(r.ChangeAction == ChangeAction.Delete && r.Col1 == 10); }
);
}
public void MixedTypes(IConnectionManager connection)
{
CreateTestTable(connection, "datatypedestination");
//Arrange
MemorySource <MyDataTypeRow> source = new MemorySource <MyDataTypeRow>(
new List <MyDataTypeRow>()
{
new MyDataTypeRow()
{
IntCol = 1,
LongCol = -1,
DecimalCol = 2.3M,
DoubleCol = 5.4,
DateTimeCol = new DateTime(2010, 1, 1, 10, 10, 10),
DateCol = new DateTime(2020, 1, 1),
StringCol = "Test",
CharCol = 'T',
DecimalStringCol = "13.4566",
NullCol = null,
EnumCol = EnumType.Value2
}
});
//Act
DbDestination <MyDataTypeRow> dest = new DbDestination <MyDataTypeRow>(connection, "datatypedestination");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
AssertFirstRow(connection, "datatypedestination");
}
public void MixedTypesWithDynamic(IConnectionManager connection)
{
CreateTestTable(connection, "datatypedestinationdynamic");
//Arrange
MemorySource source = new MemorySource();
dynamic d1 = new ExpandoObject();
d1.IntCol = 1;
d1.LongCol = -1;
d1.DecimalCol = 2.3M;
d1.DoubleCol = 5.4;
d1.DateTimeCol = new DateTime(2010, 1, 1, 10, 10, 10);
d1.DateCol = new DateTime(2020, 1, 1);
d1.StringCol = "Test";
d1.CharCol = 'T';
d1.DecimalStringCol = "13.4566";
d1.NullCol = null;
d1.EnumCol = EnumType.Value2;
//Act
DbDestination dest = new DbDestination(connection, "datatypedestinationdynamic");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
AssertFirstRow(connection, "datatypedestinationdynamic");
}
public void UnknownTable()
{
//Arrange
string[] data = { "1", "2" };
MemorySource source = new MemorySource();
source.Data.Add(data);
DBDestination dest = new DBDestination(SqlConnection, "UnknownTable");
source.LinkTo(dest);
//Act & Assert
Assert.Throws <ETLBoxException>(() =>
{
try
{
source.Execute();
dest.Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
public void IgnoreWithStringArray()
{
//Arrange
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(SqlConnection, "DestIgnoreNullValuesStringArray");
MemorySource <string[]> source = new MemorySource <string[]>();
source.DataAsList = new List <string[]>()
{
null,
new string[] { "1", "Test1" },
null,
new string[] { "2", "Test2" },
new string[] { "3", "Test3" },
null
};
DbDestination <string[]> dest = new DbDestination <string[]>(SqlConnection, "DestIgnoreNullValuesStringArray");
//Act
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
d2c.AssertTestData();
}
public void MixedTypes(IConnectionManager connection)
{
CreateTableTask.Create(connection, "datatypedestination",
new List <TableColumn>()
{
new TableColumn("IntCol", "INT", allowNulls: true),
new TableColumn("LongCol", "BIGINT", allowNulls: true),
new TableColumn("DecimalCol", "FLOAT", allowNulls: true),
new TableColumn("DoubleCol", "FLOAT", allowNulls: true),
new TableColumn("DateTimeCol", "DATETIME", allowNulls: true),
new TableColumn("DateCol", "DATE", allowNulls: true),
new TableColumn("StringCol", "VARCHAR(200)", allowNulls: true),
new TableColumn("CharCol", "CHAR(1)", allowNulls: true),
new TableColumn("DecimalStringCol", "DECIMAL(12,10)", allowNulls: true),
new TableColumn("NullCol", "CHAR(1)", allowNulls: true),
});
//Arrange
MemorySource <MyDataTypeRow> source = new MemorySource <MyDataTypeRow>(
new List <MyDataTypeRow>()
{
new MyDataTypeRow()
{
IntCol = 1,
LongCol = -1,
DecimalCol = 2.3M,
DoubleCol = 5.4,
DateTimeCol = new DateTime(2010, 1, 1, 10, 10, 10),
DateCol = new DateTime(2020, 1, 1),
StringCol = "Test",
CharCol = 'T',
DecimalStringCol = "13.4566",
NullCol = null
}
});
//Act
DbDestination <MyDataTypeRow> dest = new DbDestination <MyDataTypeRow>("datatypedestination", connection);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
// IntCol LongCol DecimalCol DoubleCol DateTimeCol DateCol StringCol CharCol DecimalStringCol NullCol
//1 - 1 2.3 5.4 2010 - 01 - 01 10:10:10.100 2020 - 01 - 01 Test T 13.4566000000 NULL
SqlTask.ExecuteReaderSingleLine(connection, "Check data", "SELECT * FROM datatypedestination",
col => Assert.True(Convert.ToInt32(col) == 1),
col => Assert.True(Convert.ToInt64(col) == -1),
col => Assert.True(Convert.ToDecimal(col) == 2.3M),
col => Assert.True(Convert.ToDecimal(col) == 5.4M),
col => Assert.True(Convert.ToDateTime(col) == new DateTime(2010, 1, 1, 10, 10, 10)),
col => Assert.True(Convert.ToDateTime(col) == new DateTime(2020, 1, 1)),
col => Assert.True(Convert.ToString(col) == "Test"),
col => Assert.True(Convert.ToString(col) == "T" || Convert.ToString(col) == "84"),
col => Assert.True(Convert.ToString(col).Replace("0", "") == "13.4566"),
col => Assert.True(col == null)
);
}
public void GroupingAndKeepingKey()
{
//Arrange
MemorySource <MyRow> source = new MemorySource <MyRow>();
source.Data = new List <MyRow>()
{
new MyRow {
Id = 1, ClassName = "Class1", DetailValue = 3.5
},
new MyRow {
Id = 2, ClassName = "Class1", DetailValue = 6.5
},
new MyRow {
Id = 3, ClassName = "Class2", DetailValue = 1.2
},
new MyRow {
Id = 4, ClassName = "Class2", DetailValue = 2.3
},
new MyRow {
Id = 5, ClassName = "Class2", DetailValue = 16.5
},
new MyRow {
Id = 6, ClassName = "Class3", DetailValue = 30.0
},
new MyRow {
Id = 6, ClassName = null, DetailValue = 14.5
},
new MyRow {
Id = 6, ClassName = null, DetailValue = 15.5
},
};
Aggregation <MyRow, MyAggRow> agg = new Aggregation <MyRow, MyAggRow>(
(row, aggValue) => aggValue.AggValue += row.DetailValue,
row => row.ClassName,
(key, agg) => agg.GroupName = (string)key
);
MemoryDestination <MyAggRow> dest = new MemoryDestination <MyAggRow>();
//Act
source.LinkTo(agg);
agg.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MyAggRow>(dest.Data,
ar => Assert.True(ar.AggValue == 10 && ar.GroupName == "Class1"),
ar => Assert.True(ar.AggValue == 20 && ar.GroupName == "Class2"),
ar => Assert.True(ar.AggValue == 30 && ar.GroupName == "Class3"),
ar => Assert.True(ar.AggValue == 30 && ar.GroupName == string.Empty)
);
}
public void WaitForLongRunningProcedure()
{
//Arrange
CreateProcedureTask.CreateOrAlter(SqlConnection, new ProcedureDefinition()
{
Name = "longrunning",
Definition = @"
WAITFOR DELAY '00:00:01';
SELECT @input + 10;",
Parameter = new List <ProcedureParameter>()
{
new ProcedureParameter()
{
Name = "input", DataType = "INT"
}
// new ProcedureParameter() { Name ="result", Out = true, DataType="INT"}
}
});
var source = new MemorySource <MySimpleRow>();
source.DataAsList.Add(new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
});
source.DataAsList.Add(new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
});
source.DataAsList.Add(new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
});
var dest = new MemoryDestination <MySimpleRow>();
var trans = new RowTransformation <MySimpleRow>(
row =>
{
row.Col1 = SqlTask.ExecuteScalar <int>(SqlConnection, "Read from procedure",
$"EXEC longrunning @input = {row.Col1}") ?? 0;
return(row);
}
);
//Act
source.LinkTo(trans);
trans.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <MySimpleRow>(dest.Data,
row => Assert.Equal(11, row.Col1),
row => Assert.Equal(12, row.Col1),
row => Assert.Equal(13, row.Col1)
);
}
public void RedirectBatch(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture d2c = new TwoColumnsTableFixture(connection, "DestLinkError");
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.Data = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = null, Col2 = "ErrorRecord"
},
new MySimpleRow()
{
Col1 = "X2", Col2 = "ErrorRecord"
},
new MySimpleRow()
{
Col1 = "1", Col2 = "Test1"
},
new MySimpleRow()
{
Col1 = "2", Col2 = "Test2"
},
new MySimpleRow()
{
Col1 = "3", Col2 = "Test3 - good, but in error batch"
},
new MySimpleRow()
{
Col1 = null, Col2 = "ErrorRecord"
},
new MySimpleRow()
{
Col1 = "3", Col2 = "Test3"
},
};
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(connection, "DestLinkError", batchSize: 2);
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
source.LinkTo(dest);
dest.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
d2c.AssertTestData();
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
private void TransferTestDataIntoDestination(List <MyMergeRow> knownGuids)
{
MemorySource <MyMergeRow> source = new MemorySource <MyMergeRow>();
source.Data = knownGuids;
DbDestination <MyMergeRow> dest = new DbDestination <MyMergeRow>(SqlConnection, "MergeDestination");
source.LinkTo(dest);
source.Execute();
dest.Wait();
}
