public void SortSimpleDataDescending()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("SortSourceNonGeneric");
source2Columns.InsertTestData();
DbSource <string[]> source = new DbSource <string[]>("SortSourceNonGeneric", Connection);
//Act
List <string[]> actual = new List <string[]>();
CustomDestination <string[]> dest = new CustomDestination <string[]>(
row => actual.Add(row)
);
Comparison <string[]> comp = new Comparison <string[]>(
(x, y) => int.Parse(y[0]) - int.Parse(x[0])
);
Sort <string[]> block = new Sort <string[]>(comp);
source.LinkTo(block);
block.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
List <int> expected = new List <int>()
{
3, 2, 1
};
Assert.Equal(expected, actual.Select(row => int.Parse(row[0])).ToList());
}
public void ColumnMappingExtended(IConnectionManager connection)
{
//Arrange
FourColumnsTableFixture source4Columns = new FourColumnsTableFixture(connection,
"SourceColumnMapping", identityColumnIndex: 0);
source4Columns.InsertTestData();
//Act
DbSource <MyExtendedRow> source = new DbSource <MyExtendedRow>("SourceColumnMapping", connection);
CustomDestination <MyExtendedRow> dest = new CustomDestination <MyExtendedRow>(
input =>
{
//Assert
Assert.InRange(input.Id, 1, 3);
Assert.StartsWith("Test", input.Text);
if (input.Id == 1)
{
Assert.Null(input.Value);
}
else
{
Assert.True(input.Value > 0);
}
Assert.InRange(input.Percentage, 1, 2);
});
source.LinkTo(dest);
source.Execute();
dest.Wait();
}
public void ExceptionalAsyncFlow()
{
//Arrange
CustomSource source = new CustomSource(
() => throw new ETLBoxException("Test Exception"), () => false
);
CustomDestination dest = new CustomDestination
(row => {; });
//Act
source.LinkTo(dest);
//Assert
Assert.Throws <ETLBoxException>(() =>
{
Task sourceT = source.ExecuteAsync();
Task destT = dest.Completion;
try
{
sourceT.Wait();
destT.Wait();
}
catch (Exception e)
{
throw e.InnerException;
}
});
}
public void InsertIntoTable()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("CustomDestinationNonGenericSource");
source2Columns.InsertTestData();
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CustomDestinationNonGenericDestination");
//Act
DbSource <string[]> source = new DbSource <string[]>(SqlConnection, "CustomDestinationNonGenericSource");
CustomDestination <string[]> dest = new CustomDestination <string[]>(
row =>
{
SqlTask.ExecuteNonQuery(SqlConnection, "Insert row",
$"INSERT INTO dbo.CustomDestinationNonGenericDestination VALUES({row[0]},'{row[1]}')");
}
);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void InsertIntoTable()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("CustomDestinationDynamicSource");
source2Columns.InsertTestData();
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CustomDestinationDynamicDestination");
//Act
DbSource <ExpandoObject> source = new DbSource <ExpandoObject>(SqlConnection, "CustomDestinationDynamicSource");
CustomDestination <ExpandoObject> dest = new CustomDestination <ExpandoObject>(
row =>
{
dynamic r = row as ExpandoObject;
SqlTask.ExecuteNonQuery(SqlConnection, "Insert row",
$"INSERT INTO dbo.CustomDestinationDynamicDestination VALUES({r.Col1},'{r.Col2}')");
}
);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
protected override void InitBufferObjects()
{
InMemoryTarget = new MemoryDestination <TInput1>(this);
PassingTarget = new CustomDestination <TInput2>(this, CrossJoinData);
if (MaxBufferSize > 0)
{
PassingTarget.MaxBufferSize = this.MaxBufferSize;
}
PassingTarget.OnCompletion = () => Buffer.Complete();
}
public ReferenceDataFlow(int key, string value)
{
var x = new MyData()
{
Key = key, Value = value
};
_source = new MemorySource <MyData>();
_source.DataAsList.Add(x);
_destination = new CustomDestination <MyData>(x => Data.Add(x.Key, x));
_source.LinkTo(_destination);
_source.ExecuteAsync();
}
public void TestDuplicateCheckInRowTrans()
{
CSVSource <Poco> source = new CSVSource <Poco>("src/DataFlowExamples/Duplicate.csv");
source.Configuration.Delimiter = ";";
source.Configuration.TrimOptions = CsvHelper.Configuration.TrimOptions.Trim;
source.Configuration.MissingFieldFound = null;
List <int> IDs = new List <int>(); //at the end of the flow, this list will contain all IDs of your source
RowTransformation <Poco, Poco> rowTrans = new RowTransformation <Poco, Poco>(input =>
{
if (IDs.Contains(input.ID))
{
input.IsDuplicate = true;
}
else
{
IDs.Add(input.ID);
}
return(input);
});
var multicast = new Multicast <Poco>();
var dest = new DBDestination <Poco>("dbo.Staging");
TableDefinition stagingTable = new TableDefinition("dbo.Staging", new List <TableColumn>()
{
new TableColumn("Key", "INT", allowNulls: false, isPrimaryKey: true, isIdentity: true),
new TableColumn("ID", "INT", allowNulls: false),
new TableColumn("Value", "NVARCHAR(100)", allowNulls: false),
new TableColumn("Name", "NVARCHAR(100)", allowNulls: false)
});
stagingTable.CreateTable();
var trash = new CustomDestination <Poco>(input => {
LogTask.Warn($"Duplicate found. ID: {input.ID} Name: {input.Name}");
});
source.LinkTo(rowTrans);
rowTrans.LinkTo(multicast);
multicast.LinkTo(dest, input => input.IsDuplicate == false);
multicast.LinkTo(trash, input => input.IsDuplicate == true);
source.Execute();
dest.Wait();
trash.Wait();
}
public void DBSourceWithColumnMapping()
{
SqlTask.ExecuteNonQuery("Create source table", @"CREATE TABLE test.Source
(Col1 nvarchar(100) null, Col2 nvarchar(100) null)");
SqlTask.ExecuteNonQuery("Insert demo data", "insert into test.Source values('Test1','Test2')");
SqlTask.ExecuteNonQuery("Insert demo data", "insert into test.Source values('Test1','Test2')");
SqlTask.ExecuteNonQuery("Insert demo data", "insert into test.Source values('Test1','Test2')");
DBSource <ColumnMapRow> source = new DBSource <ColumnMapRow>("test.Source");
CustomDestination <ColumnMapRow> dest = new CustomDestination <ColumnMapRow>(
input => {
Assert.AreEqual("Test1", input.Col1);
Assert.AreEqual("Test2", input.B);
});
source.LinkTo(dest);
source.Execute();
dest.Wait();
}
public void TestErrorLink()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
new MySimpleRow()
{
Col1 = 2, Col2 = "ErrorRecord"
},
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
},
};
CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>(
row =>
{
if (row.Col1 == 2)
{
throw new Exception("Error record!");
}
}
);
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
source.LinkTo(dest);
dest.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public void DB_CustDest()
{
TableDefinition sourceTableDefinition = CreateSourceTable("test.Source");
TableDefinition destinationTableDefinition = CreateDestinationTable("test.Destination");
DBSource <MySimpleRow> source = new DBSource <MySimpleRow>()
{
SourceTableDefinition = sourceTableDefinition
};
CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>(
row => {
SqlTask.ExecuteNonQuery("Insert row", $"insert into test.Destination values('{row.Value1}',{row.Value2})");
}
);
source.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.AreEqual(3, RowCountTask.Count("test.Destination"));
}
public void Table2JsonFile()
{
TableDefinition sourceTableDefinition = CreateSourceTable("test.Source");
DBSource <MySimpleRow> source = new DBSource <MySimpleRow>(sourceTableDefinition);
List <MySimpleRow> rows = new List <MySimpleRow>();
CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>(
row => {
rows.Add(row);
}
);
source.LinkTo(dest);
source.Execute();
dest.Wait();
string json = JsonConvert.SerializeObject(rows, Formatting.Indented);
Assert.AreEqual(json, File.ReadAllText("src/DataFlow/json_tobe.json"));
}
public void IgnoreWithObject()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.Data = new List <MySimpleRow>()
{
null,
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
null,
new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
},
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
},
null
};
//Act
List <MySimpleRow> result = new List <MySimpleRow>();
CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>(
row => result.Add(row)
);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection(result,
d => Assert.True(d.Col1 == 1 && d.Col2 == "Test1"),
d => Assert.True(d.Col1 == 2 && d.Col2 == "Test2"),
d => Assert.True(d.Col1 == 3 && d.Col2 == "Test3")
);
}
public void ColumnMapping(IConnectionManager connection)
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture(connection, "Source");
source2Columns.InsertTestData();
//Act
DBSource <ColumnMapRow> source = new DBSource <ColumnMapRow>(connection, "Source");
CustomDestination <ColumnMapRow> dest = new CustomDestination <ColumnMapRow>(
input =>
{
//Assert
Assert.InRange(input.Col1, 1, 3);
Assert.StartsWith("Test", input.B);
});
source.LinkTo(dest);
source.Execute();
dest.Wait();
}
public void CreateJsonFile()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("JSonSource");
source2Columns.InsertTestData();
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(SqlConnection, "JSonSource");
List <MySimpleRow> rows = new List <MySimpleRow>();
//Act
CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>(
row => rows.Add(row)
);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Act
string json = JsonConvert.SerializeObject(rows, Formatting.Indented);
//Assert
Assert.Equal(File.ReadAllText("res/CustomDestination/simpleJson_tobe.json"), json);
}
public void SortSimpleDataDescending()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("SortSourceNonGeneric");
source2Columns.InsertTestData();
DbSource <ExpandoObject> source = new DbSource <ExpandoObject>(Connection, "SortSourceNonGeneric");
//Act
List <ExpandoObject> actual = new List <ExpandoObject>();
CustomDestination <ExpandoObject> dest = new CustomDestination <ExpandoObject>(
row => actual.Add(row)
);
Comparison <ExpandoObject> comp = new Comparison <ExpandoObject>(
(x, y) =>
{
dynamic xo = x as ExpandoObject;
dynamic yo = y as ExpandoObject;
return(yo.Col1 - xo.Col1);
}
);
Sort <ExpandoObject> block = new Sort <ExpandoObject>(comp);
source.LinkTo(block);
block.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
List <int> expected = new List <int>()
{
3, 2, 1
};
Assert.Equal(expected, actual.Select(row => { dynamic r = row as ExpandoObject; return(r.Col1); }).Cast <int>().ToList());
}
public void InsertIntoTable()
{
//Arrange
TwoColumnsTableFixture source2Columns = new TwoColumnsTableFixture("Source");
source2Columns.InsertTestData();
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("CustomDestination");
//Act
DBSource <MySimpleRow> source = new DBSource <MySimpleRow>(Connection, "Source");
CustomDestination <MySimpleRow> dest = new CustomDestination <MySimpleRow>(
row => {
SqlTask.ExecuteNonQuery(Connection, "Insert row",
$"INSERT INTO dbo.CustomDestination VALUES({row.Col1},'{row.Col2}')");
}
);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public CrossJoin()
{
InMemoryTarget = new MemoryDestination <TInput1>(this);
PassingTarget = new CustomDestination <TInput2>(this, CrossJoinData);
PassingTarget.OnCompletion = () => Buffer.Complete();
}
public LookupTransformation()
{
LookupBuffer = new CustomDestination <TSourceOutput>(this, FillBuffer);
DefaultInitWithMatchRetrieveAttributes();
}
