public void IgnoreWithObject()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
null,
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
null,
new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
},
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
},
null
};
//Act
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("./IgnoreNullValues.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./IgnoreNullValues.csv"),
File.ReadAllText("res/CsvDestination/TwoColumns.csv"));
}
public void IgnoreWithStringArray()
{
//Arrange
MemorySource <string[]> source = new MemorySource <string[]>();
source.DataAsList = new List <string[]>()
{
null,
new string[] { "1", "Test1" },
null,
new string[] { "2", "Test2" },
new string[] { "3", "Test3" },
null
};
//Act
CsvDestination <string[]> dest = new CsvDestination <string[]>("./IgnoreNullValuesStringArray.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./IgnoreNullValuesStringArray.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsNoHeader.csv"));
}
public void SerializingDateTime()
{
int rowCount = 0;
//Arrange
CustomSource <MySeriRow> source = new CustomSource <MySeriRow>(
() =>
new MySeriRow()
{
Col1 = 1,
Col2 = new DateTime(2010, 02, 05)
},
() => rowCount++ == 1);
//Act
CsvDestination <MySeriRow> dest = new CsvDestination <MySeriRow>("./DateTimeSerialization.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./DateTimeSerialization.csv"),
File.ReadAllText("res/CsvDestination/DateTimeSerialization.csv"));
}
public void NoErrorHandling()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = "X"
},
new MySimpleRow()
{
Col1 = "1"
},
new MySimpleRow()
{
Col1 = null
}
};
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("ErrorFileNoError.csv");
//Act
//Assert
Assert.ThrowsAny <Exception>(() =>
{
source.LinkTo(dest);
source.Execute();
dest.Wait();
});
}
public void RedirectSingleRecordWithObject()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = "X"
},
new MySimpleRow()
{
Col1 = "1"
},
new MySimpleRow()
{
Col1 = "2"
},
new MySimpleRow()
{
Col1 = null
},
new MySimpleRow()
{
Col1 = "3"
},
};
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("ErrorFile.csv");
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
source.LinkTo(dest);
dest.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./ErrorFile.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsErrorLinking.csv"));
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public void SimpleFlow()
{
//Arrange
TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("CSVDestDynamicObject");
s2C.InsertTestDataSet3();
DbSource <ExpandoObject> source = new DbSource <ExpandoObject>(SqlConnection, "CSVDestDynamicObject");
//Act
CsvDestination <ExpandoObject> dest = new CsvDestination <ExpandoObject>("./SimpleWithDynamicObject.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./SimpleWithDynamicObject.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsSet3DynamicObject.csv"));
}
public void SimpleFlowWithObject()
{
//Arrange
TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("CSVDestSimple");
s2C.InsertTestDataSet3();
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>("CSVDestSimple", SqlConnection);
//Act
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("./SimpleWithObject.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./SimpleWithObject.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsSet3.csv"));
}
public void SimpleNonGeneric()
{
//Arrange
TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("CSVDestSimpleNonGeneric");
s2C.InsertTestDataSet3();
DbSource <string[]> source = new DbSource <string[]>(SqlConnection, "CSVDestSimpleNonGeneric");
//Act
CsvDestination <string[]> dest = new CsvDestination <string[]>("./SimpleNonGeneric.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
//Assert
Assert.Equal(File.ReadAllText("./SimpleNonGeneric.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsSet3NoHeader.csv"));
}
public void DisableHeader()
{
//Arrange
TwoColumnsTableFixture s2c = new TwoColumnsTableFixture("CsvSourceNoHeader");
s2c.InsertTestData();
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(SqlConnection, "CsvSourceNoHeader");
//Act
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("./ConfigurationNoHeader.csv");
dest.Configuration.HasHeaderRecord = false;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./ConfigurationNoHeader.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsNoHeader.csv"));
}
public void Run()
{
PostgresConnectionManager conMan = new PostgresConnectionManager(PostgresConnectionString);
//Import CSV
CsvSource sourceCSV = new CsvSource("NameList.csv");
DbDestination importDest = new DbDestination(conMan, "NameTable");
sourceCSV.LinkTo(importDest);
sourceCSV.Execute();
importDest.Wait();
//Export again
DbSource <NameListElement> sourceTable = new DbSource <NameListElement>(conMan, "NameTable");
CsvDestination <NameListElement> destCSV = new CsvDestination <NameListElement>("Export.csv");
destCSV.Configuration.Delimiter = ";";
sourceTable.LinkTo(destCSV);
sourceTable.Execute();
destCSV.Wait();
}
public void WriteIntoMultipleDestinations()
{
//Arrange
var source = new MemorySource <string[]>();
source.DataAsList.Add(new string[] { "Test" });
var trans = new RowTransformation <string[]>();
trans.TransformationFunc = r => throw new Exception();
var dest = new MemoryDestination <string[]>();
CreateErrorTableTask.Create(SqlConnection, "error_log");
var mc = new Multicast <ETLBoxError>();
var errorMem = new MemoryDestination <ETLBoxError>();
var errorDb = new DbDestination <ETLBoxError>(SqlConnection, "error_log");
var errorCsv = new CsvDestination <ETLBoxError>("error_csv.csv");
source.LinkTo(trans);
trans.LinkTo(dest);
//Act
trans.LinkErrorTo(mc);
mc.LinkTo(errorMem);
mc.LinkTo(errorDb);
mc.LinkTo(errorCsv);
source.Execute();
dest.Wait();
errorMem.Wait();
errorDb.Wait();
errorCsv.Wait();
//Assert
Assert.True(errorMem.Data.Count > 0);
Assert.True(RowCountTask.Count(SqlConnection, "error_log") > 0);
Assert.True(File.ReadAllText("error_csv.csv").Length > 0);
}
public void ErrorIntoCsv()
{
var source = new MemorySource <MyXmlRow>();
source.DataAsList.Add(new MyXmlRow()
{
Xml = _validXml
});
source.DataAsList.Add(new MyXmlRow()
{
Xml = _invalidXml
});
source.DataAsList.Add(new MyXmlRow()
{
Xml = _validXml
});
XmlSchemaValidation <MyXmlRow> valid = new XmlSchemaValidation <MyXmlRow>(xsdMarkup, i => i.Xml);
CsvDestination <ETLBoxError> errDest = new CsvDestination <ETLBoxError>("res/XmlValidation/Error.csv");
CsvDestination <MyXmlRow> dest = new CsvDestination <MyXmlRow>("res/XmlValidation/NoError.csv");
source.LinkTo(valid);
valid.LinkTo(dest);
valid.LinkErrorTo(errDest);
source.Execute();
dest.Wait();
errDest.Wait();
Assert.Equal(File.ReadAllText("res/XmlValidation/ValidXml.csv"),
File.ReadAllText("res/XmlValidation/NoError.csv"), ignoreCase: true, ignoreLineEndingDifferences: true, ignoreWhiteSpaceDifferences: true);
Assert.StartsWith(@"ErrorText,ReportTime,ExceptionType,RecordAsJson
The element 'Root' has invalid child element 'Child3'. List of possible elements expected: 'Child2'.",
File.ReadAllText("res/XmlValidation/Error.csv"));
}
