public void TestCsvCollect()
{
var container = new DataContainer();
var collection = new DataCollection(container, "EMAILS-UTF8.CSV");
var processor = new CsvFileProcessor();
long rowCount;
List <DataEntity> entities;
using (var stream = File.Open("../../../tests/EMAILS-UTF8.CSV", FileMode.Open))
{
entities = processor.ParseFileSchema(container, collection, stream, out rowCount);
}
var entity = entities.FirstOrDefault(x => x.Name.Equals("FROM_ADDR"));
int index = entities.IndexOf(entity);
using (var stream = File.Open("../../../tests/EMAILS-UTF8.CSV", FileMode.Open)) {
var samples =
processor.CollectSamples(container, collection, entity, index, stream, 5, 1);
Assert.Contains("*****@*****.**", samples);
}
}
public void TestCsvNoHeaderSchema()
{
var fields = new Dictionary <string, DataType>()
{
{ "column0", DataType.Int },
{ "column1", DataType.String },
{ "column2", DataType.Boolean }
};
var container = new DataContainer();
var collection = new DataCollection(container, "no-header.csv");
long rowCount;
var processor = new CsvFileProcessor(new Dictionary <string, object> {
{ "csv_has_header", false }
});
List <DataEntity> entities;
using (var stream = File.Open("../../../tests/no-header.csv", FileMode.Open))
{
entities = processor.ParseFileSchema(container, collection, stream, out rowCount);
}
Assert.Equal(1, rowCount);
foreach (var field in fields)
{
output.WriteLine($" check field {field.Key}");
var entity = entities.Find(x => x.Name.Equals(field.Key));
Assert.NotNull(entity);
Assert.Equal(field.Value, entity.DataType);
}
}
public void AddLargestNumber()
{
const string inputDir = @"d:\root\in";
const string inputFileName = @"myfile.csv";
var inputFilePath = Path.Combine(inputDir, inputFileName);
const string outputDir = @"d:\root\out";
const string outputFileName = @"myfileout.csv";
var outputFilePath = Path.Combine(outputDir, outputFileName);
var csvLines = new StringBuilder();
csvLines.AppendLine("OrderNumber,CustomerNumber,Description,Quantity");
csvLines.AppendLine("42, 100001, Shirt, II");
csvLines.AppendLine("43, 200002, Shorts, I");
csvLines.AppendLine("@ This is a comment");
csvLines.AppendLine("");
csvLines.AppendLine("44, 300003, Cap, V");
var mockFileInput = new MockFileData(csvLines.ToString());
var mockFileSystem = new MockFileSystem();
mockFileSystem.AddFile(inputFilePath, mockFileInput);
mockFileSystem.AddDirectory(outputDir);
var sut = new CsvFileProcessor(inputFilePath, outputFilePath, mockFileSystem);
sut.Process();
Assert.True(mockFileSystem.FileExists(outputFilePath));
MockFileData processedFile = mockFileSystem.GetFile(outputFilePath);
Approvals.Verify(processedFile.TextContents);
}
public void OutputProcessedOrderCsvData()
{
const string inputDir = @"c:\root\in";
const string inputFileName = "myfile.csv";
var inputFilePath = Path.Combine(inputDir, inputFileName);
const string outputDir = @"c:\root\out";
const string outputFileName = "myfileout.csv";
var outputFilePath = Path.Combine(outputDir, outputFileName);
var csvLines = new StringBuilder();
csvLines.AppendLine("OrderNumber,CustomerNumber,Description,Quantity");
csvLines.AppendLine("42, 100001, Shirt, II");
csvLines.AppendLine("43, 200002, Shorts, I");
csvLines.AppendLine("@ This is a comment");
csvLines.AppendLine("");
csvLines.Append("44, 300003, Cap, V");
var mockInputFile = new MockFileData(csvLines.ToString());
var mockFileSystem = new MockFileSystem();
mockFileSystem.AddFile(inputFilePath, mockInputFile);
mockFileSystem.AddDirectory(outputDir);
var sut = new CsvFileProcessor(inputFilePath, outputFilePath, mockFileSystem);
sut.Process();
Assert.True(mockFileSystem.FileExists(outputFilePath));
var processedFile = mockFileSystem.GetFile(outputFilePath);
var lines = processedFile.TextContents.SplitLines();
//Assert.Equal("OrderNumber,Customer,Amount", lines[0]);
//Assert.Equal("42,100001,2", lines[1]);
//Assert.Equal("43,200002,1", lines[2]);
//Assert.Equal("44,300003,5", lines[3]);
Approvals.Verify(processedFile.TextContents);
}
public void TestCsvSchema()
{
var fields = new Dictionary <string, DataType>()
{
{ "ID", DataType.Guid },
{ "DELETED", DataType.Boolean },
{ "CREATED_BY", DataType.Guid },
{ "DATE_ENTERED", DataType.DateTime },
{ "MODIFIED_USER_ID", DataType.Guid },
{ "DATE_MODIFIED", DataType.DateTime },
{ "DATE_MODIFIED_UTC", DataType.DateTime },
{ "ASSIGNED_USER_ID", DataType.Guid },
{ "TEAM_ID", DataType.String },
{ "NAME", DataType.String },
{ "DATE_START", DataType.DateTime },
{ "TIME_START", DataType.DateTime },
{ "PARENT_TYPE", DataType.String },
{ "PARENT_ID", DataType.Guid },
{ "DESCRIPTION", DataType.String },
{ "DESCRIPTION_HTML", DataType.String },
{ "FROM_ADDR", DataType.String },
{ "FROM_NAME", DataType.String },
};
var container = new DataContainer();
var collection = new DataCollection(container, "EMAILS-UTF8.CSV");
long rowCount;
var processor = new CsvFileProcessor();
List <DataEntity> entities;
using (var stream = File.Open("../../../tests/EMAILS-UTF8.CSV", FileMode.Open)) {
entities = processor.ParseFileSchema(container, collection, stream, out rowCount);
}
Assert.Equal(200, rowCount);
foreach (var field in fields)
{
output.WriteLine($" check field {field.Key}");
var entity = entities.Find(x => x.Name.Equals(field.Key));
Assert.NotNull(entity);
Assert.Equal(field.Value, entity.DataType);
}
}
public void TestCsvEncodings()
{
var container = new DataContainer();
var collection1 = new DataCollection(container, "EMAILS-UTF8.CSV");
var collection2 = new DataCollection(container, "EMAILS-UTF16.CSV");
var emails = new string[] { "*****@*****.**", "*****@*****.**", "*****@*****.**" };
var processor = new CsvFileProcessor();
long rowCount;
List <DataEntity> entities;
using (var stream = File.Open("../../../tests/EMAILS-UTF8.CSV", FileMode.Open))
{
entities = processor.ParseFileSchema(container, collection1, stream, out rowCount);
}
var entity = entities.FirstOrDefault(x => x.Name.Equals("FROM_ADDR"));
int index = entities.IndexOf(entity);
List <string> samples1;
List <string> samples2;
using (var stream = File.Open("../../../tests/EMAILS-UTF8.CSV", FileMode.Open)) {
samples1 = processor.CollectSamples(container, collection1, entity, index, stream,
emails.Length, 1);
}
using (var stream = File.Open("../../../tests/EMAILS-UTF16.CSV", FileMode.Open)) {
samples2 = processor.CollectSamples(container, collection2, entity, index, stream,
emails.Length, 1);
}
Assert.Equal(emails.Length, samples1.Count);
Assert.Equal(emails.Length, samples2.Count);
for (int i = 0; i < emails.Length; i++)
{
Assert.Equal(emails[i], samples1[i]);
Assert.Equal(emails[i], samples2[i]);
}
}
public void OutputProcessedPolicyXmlData()
{
// Create mock file system to test program
const string inputDir = @"c:\root\in";
const string inputFileName = "myFile.csv";
string inputFilePath = Path.Combine(inputDir, inputFileName);
const string outputDir = @"c:\root\out";
const string outputFileName = "myFileOut.xml";
string outputFilePath = Path.Combine(outputDir, outputFileName);
// Create dummy test data to test program produces correct values
StringBuilder csvLines = new StringBuilder();
csvLines.AppendLine("policy_number,policy_start_date,premiums,membership,discretionary_bonus,uplift_percentage");
csvLines.AppendLine("A100001,01/06/1986,10000,Y,1000,40");
csvLines.AppendLine("A100002,01/01/1990,12500,N,1350,37.5");
csvLines.AppendLine("A100003,31/12/1989,15250,N,1600,42");
csvLines.AppendLine("B100001,01/01/1995,12000,Y,2000,41");
csvLines.AppendLine("B100002,01/01/1970,18000,N,3000,43");
csvLines.AppendLine("B100003,20/07/1969,20000,Y,4000,45");
csvLines.AppendLine("C100001,01/01/1992,13000,N,1000,42");
csvLines.AppendLine("C100002,31/12/1989,15000,Y,2000,44");
csvLines.Append("C100003,01/01/1990,17000,Y,3000,46");
MockFileData mockInputFile = new MockFileData(csvLines.ToString());
MockFileSystem mockFileSystem = new MockFileSystem();
mockFileSystem.AddFile(inputFilePath, mockInputFile);
mockFileSystem.AddDirectory(outputDir);
CsvFileProcessor csvFileProcessor = new CsvFileProcessor(inputFilePath, outputFilePath, mockFileSystem);
csvFileProcessor.Process();
Assert.True(mockFileSystem.FileExists(outputFilePath));
MockFileData processedFile = mockFileSystem.GetFile(outputFilePath);
Approvals.Verify(processedFile.TextContents);
}
public void TestCsvDetect()
{
var processor = new CsvFileProcessor();
Assert.True(processor.CanProcess("EMAILS-UTF8.CSV"));
}