public void FileWithUnknownColumns_ShouldDiscardColumns()
{
var description = new CsvMetaData
{
SeparatorChar = ',',
FirstLineHasColumnNames = true,
IgnoreUnknownColumns = true,
};
//The following input has 5 columns: Id | Name | Last Name | Age | City. Only the Name, Last Name and Age will be read.
string input =
@"Id,Name,Last Name,Age,City
1,John,Doe,15,Washington
2,Jane,Doe,20,New York
";
var expected = new[]
{
new Person
{
Name = "John",
LastName = "Doe",
Age = 15
},
new Person
{
Name = "Jane",
LastName = "Doe",
Age = 20
},
};
AssertRead(input, description, expected);
}
/// <summary>
/// Executes a Read and tests whether it outputs the expected records.
/// </summary>
/// <typeparam name="T">
/// Type of the output elements.
/// </typeparam>
/// <param name="testInput">
/// String representing the contents of the file or StreamReader. This string is fed to the Read method
/// as though it came from a file or StreamReader.
/// </param>
/// <param name="metaData">
/// Passed to Read.
/// </param>
/// <param name="expected">
/// Expected output.
/// </param>
public void AssertRead <T>(string testInput, CsvMetaData metaData, IEnumerable <T> expected)
where T : class, IAssertable <T>, new()
{
IEnumerable <T> actual = TestRead <T>(testInput, metaData);
AssertCollectionsEqual <T>(actual, expected);
}
public void GoodFileNoSeparatorCharUsEnglish()
{
// Arrange
var fileDescriptionNamesUs = new CsvMetaData
{
NoSeparatorChar = true,
UseOutputFormatForParsingCsvValue = false,
FirstLineHasColumnNames = false,
EnforceCsvColumnAttribute = true, // default is false
FileCultureName = "en-US" // default is the current culture
};
string testInput =
@"AAAAAAAA34.18405/23/08\n
BBBBBBBB10.31105/12/12\n
CCCCCCCC12.00012/23/08";
var expected = new[] {
new ProductDataCharLength {
name = "AAAAAAAA", weight = 34.184, startDate = new DateTime(2008, 5, 23),
},
new ProductDataCharLength {
name = "BBBBBBBB", weight = 10.311, startDate = new DateTime(2012, 5, 12),
},
new ProductDataCharLength {
name = "CCCCCCCC", weight = 12.000, startDate = new DateTime(2008, 12, 23),
}
};
// Act and Assert
AssertRead(testInput, fileDescriptionNamesUs, expected);
}
/// <summary>
/// Used to test the Write method
/// </summary>
/// <typeparam name="T">
/// The type of the input elements.
/// </typeparam>
/// <param name="values">
/// The collection of input elements.
/// </param>
/// <param name="metaData">
/// Passed directly to write.
/// </param>
/// <returns>
/// Returns a string with the content that the Write method writes to a file or TextWriter.
/// </returns>
public string TestWrite <T>(IEnumerable <T> values, CsvMetaData metaData) where T : class
{
TextWriter stream = new StringWriter();
CsvContext cc = new CsvContext();
cc.Write(values, stream, metaData);
return(stream.ToString());
}
public void GoodFileCommaDelimitedWithTrailingSeparatorChars()
{
// Arrange
CsvMetaData metaDataNamesUs = new CsvMetaData
{
SeparatorChar = ',', // default is ','
FirstLineHasColumnNames = true,
EnforceCsvColumnAttribute = false, // default is false
FileCultureName = "en-US", // default is the current culture
IgnoreTrailingSeparatorChar = true
};
string testInput =
@"name, weight, startDate, launchTime, nbrAvailable,onsale,shopsAvailable, code, price, description,
moonbuggy, 34.184, 5/23/08, 5-May-2009 4:11 pm, 1205, true, ""Paris, New York"", 1F, $540.12, newly launched product,
""mouse trap"",45E-5, 1/2/1985, ""7 August 1988, 0:00 am"", ""4,030"", FALSE, ""This field has
a newline"", 100, ""$78,300"", ""This field has quotes(""""), and
two newlines
and a quoted """"string""""""
dog house, ""45,230,990"",29 Feb 2004, , -56, True,"""", FF10, ""12,008"",";
var expected = new[] {
new ProductData {
name = "moonbuggy", weight = 34.184, startDate = new DateTime(2008, 5, 23), launchTime = new DateTime(2009, 5, 5, 16, 11, 0),
nbrAvailable = 1205, onsale = true, shopsAvailable = "Paris, New York", hexProductCode = 31, retailPrice = 540.12M,
description = "newly launched product"
},
new ProductData {
name = "mouse trap", weight = 45E-5, startDate = new DateTime(1985, 1, 2), launchTime = new DateTime(1988, 8, 7, 0, 0, 0),
nbrAvailable = 4030, onsale = false, shopsAvailable = @"This field has
a newline", hexProductCode = 256, retailPrice = 78300M,
description = @"This field has quotes(""), and
two newlines
and a quoted ""string"""
},
new ProductData {
name = "dog house", weight = 45230990, startDate = new DateTime(2004, 2, 29), launchTime = default(DateTime),
nbrAvailable = -56, onsale = true, shopsAvailable = "", hexProductCode = 65296, retailPrice = 12008M,
description = null
}
};
// Act and Assert
AssertRead(testInput, metaDataNamesUs, expected);
}
public void GoodFileUsingOutputFormatForParsingDatesCharUsEnglish()
{
// Arrange
var fileDescriptionNamesUs = new CsvMetaData
{
SeparatorChar = ';',
FirstLineHasColumnNames = false,
UseOutputFormatForParsingCsvValue = true,
EnforceCsvColumnAttribute = true, // default is false
FileCultureName = "en-US" // default is the current culture
};
StringBuilder input = new StringBuilder();
input.Append("AAAAAAAA;052308");
input.Append(Environment.NewLine);
input.Append("BBBBBBBB;051212");
input.Append(Environment.NewLine);
input.Append(Environment.NewLine);
input.Append("CCCCCCCC;122308");
var expected = new[] {
new ProductDataParsingOutputFormat {
name = "AAAAAAAA", startDate = new DateTime(2008, 5, 23),
},
new ProductDataParsingOutputFormat {
name = "BBBBBBBB", startDate = new DateTime(2012, 5, 12),
},
new ProductDataParsingOutputFormat {
name = "CCCCCCCC", startDate = new DateTime(2008, 12, 23),
}
};
// Act and Assert
AssertRead(input.ToString(), fileDescriptionNamesUs, expected);
}
public void GoodFileTabDelimitedNoNamesInFirstLineNLnl()
{
// Arrange
CsvMetaData metaDataNonamesNl = new CsvMetaData
{
SeparatorChar = '\t', // tab character
FirstLineHasColumnNames = false,
EnforceCsvColumnAttribute = true,
FileCultureName = "nl-NL" // default is the current culture
};
string testInput =
"moonbuggy\t 23/5/08\t 5-Mei-2009 16:11 pm\t 34.184\t \"Paris, New York\"\t 1F\t €540,12\t true\t newly launched product\r\n\"mouse trap\"\t 2/1/1985\t \"7 Augustus 1988\t 0:00\"\t45E-5\t \"This field has\r\na newline\"\t 100\t \"€78.300\"\t FALSE\t \"This field has quotes(\"\"), and\r\ntwo newlines\r\nand a quoted \"\"string\"\"\"\r\ndog house\t29 Feb 2004\t \t \"45.230.990\"\t\"\"\t FF10\t \"12.008\"\t True";
var expected = new[] {
new ProductData {
name = "moonbuggy", weight = 34184, startDate = new DateTime(2008, 5, 23), launchTime = new DateTime(2009, 5, 5, 16, 11, 0),
nbrAvailable = 0, onsale = true, shopsAvailable = "Paris, New York", hexProductCode = 31, retailPrice = 540.12M,
description = "newly launched product"
},
new ProductData {
name = "mouse trap", weight = 45E-5, startDate = new DateTime(1985, 1, 2), launchTime = new DateTime(1988, 8, 7, 0, 0, 0),
nbrAvailable = 0, onsale = false, shopsAvailable = @"This field has
a newline", hexProductCode = 256, retailPrice = 78300M,
description = @"This field has quotes(""), and
two newlines
and a quoted ""string"""
},
new ProductData {
name = "dog house", weight = 45230990, startDate = new DateTime(2004, 2, 29), launchTime = default(DateTime),
nbrAvailable = 0, onsale = true, shopsAvailable = "", hexProductCode = 65296, retailPrice = 12008M,
description = null
}
};
// Act and Assert
AssertRead(testInput, metaDataNonamesNl, expected);
}
public void GoodFileCommaDelimitedUseFieldIndexForReadingDataCharUseOutputFormatForParsingUSEnglish()
{
// Arrange
CsvMetaData metaDataNamesUs = new CsvMetaData
{
SeparatorChar = ',',
IgnoreUnknownColumns = true,
UseOutputFormatForParsingCsvValue = true,
UseFieldIndexForReadingData = true,
FirstLineHasColumnNames = false,
EnforceCsvColumnAttribute = true, // default is false
FileCultureName = "en-US" // default is the current culture
};
string testInput =
"AAAAAAAA,__,34.184,05/23/08" + Environment.NewLine +
"BBBBBBBB,__,10.311,05/12/12" + Environment.NewLine +
"CCCCCCCC,__,12.000,12/23/08";
var expected = new[] {
new ProductDataSpecificFieldIndex()
{
name = "AAAAAAAA", weight = 34.184, startDate = new DateTime(2008, 5, 23),
},
new ProductDataSpecificFieldIndex {
name = "BBBBBBBB", weight = 10.311, startDate = new DateTime(2012, 5, 12),
},
new ProductDataSpecificFieldIndex {
name = "CCCCCCCC", weight = 12.000, startDate = new DateTime(2008, 12, 23),
}
};
// Act and Assert
AssertRead(testInput, metaDataNamesUs, expected);
}
public void GoodFileCommaDelimitedNamesInFirstLineNLnl()
{
// Arrange
List <ProductData> dataRows_Test = new List <ProductData>();
dataRows_Test.Add(new ProductData {
retailPrice = 4.59M, name = "Wooden toy", startDate = new DateTime(2008, 2, 1), nbrAvailable = 67
});
dataRows_Test.Add(new ProductData {
onsale = true, weight = 4.03, shopsAvailable = "Ashfield", description = ""
});
dataRows_Test.Add(new ProductData {
name = "Metal box", launchTime = new DateTime(2009, 11, 5, 4, 50, 0), description = "Great\nproduct"
});
CsvMetaData metaDataNamesNl2 = new CsvMetaData
{
SeparatorChar = ',',
FirstLineHasColumnNames = true,
EnforceCsvColumnAttribute = false,
TextEncoding = Encoding.Unicode,
FileCultureName = "nl-Nl" // default is the current culture
};
string expected =
@"name,startDate,launchTime,weight,shopsAvailable,code,price,onsale,description,nbrAvailable,unusedField
Wooden toy,1-2-2008,01 jan 00:00:00,""000,000"",,0,""€ 4,59"",False,,67,
,1-1-0001,01 jan 00:00:00,""004,030"",Ashfield,0,""€ 0,00"",True,"""",0,
Metal box,1-1-0001,05 nov 04:50:00,""000,000"",,0,""€ 0,00"",False,""Great
product"",0,
";
// Act and Assert
AssertWrite(dataRows_Test, metaDataNamesNl2, expected);
}
/// <summary>
/// Used to test the Read method.
/// </summary>
/// <typeparam name="T">
/// Type of the output elements.
/// </typeparam>
/// <param name="testInput">
/// String representing the contents of the file or StreamReader. This string is fed to the Read method
/// as though it came from a file or StreamReader.
/// </param>
/// <param name="metaData">
/// Passed to Read.
/// </param>
/// <returns>
/// Output of Read.
/// </returns>
public IEnumerable <T> TestRead <T>(string testInput, CsvMetaData metaData) where T : class, new()
{
CsvContext cc = new CsvContext();
return(cc.Read <T>(StreamReaderFromString(testInput), metaData));
}
/// <summary>
/// Executes a Write and tests whether it outputs the expected records.
/// </summary>
/// <typeparam name="T">
/// The type of the input elements.
/// </typeparam>
/// <param name="values">
/// The collection of input elements.
/// </param>
/// <param name="metaData">
/// Passed directly to write.
/// </param>
/// <param name="expected">
/// Expected output.
/// </param>
public void AssertWrite <T>(IEnumerable <T> values, CsvMetaData metaData, string expected) where T : class
{
string actual = TestWrite <T>(values, metaData);
Assert.Equal(Utils.NormalizeString(actual), Utils.NormalizeString(expected));
}
static void Main(string[] args)
{
// ------------------------------------
// Reading files, no erros
// The input files are meant to test the library code, so have lots of weird cases.
// ----
// Read comma delimited file with names in first line, US-English culture.
// Fields do not have to have CsvColumn attribute.
CsvContext cc = new CsvContext();
IEnumerable <ProductData_MissingFieldIndex> dataRows_namesUs = null;
IEnumerable <TestDataRow> dataRows_namesUsRaw = null;
CsvMetaData metaDataNamesUs = new CsvMetaData
{
SeparatorChar = ',', // default is ','
FirstLineHasColumnNames = true,
EnforceCsvColumnAttribute = false, // default is false
FileCultureName = "en-US" // default is the current culture
};
try
{
dataRows_namesUs =
cc.Read <ProductData>("TestFiles/goodfile_us.csv", metaDataNamesUs) as IEnumerable <ProductData_MissingFieldIndex>;
Utils.OutputData <ProductData>(dataRows_namesUs, "Good file, English US culture");
// -----------
// Manually change contents of file, to see whether file is read again with new values.
Utils.OutputData <ProductData>(dataRows_namesUs, "Good file, English US culture, second read");
// ------------
// Partial read - read just one record from the file
foreach (ProductData_MissingFieldIndex row in dataRows_namesUs)
{
break;
}
// -----------
// Read raw data rows
dataRows_namesUsRaw =
cc.Read <TestDataRow>("TestFiles/goodfile_us.csv", metaDataNamesUs) as IEnumerable <TestDataRow>;
Utils.OutputData <TestDataRow>(dataRows_namesUsRaw, "Good file, English US culture, Raw data rows");
}
catch (Exception e)
{
Utils.OutputException(e);
}
// ----
// Read file without names, Dutch culture, tab delimited.
// EnforceCsvColumnAttribute is not set, because it is implicitly true
// when there are no names in the first line. This is because only
// fields that have a FieldIndex can be used, which means having
// a CsvColumn attribute.
CsvMetaData metaDataNonamesNl = new CsvMetaData
{
SeparatorChar = '\t', // tab character
FirstLineHasColumnNames = false,
EnforceCsvColumnAttribute = true,
FileCultureName = "nl-NL" // default is the current culture
};
try
{
var dataRowsNonamesNl =
cc.Read <ProductData>("TestFiles/goodfile_nl.csv", metaDataNonamesNl);
Utils.OutputData <ProductData>(dataRowsNonamesNl, "Good file, Dutch culture");
}
catch (Exception e)
{
Utils.OutputException(e);
}
// ----
// Read a stream instead of a file, without names, Dutch culture, tab delimited.
// EnforceCsvColumnAttribute is not set, because it is implicitly true
// when there are no names in the first line. This is because only
// fields that have a FieldIndex can be used, which means having
// a CsvColumn attribute.
CsvMetaData metaDataNonamesNlStream = new CsvMetaData
{
SeparatorChar = '\t', // tab character
FirstLineHasColumnNames = false,
EnforceCsvColumnAttribute = true,
FileCultureName = "nl-NL" // default is the current culture
};
try
{
using (StreamReader sr =
new StreamReader("TestFiles/goodfile_nl.csv", Encoding.UTF8))
{
var dataRowsNonamesNl =
cc.Read <ProductData>(sr, metaDataNonamesNlStream);
Utils.OutputData(dataRowsNonamesNl, "Good file, Dutch culture, using stream");
}
}
catch (Exception e)
{
Utils.OutputException(e);
}
// ------------------------------------
// Reading files, with errors
// Type has duplicate FileIndices
try
{
IEnumerable <ProductData_DuplicateIndices> dataRows2 =
cc.Read <ProductData_DuplicateIndices>("TestFiles/goodfile_nl.csv", metaDataNonamesNl);
Utils.OutputData(dataRows2, "Good file, Dutch culture");
}
catch (Exception e)
{
Utils.OutputException(e);
}
// Type has required fields that do not have a FieldIndex.
try
{
IEnumerable <ProductData_MissingFieldIndex> dataRows2 =
cc.Read <ProductData_MissingFieldIndex>("TestFiles/goodfile_nl.csv", metaDataNonamesNl);
Utils.OutputData <ProductData_MissingFieldIndex>(dataRows2, "Good file, Dutch culture");
}
catch (Exception e)
{
Utils.OutputException(e);
}
// CsvFileDescription.EnforceCsvColumnAttribute is false, but needs to be true because
// CsvFileDescription.FirstLineHasColumnNames is false.
CsvMetaData metaDataBad = new CsvMetaData
{
SeparatorChar = '\t', // tab character
FirstLineHasColumnNames = false,
EnforceCsvColumnAttribute = false,
FileCultureName = "nl-NL" // default is the current culture
};
try
{
IEnumerable <ProductData> dataRows_nonamesNl_bad =
cc.Read <ProductData>("TestFiles/goodfile_nl.csv", metaDataBad);
Utils.OutputData <ProductData>(dataRows_nonamesNl_bad, "Good file, Dutch culture");
}
catch (Exception e)
{
Utils.OutputException(e);
}
// ----
// Read file with names, but one name not declared in type
try
{
IEnumerable <ProductData> dataRows_namesUs_3 =
cc.Read <ProductData>("TestFiles/badfile_unknownname.csv", metaDataNamesUs);
Utils.OutputData <ProductData>(dataRows_namesUs_3, "Bad file, English US culture, unknown name");
}
catch (Exception e)
{
Utils.OutputException(e);
}
// ----
// Read file with names, only columns with CsvColumn attribute participate.
// But one name matches a column without CsvColumn attribute.
var fileDescriptionNamesUsEnforceCsvColumn = new CsvMetaData
{
SeparatorChar = ',', // default is ','
FirstLineHasColumnNames = true,
EnforceCsvColumnAttribute = true, // default is false
FileCultureName = "en-US" // default is the current culture
};
try
{
var dataRowsNamesUs2 =
cc.Read <ProductData>("TestFiles/goodfile_us.csv", fileDescriptionNamesUsEnforceCsvColumn);
Utils.OutputData(dataRowsNamesUs2, "Good file, English US culture");
}
catch (Exception e)
{
Utils.OutputException(e);
}
// ----
// Various errors in data fields - all captured in AggregatedException
// * A row with too many fields
// * Rows with badly formatted data - letters in numeric fields, bad dates
CsvMetaData metaDataNonamesUs = new CsvMetaData
{
SeparatorChar = ',', // default is ','
FirstLineHasColumnNames = true,
EnforceCsvColumnAttribute = true,
FileCultureName = "en-US" // default is the current culture
};
try
{
var dataRowsNamesUsDataerrors =
cc.Read <ProductData>("TestFiles/badfile_us_dataerrors.csv", metaDataNonamesUs);
Utils.OutputData(
dataRowsNamesUsDataerrors,
"Bad file, English US culture, various data errors");
}
catch (Exception e)
{
Utils.OutputException(e);
}
// ------------------------------------
// Writing files
// ---------------
// Create own IEnumarable, rather then using one created by reading a file.
// Dutch, names in first line, don't limit writing to fields with CsvColumn attribute.
var dataRowsTest = new List <ProductData>();
dataRowsTest.Add(new ProductData {
retailPrice = 4.59M, name = "Wooden toy", startDate = DateTime.Parse("1/2/2008"), nbrAvailable = 67
});
dataRowsTest.Add(new ProductData {
onsale = true, weight = 4.03, shopsAvailable = "Ashfield", description = ""
});
dataRowsTest.Add(new ProductData {
name = "Metal box", launchTime = DateTime.Parse("5/11/2009 4:50"), description = "Great\nproduct"
});
var fileDescriptionNamesNl2 = new CsvMetaData
{
SeparatorChar = ',',
FirstLineHasColumnNames = true,
EnforceCsvColumnAttribute = false,
TextEncoding = Encoding.Unicode,
FileCultureName = "nl-Nl" // default is the current culture
};
try
{
cc.Write(
dataRowsTest,
"TestFiles/output_newdata_names_nl.csv",
fileDescriptionNamesNl2);
}
catch (Exception e)
{
Utils.OutputException(e);
}
// ---------------
// Write tab delimited file, US-English, no names in first line
// using FieldIndices provided in CsvColumn attributes
var fileDescription_nonamesUs_output = new CsvMetaData
{
SeparatorChar = '\t',
FirstLineHasColumnNames = false,
EnforceCsvColumnAttribute = true,
FileCultureName = "en-US" // default is the current culture
};
try
{
cc.Write <ProductData>(
dataRows_namesUs,
"TestFiles/output_nonames_us.csv",
fileDescription_nonamesUs_output);
}
catch (Exception e)
{
Utils.OutputException(e);
}
// ---------------
// Write comma delimited file, Dutch, names in first line using
// CsvColumn attributes
var fileDescriptionNamesNl = new CsvMetaData
{
SeparatorChar = ',',
FirstLineHasColumnNames = true,
EnforceCsvColumnAttribute = true,
TextEncoding = Encoding.Unicode,
FileCultureName = "nl-NL" // default is the current culture
};
try
{
cc.Write <ProductData>(
dataRows_namesUs,
"TestFiles/output_names_nl.csv",
fileDescriptionNamesNl);
}
catch (Exception e)
{
Utils.OutputException(e);
}
// Write comma delimited file, column names in first record,
// using anonymous type. Because there are no FieldIndices,
// the order of the fields on each line in the file is not guaranteed.
//
// FileCultureName is not set, so the current culture is used
// (so if you are Canadian, the system uses Canadian dates, etc.)
var fileDescriptionAnon = new CsvMetaData
{
SeparatorChar = ',',
FirstLineHasColumnNames = true,
EnforceCsvColumnAttribute = false
// use current culture
};
try
{
var query = from row in dataRows_namesUs
orderby row.weight
select new {
ProductName = row.name,
InShops = row.startDate,
Markup = row.retailPrice * (decimal)0.5
};
cc.Write(
query,
"TestFiles/output_anon.csv",
fileDescriptionAnon);
}
catch (Exception e)
{
Utils.OutputException(e);
}
// ------------------------------------
// Writing files, with errors
// If not writing field names to the first line, then you have to only
// use fields with the CsvColumn attribute, and give them all a FieldIndex.
// Otherwise, there is no reliable way to read back the data, because the order
// of the fields is not guaranteed.
try
{
cc.Write <ProductData>(
dataRows_namesUs,
"TestFiles/output_bad.csv",
metaDataBad);
}
catch (Exception e)
{
Utils.OutputException(e);
}
// CsvFileDescription settings are good, but not all fields with CsvColumn attribute
// have a FieldIndex.
CsvMetaData metaDataNonamesNl2 = new CsvMetaData
{
SeparatorChar = ',',
FirstLineHasColumnNames = false,
EnforceCsvColumnAttribute = true,
TextEncoding = Encoding.Unicode,
FileCultureName = "nl-NL" // default is the current culture
};
try
{
List <ProductData_MissingFieldIndex> emptyData = new List <ProductData_MissingFieldIndex>();
cc.Write <ProductData_MissingFieldIndex>(
emptyData,
"TestFiles/output_bad.csv",
metaDataNonamesNl2);
}
catch (Exception e)
{
Utils.OutputException(e);
}
}
