public void TestRead_WithSeparatedRecordFilter_SkipsRecordsMatchingCriteria()
{
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
const string text = @"123,Bob Smith,4/21/2017
This is not a real record
234,Jay Smith,5/21/2017";
StringReader stringReader = new StringReader(text);
var parser = new SeparatedValueReader(stringReader, schema);
parser.RecordRead += (sender, e) =>
{
e.IsSkipped = e.Values.Length < 3;
};
Assert.IsTrue(parser.Read(), "Could not read the first record.");
object[] actual1 = parser.GetValues();
CollectionAssert.AreEqual(new object[] { 123, "Bob Smith", new DateTime(2017, 04, 21) }, actual1);
Assert.IsTrue(parser.Read(), "Could not read the second record.");
object[] actual2 = parser.GetValues();
CollectionAssert.AreEqual(new object[] { 234, "Jay Smith", new DateTime(2017, 05, 21) }, actual2);
Assert.IsFalse(parser.Read(), "There should not be any more records.");
}
public void TestGetValues_BlankLeadingSection_ReturnsNull()
{
using (MemoryStream stream = new MemoryStream())
{
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new StringColumn("leading"))
.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy"
});
object[] sources = new object[] { "", 123, "Bob", new DateTime(2013, 1, 19) };
using (SeparatedValueWriter builder = new SeparatedValueWriter(stream, schema, options))
{
builder.Write(sources);
}
stream.Position = 0;
SeparatedValueReader parser = new SeparatedValueReader(stream, schema, options);
Assert.IsTrue(parser.Read(), "No records were found.");
object[] values = parser.GetValues();
Assert.AreEqual(schema.ColumnDefinitions.Count, values.Length, "The wrong number of values were read.");
Assert.AreEqual(null, values[0], "The first column was not interpreted as null.");
Assert.AreEqual(sources[1], values[1], "The second column was not parsed correctly.");
Assert.AreEqual(sources[2], values[2], "The third column was not parsed correctly.");
Assert.AreEqual(sources[3], values[3], "The forth column was not parsed correctly.");
Assert.IsFalse(parser.Read(), "Too many records were found.");
}
}
public void TestRead_WithSeparatedRecordFilter_SkipsRecordsMatchingCriteria()
{
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
SeparatedValueOptions options = new SeparatedValueOptions()
{
PartitionedRecordFilter = (record) => record.Length < 3
};
const string text = @"123,Bob Smith,4/21/2017
This is not a real record
234,Jay Smith,5/21/2017";
StringReader stringReader = new StringReader(text);
IReader parser = new SeparatedValueReader(stringReader, schema, options);
Assert.True(parser.Read(), "Could not read the first record.");
object[] actual1 = parser.GetValues();
Assert.Equal(new object[] { 123, "Bob Smith", new DateTime(2017, 04, 21) }, actual1);
Assert.True(parser.Read(), "Could not read the second record.");
object[] actual2 = parser.GetValues();
Assert.Equal(new object[] { 234, "Jay Smith", new DateTime(2017, 05, 21) }, actual2);
Assert.False(parser.Read(), "There should not be any more records.");
}
public void TestGetValues_BlankLeadingSection_ReturnsNull()
{
using (MemoryStream stream = new MemoryStream())
{
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new StringColumn("leading"))
.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy"
});
object[] sources = new object[] { "", 123, "Bob", new DateTime(2013, 1, 19) };
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter builder = new SeparatedValueWriter(stringWriter, schema, options);
builder.Write(sources);
StringReader stringReader = new StringReader(stringWriter.ToString());
SeparatedValueReader parser = new SeparatedValueReader(stringReader, schema, options);
Assert.True(parser.Read(), "No records were found.");
object[] values = parser.GetValues();
Assert.Equal(schema.ColumnDefinitions.Count, values.Length);
Assert.Equal(null, values[0]);
Assert.Equal(sources[1], values[1]);
Assert.Equal(sources[2], values[2]);
Assert.Equal(sources[3], values[3]);
Assert.False(parser.Read(), "Too many records were found.");
}
}
public void ShouldWriteSchemaIfExplicit()
{
StringWriter stringWriter = new StringWriter();
// Explicitly indicate that the first record is NOT the schema
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new StringColumn("Col1"));
SeparatedValueWriter writer = new SeparatedValueWriter(stringWriter, schema, new SeparatedValueOptions()
{
IsFirstRecordSchema = false
});
writer.WriteSchema(); // Explicitly write the schema
writer.Write(new string[] { "a" });
StringReader stringReader = new StringReader(stringWriter.ToString());
var reader = new SeparatedValueReader(stringReader, new SeparatedValueOptions()
{
IsFirstRecordSchema = true
});
var parsedSchema = reader.GetSchema();
Assert.Equal(schema.ColumnDefinitions.Count, parsedSchema.ColumnDefinitions.Count);
Assert.Equal(schema.ColumnDefinitions[0].ColumnName, parsedSchema.ColumnDefinitions[0].ColumnName);
Assert.True(reader.Read(), "The record was not retrieved after the schema.");
Assert.False(reader.Read(), "Encountered more than the expected number of records.");
}
public void TestReader_ReadThreeTypes()
{
StringWriter stringWriter = new StringWriter();
var injector = getSchemaInjector();
var writer = new SeparatedValueWriter(stringWriter, injector);
writer.Write(new object[] { "First Batch", 2 });
writer.Write(new object[] { 1, "Bob Smith", new DateTime(2018, 06, 04), 12.34m });
writer.Write(new object[] { 2, "Jane Doe", new DateTime(2018, 06, 05), 34.56m });
writer.Write(new object[] { 46.9m, 23.45m, true });
string output = stringWriter.ToString();
Assert.AreEqual(@"First Batch,2
1,Bob Smith,20180604,12.34
2,Jane Doe,20180605,34.56
46.9,23.45,True
", output);
var stringReader = new StringReader(output);
var selector = getSchemaSelector();
var reader = new SeparatedValueReader(stringReader, selector);
Assert.IsTrue(reader.Read(), "The header record could not be read.");
var headerValues = reader.GetValues();
Assert.AreEqual(2, headerValues.Length);
Assert.AreEqual("First Batch", headerValues[0]);
Assert.AreEqual(2, headerValues[1]);
Assert.IsTrue(reader.Read(), "The first data record could not be read.");
var dataValues1 = reader.GetValues();
Assert.AreEqual(4, dataValues1.Length);
Assert.AreEqual(1, dataValues1[0]);
Assert.AreEqual("Bob Smith", dataValues1[1]);
Assert.AreEqual(new DateTime(2018, 6, 4), dataValues1[2]);
Assert.AreEqual(12.34m, dataValues1[3]);
Assert.IsTrue(reader.Read(), "The second data record could not be read.");
var dataValues2 = reader.GetValues();
Assert.AreEqual(4, dataValues2.Length);
Assert.AreEqual(2, dataValues2[0]);
Assert.AreEqual("Jane Doe", dataValues2[1]);
Assert.AreEqual(new DateTime(2018, 6, 5), dataValues2[2]);
Assert.AreEqual(34.56m, dataValues2[3]);
Assert.IsTrue(reader.Read(), "The footer record could not be read.");
var footerValues = reader.GetValues();
Assert.AreEqual(3, footerValues.Length);
Assert.AreEqual(46.9m, footerValues[0]);
Assert.AreEqual(23.45m, footerValues[1]);
Assert.AreEqual(true, footerValues[2]);
Assert.IsFalse(reader.Read());
}
private static object[] parseValues(string content)
{
StringReader stringReader = new StringReader(content);
var schema = getSchema();
SeparatedValueReader reader = new SeparatedValueReader(stringReader, schema);
Assert.True(reader.Read(), "The record could not be read.");
object[] values = reader.GetValues();
Assert.False(reader.Read(), "Too many records were read.");
return values;
}
private void assertRecords(object[][] expected, SeparatedValueReader reader)
{
for (int recordIndex = 0; recordIndex != expected.Length; ++recordIndex)
{
Assert.True(reader.Read(), String.Format("The record could not be read (Record {0}).", recordIndex));
object[] actualValues = reader.GetValues();
object[] expectedValues = expected[recordIndex];
assertRecord(expectedValues, actualValues, recordIndex);
}
Assert.False(reader.Read(), "There were more records read than expected.");
}
public void TestRead_ValuesAfterEndOfFile_Throws()
{
string text = "a,b,c";
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)));
bool canRead = parser.Read();
Assert.IsTrue(canRead, "Could not read the record.");
canRead = parser.Read();
Assert.IsFalse(canRead, "We should have reached the end of the file.");
parser.GetValues();
}
private static object[] parseValues(string content)
{
StringReader stringReader = new StringReader(content);
var schema = getSchema();
SeparatedValueReader reader = new SeparatedValueReader(stringReader, schema);
Assert.True(reader.Read(), "The record could not be read.");
object[] values = reader.GetValues();
Assert.False(reader.Read(), "Too many records were read.");
return(values);
}
public void TestRead_ValuesAfterEndOfFile_Throws()
{
string text = "a,b,c";
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader);
bool canRead = parser.Read();
Assert.True(canRead, "Could not read the record.");
canRead = parser.Read();
Assert.False(canRead, "We should have reached the end of the file.");
Assert.Throws <InvalidOperationException>(() => parser.GetValues());
}
public void TestRead_SingleRecord_ReturnsTrueOnce()
{
const string text = "a,b,c";
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)));
bool canRead = parser.Read();
Assert.IsTrue(canRead, "Could not read the record.");
string[] expected = new string[] { "a", "b", "c" };
object[] actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed.");
canRead = parser.Read();
Assert.IsFalse(canRead, "No more records should have been read.");
}
private static object[] parseValues(string content)
{
byte[] encoded = Encoding.Default.GetBytes(content);
var schema = getSchema();
using (MemoryStream stream = new MemoryStream(encoded))
using (SeparatedValueReader reader = new SeparatedValueReader(stream, schema))
{
Assert.IsTrue(reader.Read(), "The record could not be read.");
object[] values = reader.GetValues();
Assert.IsFalse(reader.Read(), "Too many records were read.");
return(values);
}
}
public void TestRead_SingleRecord_ReturnsTrueOnce()
{
const string text = "a,b,c";
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader);
bool canRead = parser.Read();
Assert.True(canRead, "Could not read the record.");
object[] expected = new object[] { "a", "b", "c" };
object[] actual = parser.GetValues();
Assert.Equal(expected, actual);
canRead = parser.Read();
Assert.False(canRead, "No more records should have been read.");
}
public IEnumerable <T> GetData(string filePath, string separator = ";")
{
using (StreamReader streamReader = new StreamReader(filePath))
{
SeparatedValueOptions options = new SeparatedValueOptions {
IsFirstRecordSchema = true, Separator = separator, Quote = '\''
};
var separatedValueSchema = new SeparatedValueSchema();
var properties = Type.GetProperties();
foreach (var property in properties)
{
var propertyName = property.Name;
var columnDefinition = GetColumnDefinition(property.PropertyType, propertyName);
separatedValueSchema.AddColumn(columnDefinition);
}
SeparatedValueReader separatedValueReader = new SeparatedValueReader(streamReader, separatedValueSchema, options);
List <T> values = new List <T>();
while (separatedValueReader.Read())
{
var rowValues = separatedValueReader.GetValues();
var tRow = new T();
var propertiesObjectValues = properties.Zip(rowValues, (a, b) => new { PropType = a, PropValue = b });
foreach (var propertiesObjectValue in propertiesObjectValues)
{
Type.GetProperty(propertiesObjectValue.PropType.Name).SetValue(tRow, propertiesObjectValue.PropValue);
}
values.Add(tRow);
}
return(values);
}
}
public void TestRead_RecordWithCP1251Characters_ReturnsCorrectCharacters()
{
//---- Arrange -----------------------------------------------------
// Need to convert the string to target encoding because otherwise a string declared in VS will always be encoded as UTF-8
var text = Encoding.Convert(Encoding.UTF8, Encoding.GetEncoding(1251), Encoding.UTF8.GetBytes(@"123;Лучиано;1/17/2014"));
var schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id")).AddColumn(new StringColumn("name")).AddColumn(new DateTimeColumn("created"));
var options = new SeparatedValueOptions
{
IsFirstRecordSchema = false,
Separator = ";",
Encoding = Encoding.GetEncoding(1251)
};
var testee = new SeparatedValueReader(new MemoryStream(text), schema, options);
//---- Act ---------------------------------------------------------
var result = testee.Read();
//---- Assert ------------------------------------------------------
Assert.IsTrue(result, "Could not read the record.");
object[] expected = { 123, "Лучиано", new DateTime(2014, 1, 17) };
object[] actual = testee.GetValues();
CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed.");
}
public void TestRead_ZeroLengthColumn()
{
//---- Arrange -----------------------------------------------------
var text = "104\t20\t1000\t00\tLausanne\tLausanne\tVD\t2\t\t0\t130\t5586\t19880301";
var options = new SeparatedValueOptions {
IsFirstRecordSchema = false, Separator = "\t"
};
var schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("OnrpId"))
.AddColumn(new Int32Column("Type"))
.AddColumn(new Int32Column("ZipCode"))
.AddColumn(new StringColumn("ZipCodeAddOn"))
.AddColumn(new StringColumn("TownShortName"))
.AddColumn(new StringColumn("TownOfficialName"))
.AddColumn(new StringColumn("CantonAbbreviation"))
.AddColumn(new Int16Column("MainLanguageCode"))
.AddColumn(new Int16Column("OtherLanguageCode"))
.AddColumn(new ByteColumn("HasSortfileData"))
.AddColumn(new Int32Column("LetterServiceOnrpId"))
.AddColumn(new Int32Column("MunicipalityId"))
.AddColumn(new StringColumn("ValidFrom"));
var testee = new SeparatedValueReader(new MemoryStream(Encoding.GetEncoding(1252).GetBytes(text)), options);
//---- Act ---------------------------------------------------------
var result = testee.Read();
//---- Assert ------------------------------------------------------
Assert.IsTrue(result);
Assert.AreEqual(schema.ColumnDefinitions.Count, testee.GetValues().Count());
}
public void TestReader_UnknownType()
{
var stringReader = new StringReader("What's this weird thing?");
var selector = getSchemaSelector();
var reader = new SeparatedValueReader(stringReader, selector);
reader.Read();
}
public void TestReader_creativyst_example()
{
const string text = @"John,Doe,120 jefferson st.,Riverside, NJ, 08075
Jack,McGinnis,220 hobo Av.,Phila, PA,09119
""John """"Da Man"""""",Repici,120 Jefferson St.,Riverside, NJ,08075
Stephen,Tyler,""7452 Terrace """"At the Plaza"""" road"",SomeTown,SD, 91234
,Blankman,,SomeTown, SD, 00298
""Joan """"the bone"""", Anne"",Jet,""9th, at Terrace plc"",Desert City, CO,00123
";
StringReader stringReader = new StringReader(text);
SeparatedValueReader reader = new SeparatedValueReader(stringReader);
Assert.True(reader.Read(), "Could not read the first record.");
assertValues(reader, "John", "Doe", "120 jefferson st.", "Riverside", "NJ", "08075");
Assert.True(reader.Read(), "Could not read the second record.");
assertValues(reader, "Jack", "McGinnis", "220 hobo Av.", "Phila", "PA", "09119");
Assert.True(reader.Read(), "Could not read the third record.");
assertValues(reader, "John \"Da Man\"", "Repici", "120 Jefferson St.", "Riverside", "NJ", "08075");
Assert.True(reader.Read(), "Could not read the fourth record.");
assertValues(reader, "Stephen", "Tyler", "7452 Terrace \"At the Plaza\" road", "SomeTown", "SD", "91234");
Assert.True(reader.Read(), "Could not read the fifth record.");
assertValues(reader, "", "Blankman", "", "SomeTown", "SD", "00298");
Assert.True(reader.Read(), "Could not read the sixth record.");
assertValues(reader, "Joan \"the bone\", Anne", "Jet", "9th, at Terrace plc", "Desert City", "CO", "00123");
Assert.False(reader.Read(), "Read too many records.");
}
public void TestReader_UnknownType_IgnoreUnknown_SkipsRecord()
{
var stringReader = new StringReader("What's this weird thing?");
var selector = getSchemaSelector();
var reader = new SeparatedValueReader(stringReader, selector);
reader.RecordError += (o, e) => e.IsHandled = true;
Assert.IsFalse(reader.Read());
}
public bool Read()
{
if (!reader.Read())
{
return(false);
}
object[] values = reader.GetValues();
Current = selector.Reader(values);
return(true);
}
public void TestReader_BadDecimal_ThrowsException()
{
const string data = "bad";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new DecimalColumn("value"));
SeparatedValueReader reader = new SeparatedValueReader(new StringReader(data), schema);
Assert.ThrowsException <RecordProcessingException>(() => reader.Read());
}
public void TestRead_InvalidConversion_Throws()
{
const string text = "a";
StringReader stringReader = new StringReader(text);
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("First"));
SeparatedValueReader parser = new SeparatedValueReader(stringReader, schema);
Assert.Throws <RecordProcessingException>(() => parser.Read());
}
public void RunFlatFiles_NoSchema()
{
var reader = new StringReader(data);
var csvReader = new SeparatedValueReader(reader);
var people = new List <object[]>();
while (csvReader.Read())
{
people.Add(csvReader.GetValues());
}
}
public void ShouldNotFindRecordsInEmptyFile()
{
string source = String.Empty;
StringReader stringReader = new StringReader(source);
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = false
};
SeparatedValueReader reader = new SeparatedValueReader(stringReader, options);
Assert.False(reader.Read(), "No records should be read from an empty file.");
}
public void TestGetSchema_SchemaProvided_WrongNumberOfColumns_Throws()
{
const string text = @"123,Bob";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema);
parser.Read();
}
public void TestGetSchema_FirstRecordSchema_WrongNumberOfColumns_Throws()
{
const string text = @"id,name,created
123,Bob,1/19/2013,Hello";
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)), options);
parser.Read();
}
public void TestReadWrite_Comments()
{
StringWriter output = new StringWriter();
SeparatedValueWriter writer = new SeparatedValueWriter(output);
writer.Write(new[] { "a", "b", "c" });
writer.WriteRaw("# Hello, world!!!", true);
writer.Write(new[] { "d", "e", "f" });
StringReader input = new StringReader(output.ToString());
SeparatedValueReader reader = new SeparatedValueReader(input);
reader.RecordRead += (sender, e) =>
{
e.IsSkipped = e.Values.Length > 0 && e.Values[0].StartsWith("#");
};
Assert.IsTrue(reader.Read());
CollectionAssert.AreEqual(new[] { "a", "b", "c" }, reader.GetValues());
Assert.IsTrue(reader.Read());
CollectionAssert.AreEqual(new[] { "d", "e", "f" }, reader.GetValues());
Assert.IsFalse(reader.Read());
}
public void TestGetSchema_FirstRecordSchema_TooFewColumns_Throws()
{
const string text = @"id,name,created
123,Bob";
StringReader stringReader = new StringReader(text);
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
SeparatedValueReader parser = new SeparatedValueReader(stringReader, options);
Assert.Throws <FlatFileException>(() => parser.Read());
}
private static object[] parseValues(string content)
{
byte[] encoded = Encoding.Default.GetBytes(content);
var schema = getSchema();
using (MemoryStream stream = new MemoryStream(encoded))
using (SeparatedValueReader reader = new SeparatedValueReader(stream, schema))
{
Assert.IsTrue(reader.Read(), "The record could not be read.");
object[] values = reader.GetValues();
Assert.IsFalse(reader.Read(), "Too many records were read.");
return values;
}
}
public void TestRead_MultipleCallsToValues_ReturnsSameValues()
{
string text = "a,b,c";
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)));
bool canRead = parser.Read();
Assert.IsTrue(canRead, "Could not read the record.");
string[] expected = new string[] { "a", "b", "c" };
object[] actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed.");
actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The same values were not returned multiple times.");
}
public void ShouldThrowSyntaxExceptionIfQuoteFollowedByNonSeparator()
{
string source = "'a'b";
StringReader stringReader = new StringReader(source);
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = false,
Quote = '\''
};
SeparatedValueReader reader = new SeparatedValueReader(stringReader, options);
Assert.Throws <FlatFileException>(() => reader.Read());
}
public void TestRead_MultipleCallsToValues_ReturnsSameValues()
{
string text = "a,b,c";
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)));
bool canRead = parser.Read();
Assert.IsTrue(canRead, "Could not read the record.");
string[] expected = new string[] { "a", "b", "c" };
object[] actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed.");
actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The same values were not returned multiple times.");
}
public void TestRead_ValuesAfterEndOfFile_Throws()
{
string text = "a,b,c";
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader);
bool canRead = parser.Read();
Assert.True(canRead, "Could not read the record.");
canRead = parser.Read();
Assert.False(canRead, "We should have reached the end of the file.");
Assert.Throws<InvalidOperationException>(() => parser.GetValues());
}
public void TestRead_ZeroLengthColumn()
{
//---- Arrange -----------------------------------------------------
var text = "104\t20\t1000\t00\tLausanne\tLausanne\tVD\t2\t\t0\t130\t5586\t19880301";
var options = new SeparatedValueOptions { IsFirstRecordSchema = false, Separator = "\t" };
var schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("OnrpId"))
.AddColumn(new Int32Column("Type"))
.AddColumn(new Int32Column("ZipCode"))
.AddColumn(new StringColumn("ZipCodeAddOn"))
.AddColumn(new StringColumn("TownShortName"))
.AddColumn(new StringColumn("TownOfficialName"))
.AddColumn(new StringColumn("CantonAbbreviation"))
.AddColumn(new Int16Column("MainLanguageCode"))
.AddColumn(new Int16Column("OtherLanguageCode"))
.AddColumn(new ByteColumn("HasSortfileData"))
.AddColumn(new Int32Column("LetterServiceOnrpId"))
.AddColumn(new Int32Column("MunicipalityId"))
.AddColumn(new StringColumn("ValidFrom"));
StringReader stringReader = new StringReader(text);
var testee = new SeparatedValueReader(stringReader, options);
//---- Act ---------------------------------------------------------
var result = testee.Read();
//---- Assert ------------------------------------------------------
Assert.True(result);
Assert.Equal(schema.ColumnDefinitions.Count, testee.GetValues().Count());
}
public void TestGetSchema_SchemaProvided_FirstRecordSchema_SkipsFirstRecord()
{
const string text = @"id,name,created";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
IReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema, options);
ISchema actual = parser.GetSchema();
Assert.AreSame(schema, actual, "The schema was passed did not take priority.");
Assert.IsFalse(parser.Read(), "The schema record was not skipped.");
}
public void TestRead_EmbeddedQuote_ParsesCorrectly()
{
var text = @"123;Todd's Bait Shop;1/17/2014";
var schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"));
schema.AddColumn(new StringColumn("name"));
schema.AddColumn(new DateTimeColumn("created"));
var options = new SeparatedValueOptions
{
IsFirstRecordSchema = false,
Separator = ";"
};
StringReader stringReader = new StringReader(text);
var reader = new SeparatedValueReader(stringReader, schema, options);
var result = reader.Read();
Assert.True(result, "Could not read the record.");
object[] expected = { 123, "Todd's Bait Shop", new DateTime(2014, 1, 17) };
object[] actual = reader.GetValues();
Assert.Equal(expected, actual);
}
public void TestRead_MultipleCallsToValues_ReturnsSameValues()
{
string text = "a,b,c";
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader);
bool canRead = parser.Read();
Assert.True(canRead, "Could not read the record.");
object[] expected = new object[] { "a", "b", "c" };
object[] actual = parser.GetValues();
Assert.Equal(expected, actual);
actual = parser.GetValues();
Assert.Equal(expected, actual);
}
public void TestRead_SkipRecord_NoParsingError()
{
const string text = "a,b,c";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("A"));
schema.AddColumn(new DateTimeColumn("B"));
schema.AddColumn(new GuidColumn("C"));
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader, schema);
bool canRead = parser.Skip();
Assert.True(canRead, "Could not skip the record.");
canRead = parser.Read();
Assert.False(canRead, "No more records should have been read.");
}
public void TestRead_ValuesAfterEndOfFile_Throws()
{
string text = "a,b,c";
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)));
bool canRead = parser.Read();
Assert.IsTrue(canRead, "Could not read the record.");
canRead = parser.Read();
Assert.IsFalse(canRead, "We should have reached the end of the file.");
parser.GetValues();
}
public void TestReader_creativyst_example()
{
const string text = @"John,Doe,120 jefferson st.,Riverside, NJ, 08075
Jack,McGinnis,220 hobo Av.,Phila, PA,09119
""John """"Da Man"""""",Repici,120 Jefferson St.,Riverside, NJ,08075
Stephen,Tyler,""7452 Terrace """"At the Plaza"""" road"",SomeTown,SD, 91234
,Blankman,,SomeTown, SD, 00298
""Joan """"the bone"""", Anne"",Jet,""9th, at Terrace plc"",Desert City, CO,00123
";
StringReader stringReader = new StringReader(text);
SeparatedValueReader reader = new SeparatedValueReader(stringReader);
Assert.True(reader.Read(), "Could not read the first record.");
assertValues(reader, "John", "Doe", "120 jefferson st.", "Riverside", "NJ", "08075");
Assert.True(reader.Read(), "Could not read the second record.");
assertValues(reader, "Jack", "McGinnis", "220 hobo Av.", "Phila", "PA", "09119");
Assert.True(reader.Read(), "Could not read the third record.");
assertValues(reader, "John \"Da Man\"", "Repici", "120 Jefferson St.", "Riverside", "NJ", "08075");
Assert.True(reader.Read(), "Could not read the fourth record.");
assertValues(reader, "Stephen", "Tyler", "7452 Terrace \"At the Plaza\" road", "SomeTown", "SD", "91234");
Assert.True(reader.Read(), "Could not read the fifth record.");
assertValues(reader, "", "Blankman","", "SomeTown", "SD", "00298");
Assert.True(reader.Read(), "Could not read the sixth record.");
assertValues(reader, "Joan \"the bone\", Anne", "Jet", "9th, at Terrace plc", "Desert City", "CO", "00123");
Assert.False(reader.Read(), "Read too many records.");
}
public void TestGetSchema_FirstRecordSchema_TooManyColumns_IgnoresTrailing()
{
const string text = @"id,name,created
123,Bob,1/19/2013,Hello";
StringReader stringReader = new StringReader(text);
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
SeparatedValueReader parser = new SeparatedValueReader(stringReader, options);
Assert.True(parser.Read(), "The record could not be read.");
Assert.Equal(parser.GetSchema().ColumnDefinitions.Count, parser.GetValues().Length);
;
}
public void TestGetSchema_SchemaProvided_WrongNumberOfColumns_Throws()
{
const string text = @"123,Bob";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader, schema);
Assert.Throws<FlatFileException>(() => parser.Read());
}
public void TestGetSchema_SchemaProvided_ParsesValues_Quoted()
{
const string text = "123,\"Bob\",1/19/2013";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader, schema);
Assert.True(parser.Read(), "The first record was skipped.");
object[] actual = parser.GetValues();
object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
Assert.Equal(expected, actual);
}
public void TestGetValues_BlankTrailingSection_ReturnsNull()
{
using (MemoryStream stream = new MemoryStream())
{
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" })
.AddColumn(new StringColumn("trailing"));
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19), "" };
using (SeparatedValueWriter builder = new SeparatedValueWriter(stream, schema, options))
{
builder.Write(sources);
}
stream.Position = 0;
SeparatedValueReader parser = new SeparatedValueReader(stream, schema, options);
Assert.IsTrue(parser.Read(), "No records were found.");
object[] values = parser.GetValues();
Assert.AreEqual(schema.ColumnDefinitions.Count, values.Length, "The wrong number of values were read.");
Assert.AreEqual(sources[0], values[0], "The first column was not parsed correctly.");
Assert.AreEqual(sources[1], values[1], "The second column was not parsed correctly.");
Assert.AreEqual(sources[2], values[2], "The third column was not parsed correctly.");
Assert.AreEqual(null, values[3], "The forth column was not interpreted as null.");
Assert.IsFalse(parser.Read(), "Too many records were found.");
}
}
public void TestRead_SingleRecord_ReturnsTrueOnce()
{
const string text = "a,b,c";
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)));
bool canRead = parser.Read();
Assert.IsTrue(canRead, "Could not read the record.");
string[] expected = new string[] { "a", "b", "c" };
object[] actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed.");
canRead = parser.Read();
Assert.IsFalse(canRead, "No more records should have been read.");
}
public void TestRead_SingleRecord_ReturnsTrueOnce()
{
const string text = "a,b,c";
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader);
bool canRead = parser.Read();
Assert.True(canRead, "Could not read the record.");
object[] expected = new object[] { "a", "b", "c" };
object[] actual = parser.GetValues();
Assert.Equal(expected, actual);
canRead = parser.Read();
Assert.False(canRead, "No more records should have been read.");
}
public void TestGetSchema_FirstRecordSchema_TooFewColumns_Throws()
{
const string text = @"id,name,created
123,Bob";
StringReader stringReader = new StringReader(text);
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
SeparatedValueReader parser = new SeparatedValueReader(stringReader, options);
Assert.Throws<FlatFileException>(() => parser.Read());
}
public void TestGetSchema_SchemaProvided_WrongNumberOfColumns_Throws()
{
const string text = @"123,Bob";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema);
parser.Read();
}
public void TestRead_RecordWithCP1252Characters_ReturnsCorrectCharacters()
{
//---- Arrange -----------------------------------------------------
// Need to convert the string to target encoding because otherwise a string declared in VS will always be encoded as UTF-8
var text = Encoding.Convert(Encoding.UTF8, Encoding.GetEncoding(1252), Encoding.UTF8.GetBytes(@"123;Müller;1/17/2014"));
var schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id")).AddColumn(new StringColumn("name")).AddColumn(new DateTimeColumn("created"));
var options = new SeparatedValueOptions
{
IsFirstRecordSchema = false,
Separator = ";" ,
Encoding = Encoding.GetEncoding(1252)
};
var testee = new SeparatedValueReader(new MemoryStream(text), schema, options);
//---- Act ---------------------------------------------------------
var result = testee.Read();
//---- Assert ------------------------------------------------------
Assert.IsTrue(result, "Could not read the record.");
object[] expected = { 123, "Müller", new DateTime(2014, 1, 17) };
object[] actual = testee.GetValues();
CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed.");
}
public void TestGetSchema_SchemaProvided_ParsesValues()
{
const string text = @"123,Bob,1/19/2013";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema);
Assert.IsTrue(parser.Read(), "The first record was skipped.");
object[] actual = parser.GetValues();
object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
CollectionAssert.AreEqual(expected, actual, "The values were not parsed as expected.");
}
public void TestGetSchema_SchemaProvided_FirstRecordSchema_SkipsFirstRecord()
{
const string text = @"id,name,created";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
StringReader stringReader = new StringReader(text);
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
IReader parser = new SeparatedValueReader(stringReader, schema, options);
ISchema actual = parser.GetSchema();
Assert.Same(schema, actual);
Assert.False(parser.Read(), "The schema record was not skipped.");
}
public void TestGetSchema_FirstRecordSchema_WrongNumberOfColumns_Throws()
{
const string text = @"id,name,created
123,Bob,1/19/2013,Hello";
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)), options);
parser.Read();
}
public void TestGetValues_BlankTrailingSection_ReturnsNull()
{
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" })
.AddColumn(new StringColumn("trailing"));
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19), "" };
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter builder = new SeparatedValueWriter(stringWriter, schema, options);
builder.Write(sources);
StringReader stringReader = new StringReader(stringWriter.ToString());
SeparatedValueReader parser = new SeparatedValueReader(stringReader, schema, options);
Assert.True(parser.Read(), "No records were found.");
object[] values = parser.GetValues();
Assert.Equal(schema.ColumnDefinitions.Count, values.Length);
Assert.Equal(sources[0], values[0]);
Assert.Equal(sources[1], values[1]);
Assert.Equal(sources[2], values[2]);
Assert.Equal(null, values[3]);
Assert.False(parser.Read(), "Too many records were found.");
}
