public void TestReadFlatFile_ClearsSchemaAndData()
{
// fill the table with some existing columns, constraints and data
DataTable table = new DataTable();
DataColumn column = table.Columns.Add("blah", typeof(int));
table.Columns.Add("bleh", typeof(string));
table.Constraints.Add("PK_blah", column, true);
DataRow row = table.Rows.Add(new object[] { 123, "dopey" });
row.AcceptChanges();
const string text = @"id,name,created,avg
123,Bob,12/31/2012,3.14159";
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
StringReader stringReader = new StringReader(text);
IReader parser = new SeparatedValueReader(stringReader, options);
table.ReadFlatFile(parser);
Assert.Equal(4, table.Columns.Count);
Assert.True(table.Columns.Contains("id"), "The ID column was not extracted.");
Assert.True(table.Columns.Contains("name"), "The name column was not extracted.");
Assert.True(table.Columns.Contains("created"), "The created column was not extracted.");
Assert.True(table.Columns.Contains("avg"), "The AVG column was not extracted.");
Assert.Equal(1, table.Rows.Count);
row = table.Rows[0];
object[] expected = new object[] { "123", "Bob", "12/31/2012", "3.14159" };
object[] values = row.ItemArray;
Assert.Equal(expected, values);
}
public void TestReadFlatFile_DataTableNull_Throws()
{
DataTable table = null;
StringReader stringReader = new StringReader(String.Empty);
IReader parser = new SeparatedValueReader(stringReader);
Assert.Throws<ArgumentNullException>(() => DataTableExtensions.ReadFlatFile(table, parser));
}
private static object[] parseValues(string content)
{
StringReader stringReader = new StringReader(content);
var schema = getSchema();
SeparatedValueReader reader = new SeparatedValueReader(stringReader, schema);
Assert.True(reader.Read(), "The record could not be read.");
object[] values = reader.GetValues();
Assert.False(reader.Read(), "Too many records were read.");
return values;
}
public void TestGetSchema_Extracted_ReturnsColumnNames()
{
string text = "a,b,c";
StringReader stringReader = new StringReader(text);
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
IReader parser = new SeparatedValueReader(stringReader, options);
ISchema schema = parser.GetSchema();
Assert.True(schema.ColumnDefinitions.All(d => d is StringColumn), "Not all of the columns were treated as strings.");
string[] actual = schema.ColumnDefinitions.Select(d => d.ColumnName).ToArray();
string[] expected = new string[] { "a", "b", "c" };
Assert.Equal(expected, actual);
}
private static object[] parseValues(string content)
{
byte[] encoded = Encoding.Default.GetBytes(content);
var schema = getSchema();
using (MemoryStream stream = new MemoryStream(encoded))
using (SeparatedValueReader reader = new SeparatedValueReader(stream, schema))
{
Assert.IsTrue(reader.Read(), "The record could not be read.");
object[] values = reader.GetValues();
Assert.IsFalse(reader.Read(), "Too many records were read.");
return values;
}
}
public void TestReadFlatFile_ColumnMissing_InsertsNulls()
{
// fill the table with some existing columns, constraints and data
DataTable table = new DataTable();
DataColumn idColumn = table.Columns.Add("id", typeof(int));
DataColumn nameColumn = table.Columns.Add("name", typeof(string));
DataColumn createdColumn = table.Columns.Add("created", typeof(DateTime));
DataColumn avgColumn = table.Columns.Add("avg", typeof(string));
table.Constraints.Add("PK_blah", idColumn, true);
DataRow row = table.Rows.Add(new object[] { 1, "Bob", new DateTime(2018, 07, 16), "12.34" });
row.AcceptChanges();
const string text = @"id,name,created
2,John,07/17/2018
3,Susan,07/18/2018";
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
StringReader stringReader = new StringReader(text);
var schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"));
schema.AddColumn(new StringColumn("name"));
schema.AddColumn(new DateTimeColumn("created"));
IReader csvReader = new SeparatedValueReader(stringReader, schema, options);
table.ReadFlatFile(csvReader);
Assert.AreEqual(4, table.Columns.Count);
Assert.IsTrue(table.Columns.Contains("id"), "The ID column was not extracted.");
Assert.IsTrue(table.Columns.Contains("name"), "The name column was not extracted.");
Assert.IsTrue(table.Columns.Contains("created"), "The created column was not extracted.");
Assert.IsTrue(table.Columns.Contains("avg"), "The AVG column was not extracted.");
Assert.AreEqual(3, table.Rows.Count);
CollectionAssert.AreEqual(new object[] { 1, "Bob", new DateTime(2018, 07, 16), "12.34" }, table.Rows[0].ItemArray);
CollectionAssert.AreEqual(new object[] { 2, "John", new DateTime(2018, 07, 17), DBNull.Value }, table.Rows[1].ItemArray);
CollectionAssert.AreEqual(new object[] { 3, "Susan", new DateTime(2018, 07, 18), DBNull.Value }, table.Rows[2].ItemArray);
}
public void TestGetSchema_SchemaProvided_FirstRecordSchema_SkipsFirstRecord()
{
const string text = @"id,name,created";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
StringReader stringReader = new StringReader(text);
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
IReader parser = new SeparatedValueReader(stringReader, schema, options);
ISchema actual = parser.GetSchema();
Assert.Same(schema, actual);
Assert.False(parser.Read(), "The schema record was not skipped.");
}
public IEnumerable <int[]> GetData(string filePath, string separator = ";")
{
using (StreamReader streamReader = new StreamReader(filePath))
{
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = false, Separator = separator, Quote = '\''
};
SeparatedValueReader reader = new SeparatedValueReader(streamReader, options);
List <int[]> values = new List <int[]>();
while (reader.Read())
{
var actualValues = reader.GetValues();
values.Add(Array.ConvertAll(actualValues, x => Convert.ToInt32(x)));
}
return(values);
}
}
public void TestReadFlatFile_ExtractsSchema_PopulatesTable()
{
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "MM/dd/yyyy", OutputFormat = "MM/dd/yyyy"
})
.AddColumn(new DecimalColumn("avg"));
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
using (MemoryStream stream = new MemoryStream())
{
using (SeparatedValueWriter builder = new SeparatedValueWriter(stream, schema, options))
{
builder.Write(new object[] { 123, "Bob", new DateTime(2012, 12, 31), 3.14159m });
}
stream.Position = 0;
DataTable table = new DataTable();
IReader parser = new SeparatedValueReader(stream, options);
table.ReadFlatFile(parser);
Assert.AreEqual(4, table.Columns.Count, "The wrong number of columns were extracted.");
Assert.IsTrue(table.Columns.Contains("id"), "The ID column was not extracted.");
Assert.IsTrue(table.Columns.Contains("name"), "The name column was not extracted.");
Assert.IsTrue(table.Columns.Contains("created"), "The created column was not extracted.");
Assert.IsTrue(table.Columns.Contains("avg"), "The AVG column was not extracted.");
Assert.AreEqual(1, table.Rows.Count, "Not all of the records were extracted.");
DataRow row = table.Rows[0];
object[] expected = new object[] { "123", "Bob", "12/31/2012", "3.14159" };
object[] values = row.ItemArray;
CollectionAssert.AreEqual(expected, values, "The wrong values were extracted");
}
}
public void ShouldPreserveLeadingAndTrailingWhitespaceIfConfigured()
{
string source = " a ";
StringReader stringReader = new StringReader(source);
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new StringColumn("a")
{
Trim = false
});
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = false, PreserveWhiteSpace = true
};
SeparatedValueReader reader = new SeparatedValueReader(stringReader, schema, options);
object[][] expected = new object[][]
{
new object[] { " a " }
};
assertRecords(expected, reader);
}
public void ShouldPreserveWhiteSpaceIfConfigured_AllWhitespace()
{
string source = " \t\n\r ";
StringReader stringReader = new StringReader(source);
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new StringColumn("a")
{
Trim = false, NullFormatter = NullFormatter.ForValue(null)
});
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = false, RecordSeparator = "\r\n", PreserveWhiteSpace = true
};
SeparatedValueReader reader = new SeparatedValueReader(stringReader, schema, options);
object[][] expected = new object[][]
{
new object[] { " \t\n\r " }
};
assertRecords(expected, reader);
}
public void ShouldNotStripEmbeddedWhitespace_PreservingWhiteSpace()
{
string source = " a b ";
StringReader stringReader = new StringReader(source);
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new StringColumn("a")
{
Trim = false
});
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = false, PreserveWhiteSpace = true
};
SeparatedValueReader reader = new SeparatedValueReader(stringReader, schema, options);
object[][] expected = new object[][]
{
new object[] { " a b " }
};
assertRecords(expected, reader);
}
public void ShouldIgnoreBadRows()
{
const string data = @"0,2018-02-30
1,2018-10-02,{FC6AE158-F5E9-49CC-A76A-E48F3FBE6BC1}";
var stringReader = new StringReader(data);
var schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("Int32"));
schema.AddColumn(new DateTimeColumn("DateTime"));
schema.AddColumn(new GuidColumn("Guid"));
var csvReader = new SeparatedValueReader(stringReader, schema);
csvReader.RecordError += (sender, e) =>
{
Assert.IsNotNull(e.RecordContext, "The record context was null");
Assert.AreEqual(e.RecordContext.PhysicalRecordNumber, 1);
Assert.IsNotNull(e.Exception, "The exception was null.");
e.IsHandled = true;
};
Assert.IsTrue(csvReader.Read(), "The second record could not be read.");
Assert.IsFalse(csvReader.Read(), "Too many records were read -- probably the first record was not skipped.");
}
public void TestReadWrite_Comments()
{
StringWriter output = new StringWriter();
SeparatedValueWriter writer = new SeparatedValueWriter(output);
writer.Write(new[] { "a", "b", "c" });
writer.WriteRaw("# Hello, world!!!", true);
writer.Write(new[] { "d", "e", "f" });
StringReader input = new StringReader(output.ToString());
SeparatedValueReader reader = new SeparatedValueReader(input);
reader.RecordRead += (sender, e) =>
{
e.IsSkipped = e.Values.Length > 0 && e.Values[0].StartsWith("#");
};
Assert.IsTrue(reader.Read());
CollectionAssert.AreEqual(new[] { "a", "b", "c" }, reader.GetValues());
Assert.IsTrue(reader.Read());
CollectionAssert.AreEqual(new[] { "d", "e", "f" }, reader.GetValues());
Assert.IsFalse(reader.Read());
}
public void TestNullableExtensions_AllNull()
{
string data = String.Join(",", typeof(NullableValues).GetProperties().Select(x => (string)null));
var schema = GetSchema();
var stringReader = new StringReader(data);
var csvReader = new SeparatedValueReader(stringReader, schema);
var dataReader = new FlatFileDataReader(csvReader);
Assert.IsTrue(dataReader.Read(), "A record was not read.");
Assert.IsNull(dataReader.GetNullableByte(0), "The byte was not null.");
Assert.IsNull(dataReader.GetNullableInt16(1), "The byte was not null.");
Assert.IsNull(dataReader.GetNullableInt32(2), "The byte was not null.");
Assert.IsNull(dataReader.GetNullableInt64(3), "The byte was not null.");
Assert.IsNull(dataReader.GetNullableFloat(4), "The byte was not null.");
Assert.IsNull(dataReader.GetNullableDouble(5), "The byte was not null.");
Assert.IsNull(dataReader.GetNullableDecimal(6), "The byte was not null.");
Assert.IsNull(dataReader.GetNullableString(7), "The byte was not null.");
Assert.IsNull(dataReader.GetNullableDateTime(8), "The byte was not null.");
Assert.IsNull(dataReader.GetNullableGuid(9), "The byte was not null.");
Assert.IsNull(dataReader.GetNullableEnum <DayOfWeek>(10), "The byte was not null.");
Assert.IsFalse(dataReader.Read(), "Too many records were read.");
}
public void TestReadFlatFile_ExtractsSchema_PopulatesTable()
{
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "MM/dd/yyyy", OutputFormat = "MM/dd/yyyy"
})
.AddColumn(new DecimalColumn("avg"));
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter builder = new SeparatedValueWriter(stringWriter, schema, options);
builder.Write(new object[] { 123, "Bob", new DateTime(2012, 12, 31), 3.14159m });
StringReader stringReader = new StringReader(stringWriter.ToString());
DataTable table = new DataTable();
IReader parser = new SeparatedValueReader(stringReader, options);
table.ReadFlatFile(parser);
Assert.Equal(4, table.Columns.Count);
Assert.True(table.Columns.Contains("id"), "The ID column was not extracted.");
Assert.True(table.Columns.Contains("name"), "The name column was not extracted.");
Assert.True(table.Columns.Contains("created"), "The created column was not extracted.");
Assert.True(table.Columns.Contains("avg"), "The AVG column was not extracted.");
Assert.Equal(1, table.Rows.Count);
DataRow row = table.Rows[0];
object[] expected = new object[] { "123", "Bob", "12/31/2012", "3.14159" };
object[] values = row.ItemArray;
Assert.Equal(expected, values);
}
public void ShouldIncludeSpacesBetweenQuotes()
{
string source = "' a '";
StringReader stringReader = new StringReader(source);
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = false,
Quote = '\''
};
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new StringColumn("a")
{
Trim = false
});
SeparatedValueReader reader = new SeparatedValueReader(stringReader, schema, options);
object[][] expected = new object[][]
{
new object[] { " a " }
};
assertRecords(expected, reader);
}
public void TestRead_InspectRawRecords()
{
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
const string text = @"123,""Bob Smith"",4/21/2017";
StringReader stringReader = new StringReader(text);
var reader = new SeparatedValueReader(stringReader, schema);
reader.RecordRead += (sender, e) => {
Assert.AreEqual(@"123,""Bob Smith"",4/21/2017", e.RecordContext.Record);
CollectionAssert.AreEqual(new[] { "123", "Bob Smith", "4/21/2017" }, e.RecordContext.Values);
};
reader.RecordParsed += (sender, e) => {
Assert.AreEqual(@"123,""Bob Smith"",4/21/2017", e.RecordContext.Record);
CollectionAssert.AreEqual(new[] { "123", "Bob Smith", "4/21/2017" }, e.RecordContext.Values);
};
Assert.IsTrue(reader.Read());
Assert.IsFalse(reader.Read());
}
public void TestRead_EmbeddedQuote_ParsesCorrectly()
{
var text = @"123;Todd's Bait Shop;1/17/2014";
var schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"));
schema.AddColumn(new StringColumn("name"));
schema.AddColumn(new DateTimeColumn("created"));
var options = new SeparatedValueOptions
{
IsFirstRecordSchema = false,
Separator = ";"
};
StringReader stringReader = new StringReader(text);
var reader = new SeparatedValueReader(stringReader, schema, options);
var result = reader.Read();
Assert.True(result, "Could not read the record.");
object[] expected = { 123, "Todd's Bait Shop", new DateTime(2014, 1, 17) };
object[] actual = reader.GetValues();
Assert.Equal(expected, actual);
}
public void ShouldIgnoreRecordSeparatorsWithinQuotes()
{
string source = @"'John','Smith','123 Playtown Place', 'Grangewood','CA' ,12345,'John likes to travel to far away places.
His favorite travel spots are Tannis, Venice and Chicago.
When he''s not traveling, he''s at home with his lovely wife, children and leather armchair.'
Mary,Smith,'1821 Grover''s Village',West Chattingham,WA,43221,'Likes cats.'";
StringReader stringReader = new StringReader(source);
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = false,
Quote = '\''
};
SeparatedValueReader reader = new SeparatedValueReader(stringReader, options);
object[][] expected = new object[][]
{
new object[]
{ "John", "Smith", "123 Playtown Place", "Grangewood", "CA", "12345", @"John likes to travel to far away places.
His favorite travel spots are Tannis, Venice and Chicago.
When he's not traveling, he's at home with his lovely wife, children and leather armchair." },
new object[] { "Mary", "Smith", "1821 Grover's Village", "West Chattingham", "WA", "43221", "Likes cats." }
};
assertRecords(expected, reader);
}
public void ShouldSubstituteBadValues_CSV()
{
const string data = @"ABC,2018-02-30,{1234-5678-9123-000000}";
var stringReader = new StringReader(data);
var schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("Int32"));
schema.AddColumn(new DateTimeColumn("DateTime"));
schema.AddColumn(new GuidColumn("Guid"));
var csvReader = new SeparatedValueReader(stringReader, schema);
csvReader.ColumnError += (sender, e) =>
{
if (e.ColumnContext.ColumnDefinition.ColumnName == "Int32")
{
e.Substitution = 1;
e.IsHandled = true;
}
else if (e.ColumnContext.ColumnDefinition.ColumnName == "DateTime")
{
e.Substitution = new DateTime(2018, 07, 08);
e.IsHandled = true;
}
else if (e.ColumnContext.ColumnDefinition.ColumnName == "Guid")
{
e.Substitution = Guid.Empty;
e.IsHandled = true;
}
};
Assert.IsTrue(csvReader.Read(), "Could not read the first record.");
var values = csvReader.GetValues();
var expected = new object[] { 1, new DateTime(2018, 07, 08), Guid.Empty };
CollectionAssert.AreEqual(expected, values, "The wrong values were substituted.");
Assert.IsFalse(csvReader.Read(), "Read too many records.");
}
public void TestRead_MultipleCallsToValues_ReturnsSameValues()
{
string text = "a,b,c";
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)));
bool canRead = parser.Read();
Assert.IsTrue(canRead, "Could not read the record.");
string[] expected = new string[] { "a", "b", "c" };
object[] actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed.");
actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The same values were not returned multiple times.");
}
public void TestGetSchema_FirstRecordSchema_TooManyColumns_IgnoresTrailing()
{
const string text = @"id,name,created
123,Bob,1/19/2013,Hello";
StringReader stringReader = new StringReader(text);
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
SeparatedValueReader parser = new SeparatedValueReader(stringReader, options);
Assert.True(parser.Read(), "The record could not be read.");
Assert.Equal(parser.GetSchema().ColumnDefinitions.Count, parser.GetValues().Length);
;
}
public void TestGetSchema_FirstRecordSchema_TooFewColumns_Throws()
{
const string text = @"id,name,created
123,Bob";
StringReader stringReader = new StringReader(text);
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
SeparatedValueReader parser = new SeparatedValueReader(stringReader, options);
Assert.Throws<FlatFileException>(() => parser.Read());
}
public void TestGetSchema_SchemaProvided_ParsesValues_Quoted()
{
const string text = "123,\"Bob\",1/19/2013";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader, schema);
Assert.True(parser.Read(), "The first record was skipped.");
object[] actual = parser.GetValues();
object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
Assert.Equal(expected, actual);
}
public void TestGetSchema_SchemaProvided_WrongNumberOfColumns_Throws()
{
const string text = @"123,Bob";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader, schema);
Assert.Throws<FlatFileException>(() => parser.Read());
}
public void TestReader_creativyst_example()
{
const string text = @"John,Doe,120 jefferson st.,Riverside, NJ, 08075
Jack,McGinnis,220 hobo Av.,Phila, PA,09119
""John """"Da Man"""""",Repici,120 Jefferson St.,Riverside, NJ,08075
Stephen,Tyler,""7452 Terrace """"At the Plaza"""" road"",SomeTown,SD, 91234
,Blankman,,SomeTown, SD, 00298
""Joan """"the bone"""", Anne"",Jet,""9th, at Terrace plc"",Desert City, CO,00123
";
StringReader stringReader = new StringReader(text);
SeparatedValueReader reader = new SeparatedValueReader(stringReader);
Assert.True(reader.Read(), "Could not read the first record.");
assertValues(reader, "John", "Doe", "120 jefferson st.", "Riverside", "NJ", "08075");
Assert.True(reader.Read(), "Could not read the second record.");
assertValues(reader, "Jack", "McGinnis", "220 hobo Av.", "Phila", "PA", "09119");
Assert.True(reader.Read(), "Could not read the third record.");
assertValues(reader, "John \"Da Man\"", "Repici", "120 Jefferson St.", "Riverside", "NJ", "08075");
Assert.True(reader.Read(), "Could not read the fourth record.");
assertValues(reader, "Stephen", "Tyler", "7452 Terrace \"At the Plaza\" road", "SomeTown", "SD", "91234");
Assert.True(reader.Read(), "Could not read the fifth record.");
assertValues(reader, "", "Blankman","", "SomeTown", "SD", "00298");
Assert.True(reader.Read(), "Could not read the sixth record.");
assertValues(reader, "Joan \"the bone\", Anne", "Jet", "9th, at Terrace plc", "Desert City", "CO", "00123");
Assert.False(reader.Read(), "Read too many records.");
}
public void TestRead_GetValuesWithoutReading_Throws()
{
string text = "a,b,c";
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)));
parser.GetValues();
}
public void TestGetSchema_NotExtracted_Throws()
{
string text = "a,b,c";
StringReader stringReader = new StringReader(text);
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = false };
IReader parser = new SeparatedValueReader(stringReader, options);
Assert.Throws<InvalidOperationException>(() => parser.GetSchema());
}
public SeparatedValueTypedReader(SeparatedValueReader reader, IMapper <TEntity> mapper)
: base(reader, mapper)
{
this.reader = reader;
}
public void TestRead_GetValuesWithoutReading_Throws()
{
string text = "a,b,c";
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader);
Assert.Throws<InvalidOperationException>(() => parser.GetValues());
}
public void TestRead_EmbeddedQuote_ParsesCorrectly()
{
var text = @"123;Todd's Bait Shop;1/17/2014";
var schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"));
schema.AddColumn(new StringColumn("name"));
schema.AddColumn(new DateTimeColumn("created"));
var options = new SeparatedValueOptions
{
IsFirstRecordSchema = false,
Separator = ";"
};
StringReader stringReader = new StringReader(text);
var reader = new SeparatedValueReader(stringReader, schema, options);
var result = reader.Read();
Assert.True(result, "Could not read the record.");
object[] expected = { 123, "Todd's Bait Shop", new DateTime(2014, 1, 17) };
object[] actual = reader.GetValues();
Assert.Equal(expected, actual);
}
/// <summary>
/// Parse an AttractMode DAT and return all found games within
/// </summary>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="indexId">Index ID for the DAT</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
protected override void ParseFile(string filename, int indexId, bool keep, bool throwOnError = false)
{
// Open a file reader
Encoding enc = FileExtensions.GetEncoding(filename);
SeparatedValueReader svr = new SeparatedValueReader(FileExtensions.TryOpenRead(filename), enc)
{
Header = true,
Quotes = false,
Separator = ';',
VerifyFieldCount = true
};
// If we're somehow at the end of the stream already, we can't do anything
if (svr.EndOfStream)
{
return;
}
// Read in the header
svr.ReadHeader();
// Header values should match
// #Name;Title;Emulator;CloneOf;Year;Manufacturer;Category;Players;Rotation;Control;Status;DisplayCount;DisplayType;AltRomname;AltTitle;Extra;Buttons
// Loop through all of the data lines
while (!svr.EndOfStream)
{
try
{
// Get the current line, split and parse
svr.ReadNextLine();
Rom rom = new Rom
{
Name = "-",
Size = Constants.SizeZero,
CRC = Constants.CRCZero,
MD5 = Constants.MD5Zero,
SHA1 = Constants.SHA1Zero,
ItemStatus = ItemStatus.None,
Machine = new Machine
{
Name = svr.Line[0], // #Name
Description = svr.Line[1], // Title
CloneOf = svr.Line[3], // CloneOf
Year = svr.Line[4], // Year
Manufacturer = svr.Line[5], // Manufacturer
Category = svr.Line[6], // Category
Players = svr.Line[7], // Players
Rotation = svr.Line[8], // Rotation
Control = svr.Line[9], // Control
Status = svr.Line[10], // Status
DisplayCount = svr.Line[11], // DisplayCount
DisplayType = svr.Line[12], // DisplayType
Comment = svr.Line[15], // Extra
Buttons = svr.Line[16], // Buttons
},
AltName = svr.Line[13], // AltRomname
AltTitle = svr.Line[14], // AltTitle
Source = new Source
{
Index = indexId,
Name = filename,
},
};
// Now process and add the rom
ParseAddHelper(rom);
}
catch (Exception ex)
{
string message = $"'{filename}' - There was an error parsing line {svr.LineNumber} '{svr.CurrentLine}'";
logger.Error(ex, message);
if (throwOnError)
{
svr.Dispose();
throw new Exception(message, ex);
}
}
}
svr.Dispose();
}
public void TestReadFlatFile_ExtractsSchema_PopulatesTable()
{
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created") { InputFormat = "MM/dd/yyyy", OutputFormat = "MM/dd/yyyy" })
.AddColumn(new DecimalColumn("avg"));
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter builder = new SeparatedValueWriter(stringWriter, schema, options);
builder.Write(new object[] { 123, "Bob", new DateTime(2012, 12, 31), 3.14159m });
StringReader stringReader = new StringReader(stringWriter.ToString());
DataTable table = new DataTable();
IReader parser = new SeparatedValueReader(stringReader, options);
table.ReadFlatFile(parser);
Assert.Equal(4, table.Columns.Count);
Assert.True(table.Columns.Contains("id"), "The ID column was not extracted.");
Assert.True(table.Columns.Contains("name"), "The name column was not extracted.");
Assert.True(table.Columns.Contains("created"), "The created column was not extracted.");
Assert.True(table.Columns.Contains("avg"), "The AVG column was not extracted.");
Assert.Equal(1, table.Rows.Count);
DataRow row = table.Rows[0];
object[] expected = new object[] { "123", "Bob", "12/31/2012", "3.14159" };
object[] values = row.ItemArray;
Assert.Equal(expected, values);
}
public void TestReadFlatFile_SchemaProvided_TypesUsed()
{
const string text = @"123,Bob,12/31/2012,3.14159";
DataTable table = new DataTable();
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"))
.AddColumn(new DoubleColumn("avg"));
StringReader stringReader = new StringReader(text);
IReader parser = new SeparatedValueReader(stringReader, schema);
table.ReadFlatFile(parser);
Assert.Equal(4, table.Columns.Count);
Assert.True(table.Columns.Contains("id"), "The ID column was not extracted.");
Assert.True(table.Columns.Contains("name"), "The name column was not extracted.");
Assert.True(table.Columns.Contains("created"), "The created column was not extracted.");
Assert.True(table.Columns.Contains("avg"), "The AVG column was not extracted.");
Assert.Equal(1, table.Rows.Count);
DataRow row = table.Rows[0];
object[] expected = new object[] { 123, "Bob", new DateTime(2012, 12, 31), 3.14159 };
object[] values = row.ItemArray;
Assert.Equal(expected, values);
}
public void TestGetValues_BlankTrailingSection_ReturnsNull()
{
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" })
.AddColumn(new StringColumn("trailing"));
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19), "" };
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter builder = new SeparatedValueWriter(stringWriter, schema, options);
builder.Write(sources);
StringReader stringReader = new StringReader(stringWriter.ToString());
SeparatedValueReader parser = new SeparatedValueReader(stringReader, schema, options);
Assert.True(parser.Read(), "No records were found.");
object[] values = parser.GetValues();
Assert.Equal(schema.ColumnDefinitions.Count, values.Length);
Assert.Equal(sources[0], values[0]);
Assert.Equal(sources[1], values[1]);
Assert.Equal(sources[2], values[2]);
Assert.Equal(null, values[3]);
Assert.False(parser.Read(), "Too many records were found.");
}
public void TestRead_SingleRecord_ReturnsTrueOnce()
{
const string text = "a,b,c";
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)));
bool canRead = parser.Read();
Assert.IsTrue(canRead, "Could not read the record.");
string[] expected = new string[] { "a", "b", "c" };
object[] actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed.");
canRead = parser.Read();
Assert.IsFalse(canRead, "No more records should have been read.");
}
private static void assertValues(SeparatedValueReader reader, string firstName, string lastName, string street, string city, string state, string zip)
{
object[] values = reader.GetValues();
Assert.Equal(6, values.Length);
Assert.Equal(firstName, values[0]);
Assert.Equal(lastName, values[1]);
Assert.Equal(street, values[2]);
Assert.Equal(city, values[3]);
Assert.Equal(state, values[4]);
Assert.Equal(zip, values[5]);
}
public void TestRead_ValuesAfterEndOfFile_Throws()
{
string text = "a,b,c";
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)));
bool canRead = parser.Read();
Assert.IsTrue(canRead, "Could not read the record.");
canRead = parser.Read();
Assert.IsFalse(canRead, "We should have reached the end of the file.");
parser.GetValues();
}
public void TestRead_RecordWithCP1252Characters_ReturnsCorrectCharacters()
{
//---- Arrange -----------------------------------------------------
// Need to convert the string to target encoding because otherwise a string declared in VS will always be encoded as UTF-8
var text = Encoding.Convert(Encoding.UTF8, Encoding.GetEncoding(1252), Encoding.UTF8.GetBytes(@"123;Müller;1/17/2014"));
var schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id")).AddColumn(new StringColumn("name")).AddColumn(new DateTimeColumn("created"));
var options = new SeparatedValueOptions
{
IsFirstRecordSchema = false,
Separator = ";" ,
Encoding = Encoding.GetEncoding(1252)
};
var testee = new SeparatedValueReader(new MemoryStream(text), schema, options);
//---- Act ---------------------------------------------------------
var result = testee.Read();
//---- Assert ------------------------------------------------------
Assert.IsTrue(result, "Could not read the record.");
object[] expected = { 123, "Müller", new DateTime(2014, 1, 17) };
object[] actual = testee.GetValues();
CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed.");
}
public Core.ResultDefinition ValidateFile()
{
var schemaData = string.Empty;
var schemaParser = new SchemaParser();
var validationResults = new List <ValidationResult>();
var validator = new Validators.FixedLengthValidator();
if (this.RawSchema == null)
{
try
{
using (var sr = new StreamReader(File.OpenRead(this.SchemaFilePath)))
{
schemaData = sr.ReadToEnd();
}
}
catch (Exception ex)
{
var errorMsg = String.Format(null, SharedResources.SchemaError, ex.Message);
Console.WriteLine($"{errorMsg}");
}
this.RawSchema = schemaParser.ParseSchema(schemaData);
}
var separatedFileSchemaBuilder = new SeparatedSchemaBuilder();
var separatedFileSchema = separatedFileSchemaBuilder.BuildSchema(this.RawSchema);
var schemas = separatedFileSchema.SeparatedRecordSchemas;
var count = 0;
var options = new SeparatedValueOptions
{
Separator = separatedFileSchema.Delimeter,
PreserveWhiteSpace = true
};
foreach (var inputLine in File.ReadLines(this.FilePath))
{
Console.WriteLine($"Started parsing line...{count}");
if (inputLine != null)
{
SeparatedValueSchema actualSchema;
TextReader stringReader = new StringReader(inputLine);
var schema = schemas.FirstOrDefault(x => inputLine.StartsWith(x.RecordIdentifier));
count++;
var innerResults = new List <ValidationResult>();
var line = inputLine;
if (schema != null)
{
var parser = new SeparatedValueReader(stringReader, schema.SeparatedValueSchema, options);
if (!inputLine.Contains(options.Separator))
{
var error = new ValidationResult
{
Record = inputLine,
ErrorMessages = new List <string>
{
$"Column Separator Error: Record could not be parsed as separator {options.Separator} defined in schema is not found.",
},
HasErrors = true,
RowNumber = count,
};
validationResults.Add(error);
continue;
}
try
{
actualSchema = parser.GetSchema();
parser.Read();
var values = parser.GetValues();
for (int i = 0; i < values.Length; i++)
{
var validationResult = new ValidationResult
{
Record = inputLine,
ColumnName = actualSchema.ColumnDefinitions[i].ColumnName,
ColumnType = actualSchema.ColumnDefinitions[i].ColumnType.Name,
ActualRowLength = inputLine.Length,
MaxColumns = actualSchema.ColumnDefinitions.HandledCount,
ParsedValueLength = (values[i].ToString() ?? string.Empty).Length,
RawValueLength = (values[i].ToString() ?? string.Empty).Length,
ParsedValue = values[i].ToString() ?? string.Empty,
RawValue = values[i].ToString() ?? string.Empty,
RowNumber = count,
ParsedValuesCount = values.Count()
};
}
}
catch (Exception ex)
{
var errorMsg = string.Empty;
if (ex.InnerException != null && ex.InnerException.Message != null)
{
errorMsg = ex.InnerException.Message;
Console.WriteLine(ex.InnerException.Message);
}
else
{
errorMsg = ex.Message;
Console.WriteLine(ex.Message);
}
var error = new ValidationResult
{
RowNumber = count,
Record = inputLine,
ErrorMessages = new List <string>
{
$"Exception: {errorMsg}"
},
HasErrors = true
};
validationResults.Add(error);
}
}
else
{
var error = new ValidationResult
{
RowNumber = count,
Record = inputLine,
ErrorMessages = new List <string>
{
$"Exception: The row does not contain a record identifier that match the schema."
},
HasErrors = true
};
validationResults.Add(error);
}
}
}
var results = new Core.ResultDefinition
{
Results = validationResults,
TotalLinesProcessed = count,
FileFormat = this.RawSchema.FileFormat
};
return(results);
}
public void TestGetSchema_SchemaProvided_FirstRecordSchema_SkipsFirstRecord()
{
const string text = @"id,name,created";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
StringReader stringReader = new StringReader(text);
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
IReader parser = new SeparatedValueReader(stringReader, schema, options);
ISchema actual = parser.GetSchema();
Assert.Same(schema, actual);
Assert.False(parser.Read(), "The schema record was not skipped.");
}
/// <summary>
/// Parse an Everdrive SMDB file and return all found games within
/// </summary>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="indexId">Index ID for the DAT</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
protected override void ParseFile(string filename, int indexId, bool keep, bool throwOnError = false)
{
// Open a file reader
Encoding enc = FileExtensions.GetEncoding(filename);
SeparatedValueReader svr = new SeparatedValueReader(FileExtensions.TryOpenRead(filename), enc)
{
Header = false,
Quotes = false,
Separator = '\t',
VerifyFieldCount = false,
};
while (!svr.EndOfStream)
{
try
{
// If we can't read the next line, break
if (!svr.ReadNextLine())
{
break;
}
// If the line returns null somehow, skip
if (svr.Line == null)
{
continue;
}
/*
* The gameinfo order is as follows
* 0 - SHA-256
* 1 - Machine Name/Filename
* 2 - SHA-1
* 3 - MD5
* 4 - CRC32
*/
string[] fullname = svr.Line[1].Split('/');
Rom rom = new Rom
{
Name = svr.Line[1].Substring(fullname[0].Length + 1),
Size = null, // No size provided, but we don't want the size being 0
CRC = svr.Line[4],
MD5 = svr.Line[3],
SHA1 = svr.Line[2],
SHA256 = svr.Line[0],
ItemStatus = ItemStatus.None,
Machine = new Machine
{
Name = fullname[0],
Description = fullname[0],
},
Source = new Source
{
Index = indexId,
Name = filename,
},
};
// Now process and add the rom
ParseAddHelper(rom);
}
catch (Exception ex)
{
string message = $"'{filename}' - There was an error parsing line {svr.LineNumber} '{svr.CurrentLine}'";
logger.Error(ex, message);
if (throwOnError)
{
svr.Dispose();
throw new Exception(message, ex);
}
}
}
svr.Dispose();
}
public void TestGetValues_BlankTrailingSection_ReturnsNull()
{
using (MemoryStream stream = new MemoryStream())
{
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" })
.AddColumn(new StringColumn("trailing"));
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19), "" };
using (SeparatedValueWriter builder = new SeparatedValueWriter(stream, schema, options))
{
builder.Write(sources);
}
stream.Position = 0;
SeparatedValueReader parser = new SeparatedValueReader(stream, schema, options);
Assert.IsTrue(parser.Read(), "No records were found.");
object[] values = parser.GetValues();
Assert.AreEqual(schema.ColumnDefinitions.Count, values.Length, "The wrong number of values were read.");
Assert.AreEqual(sources[0], values[0], "The first column was not parsed correctly.");
Assert.AreEqual(sources[1], values[1], "The second column was not parsed correctly.");
Assert.AreEqual(sources[2], values[2], "The third column was not parsed correctly.");
Assert.AreEqual(null, values[3], "The forth column was not interpreted as null.");
Assert.IsFalse(parser.Read(), "Too many records were found.");
}
}
public void TestRead_MultipleCallsToValues_ReturnsSameValues()
{
string text = "a,b,c";
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader);
bool canRead = parser.Read();
Assert.True(canRead, "Could not read the record.");
object[] expected = new object[] { "a", "b", "c" };
object[] actual = parser.GetValues();
Assert.Equal(expected, actual);
actual = parser.GetValues();
Assert.Equal(expected, actual);
}
public void TestRead_ZeroLengthColumn()
{
//---- Arrange -----------------------------------------------------
var text = "104\t20\t1000\t00\tLausanne\tLausanne\tVD\t2\t\t0\t130\t5586\t19880301";
var options = new SeparatedValueOptions { IsFirstRecordSchema = false, Separator = "\t" };
var schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("OnrpId"))
.AddColumn(new Int32Column("Type"))
.AddColumn(new Int32Column("ZipCode"))
.AddColumn(new StringColumn("ZipCodeAddOn"))
.AddColumn(new StringColumn("TownShortName"))
.AddColumn(new StringColumn("TownOfficialName"))
.AddColumn(new StringColumn("CantonAbbreviation"))
.AddColumn(new Int16Column("MainLanguageCode"))
.AddColumn(new Int16Column("OtherLanguageCode"))
.AddColumn(new ByteColumn("HasSortfileData"))
.AddColumn(new Int32Column("LetterServiceOnrpId"))
.AddColumn(new Int32Column("MunicipalityId"))
.AddColumn(new StringColumn("ValidFrom"));
StringReader stringReader = new StringReader(text);
var testee = new SeparatedValueReader(stringReader, options);
//---- Act ---------------------------------------------------------
var result = testee.Read();
//---- Assert ------------------------------------------------------
Assert.True(result);
Assert.Equal(schema.ColumnDefinitions.Count, testee.GetValues().Count());
}
public void TestRead_SingleRecord_ReturnsTrueOnce()
{
const string text = "a,b,c";
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader);
bool canRead = parser.Read();
Assert.True(canRead, "Could not read the record.");
object[] expected = new object[] { "a", "b", "c" };
object[] actual = parser.GetValues();
Assert.Equal(expected, actual);
canRead = parser.Read();
Assert.False(canRead, "No more records should have been read.");
}
public void TestRead_SkipRecord_NoParsingError()
{
const string text = "a,b,c";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("A"));
schema.AddColumn(new DateTimeColumn("B"));
schema.AddColumn(new GuidColumn("C"));
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader, schema);
bool canRead = parser.Skip();
Assert.True(canRead, "Could not skip the record.");
canRead = parser.Read();
Assert.False(canRead, "No more records should have been read.");
}
public MultiplexingSeparatedValueTypedReader(SeparatedValueReader reader, SeparatedValueTypeMapperSelector selector)
{
this.reader = reader;
this.selector = selector;
}
public void TestRead_ValuesAfterEndOfFile_Throws()
{
string text = "a,b,c";
StringReader stringReader = new StringReader(text);
SeparatedValueReader parser = new SeparatedValueReader(stringReader);
bool canRead = parser.Read();
Assert.True(canRead, "Could not read the record.");
canRead = parser.Read();
Assert.False(canRead, "We should have reached the end of the file.");
Assert.Throws<InvalidOperationException>(() => parser.GetValues());
}
/// <summary>
/// Parse a character-separated value DAT and return all found games and roms within
/// </summary>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="indexId">Index ID for the DAT</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
protected override void ParseFile(string filename, int indexId, bool keep, bool throwOnError = false)
{
// Open a file reader
Encoding enc = FileExtensions.GetEncoding(filename);
SeparatedValueReader svr = new SeparatedValueReader(FileExtensions.TryOpenRead(filename), enc)
{
Header = true,
Quotes = true,
Separator = _delim,
VerifyFieldCount = true,
};
// If we're somehow at the end of the stream already, we can't do anything
if (svr.EndOfStream)
{
return;
}
// Read in the header
svr.ReadHeader();
// Loop through all of the data lines
while (!svr.EndOfStream)
{
try
{
// Get the current line, split and parse
svr.ReadNextLine();
// Create mapping dictionary
var mappings = new Dictionary <Field, string>();
// Now we loop through and get values for everything
for (int i = 0; i < svr.HeaderValues.Count; i++)
{
Field key = svr.HeaderValues[i].AsField();
string value = svr.Line[i];
mappings[key] = value;
}
// Set DatHeader fields
DatHeader header = new DatHeader();
header.SetFields(mappings);
Header.ConditionalCopy(header);
// Set Machine and DatItem fields
if (mappings.ContainsKey(Field.DatItem_Type))
{
DatItem datItem = DatItem.Create(mappings[Field.DatItem_Type].AsItemType());
datItem.SetFields(mappings);
datItem.Source = new Source(indexId, filename);
ParseAddHelper(datItem);
}
}
catch (Exception ex)
{
string message = $"'{filename}' - There was an error parsing line {svr.LineNumber} '{svr.CurrentLine}'";
logger.Error(ex, message);
if (throwOnError)
{
svr.Dispose();
throw new Exception(message, ex);
}
}
}
svr.Dispose();
}
public void TestGetSchema_SchemaProvided_WrongNumberOfColumns_Throws()
{
const string text = @"123,Bob";
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created"));
SeparatedValueReader parser = new SeparatedValueReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema);
parser.Read();
}
