public void TestGetValues_NoRecordSeparator_SplitsFile() { const string text = " 123 Bob 1/19/2013 234 Sam12/20/2013"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); FixedLengthOptions options = new FixedLengthOptions() { HasRecordSeparator = false }; StringReader stringReader = new StringReader(text); FixedLengthReader parser = new FixedLengthReader(stringReader, schema, options); Assert.IsTrue(parser.Read(), "Could not read the first record."); object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) }; object[] actual = parser.GetValues(); CollectionAssert.AreEqual(expected, actual); Assert.IsTrue(parser.Read(), "Could not read the second record."); expected = new object[] { 234, "Sam", new DateTime(2013, 12, 20) }; actual = parser.GetValues(); CollectionAssert.AreEqual(expected, actual); }
public void TestRead_ValuesAfterEndOfFile_Throws() { const string text = @" 123 Bob 1/19/2013"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); FixedLengthReader parser = new FixedLengthReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema); Assert.IsTrue(parser.Read(), "Could not read the record."); Assert.IsFalse(parser.Read(), "We should have reached the end of the file."); parser.GetValues(); }
public void TestRead_RecordWithCP1251Characters_ReturnsCorrectCharacters() { //---- Arrange ----------------------------------------------------- // Need to convert the string to target encoding because otherwise a string declared in VS will always be encoded as UTF-8 var text = Encoding.Convert(Encoding.UTF8, Encoding.GetEncoding(1251), Encoding.UTF8.GetBytes(@" 123 Лучиано 1/17/2014")); var schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); var options = new FixedLengthOptions() { Encoding = Encoding.GetEncoding(1251) }; var testee = new FixedLengthReader(new MemoryStream(text), schema, options); //---- Act --------------------------------------------------------- var result = testee.Read(); //---- Assert ------------------------------------------------------ Assert.IsTrue(result, "Could not read the record."); object[] expected = { 123, "Лучиано", new DateTime(2014, 1, 17) }; object[] actual = testee.GetValues(); CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed."); }
public void TestGetValues_WithPartitionedRecordFilter_SkipAllRecords() { FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10) { Alignment = FixedAlignment.RightAligned }) .AddColumn(new StringColumn("name"), new Window(25) { Alignment = FixedAlignment.RightAligned }) .AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy" }, new Window(10) { Alignment = FixedAlignment.RightAligned }); const string lines = @" 123 Bob Smith 4/21/2017 -1 Jay Smith 8/14/2017 234 Jay Smith 5/21/2017"; StringReader stringReader = new StringReader(lines); FixedLengthReader parser = new FixedLengthReader(stringReader, schema); parser.RecordPartitioned += (sender, e) => { e.IsSkipped = true; }; Assert.IsFalse(parser.Read(), "All records should have been skipped."); }
public void TestRead_ValuesAfterEndOfFile_Throws() { const string text = @" 123 Bob 1/19/2013"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); StringReader stringReader = new StringReader(text); FixedLengthReader parser = new FixedLengthReader(stringReader, schema); Assert.IsTrue(parser.Read(), "Could not read the record."); Assert.IsFalse(parser.Read(), "We should have reached the end of the file."); Assert.ThrowsException <InvalidOperationException>(() => parser.GetValues()); }
public void TestRead_SingleRecord_ReturnsTrueOnce() { const string text = @" 123 Bob 1/19/2013"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); FixedLengthReader parser = new FixedLengthReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema); Assert.IsTrue(parser.Read(), "Could not read the record."); object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) }; object[] actual = parser.GetValues(); CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed."); Assert.IsFalse(parser.Read(), "No more records should have been read."); }
public void TestReader_UnknownType() { var stringReader = new StringReader("What's this weird thing?"); var selector = getSchemaSelector(); var reader = new FixedLengthReader(stringReader, selector); reader.Read(); }
public void TestReader_UnknownType_IgnoreUnknown_SkipsRecord() { var stringReader = new StringReader("What's this weird thing?"); var selector = getSchemaSelector(); var reader = new FixedLengthReader(stringReader, selector); reader.RecordError += (o, e) => e.IsHandled = true; Assert.IsFalse(reader.Read()); }
public void TestRead_SingleRecord_ReturnsTrueOnce() { const string text = @" 123 Bob 1/19/2013"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); StringReader stringReader = new StringReader(text); FixedLengthReader parser = new FixedLengthReader(stringReader, schema); Assert.True(parser.Read(), "Could not read the record."); object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) }; object[] actual = parser.GetValues(); Assert.Equal(expected, actual); Assert.False(parser.Read(), "No more records should have been read."); }
public bool Read() { if (!reader.Read()) { return(false); } object[] values = reader.GetValues(); Current = selector.Reader(values); return(true); }
public void TestGetValues_WithPartitionedRecordFilter_SkipRecordsMatchingCriteria() { FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10) { Alignment = FixedAlignment.RightAligned }) .AddColumn(new StringColumn("name"), new Window(25) { Alignment = FixedAlignment.RightAligned }) .AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy" }, new Window(10) { Alignment = FixedAlignment.RightAligned }); const string lines = @" 123 Bob Smith 4/21/2017 -1 Jay Smith 8/14/2017 234 Jay Smith 5/21/2017"; StringReader stringReader = new StringReader(lines); FixedLengthReader parser = new FixedLengthReader(stringReader, schema); parser.RecordPartitioned += (sender, e) => { e.IsSkipped = e.Values.Length == 3 && e.Values[0].StartsWith("-"); }; Assert.IsTrue(parser.Read(), "Could not read the first record."); object[] actual1 = parser.GetValues(); CollectionAssert.AreEqual(new object[] { 123, "Bob Smith", new DateTime(2017, 04, 21) }, actual1); Assert.IsTrue(parser.Read(), "Could not read the second record."); object[] actual2 = parser.GetValues(); CollectionAssert.AreEqual(new object[] { 234, "Jay Smith", new DateTime(2017, 05, 21) }, actual2); Assert.IsFalse(parser.Read(), "There should not be any more records."); }
public void TestGetValues_WithUnpartitionedRecordFilter_SkipRecordsMatchingCriteria() { FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10) { Alignment = FixedAlignment.RightAligned }) .AddColumn(new StringColumn("name"), new Window(25) { Alignment = FixedAlignment.RightAligned }) .AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy" }, new Window(10) { Alignment = FixedAlignment.RightAligned }); FixedLengthOptions options = new FixedLengthOptions() { UnpartitionedRecordFilter = (record) => record.StartsWith("a") }; const string lines = @" 123 Bob Smith 4/21/2017 a weird row that should be skipped 234 Jay Smith 5/21/2017"; StringReader stringReader = new StringReader(lines); FixedLengthReader parser = new FixedLengthReader(stringReader, schema, options); Assert.True(parser.Read(), "Could not read the first record."); object[] actual1 = parser.GetValues(); Assert.Equal(new object[] { 123, "Bob Smith", new DateTime(2017, 04, 21) }, actual1); Assert.True(parser.Read(), "Could not read the second record."); object[] actual2 = parser.GetValues(); Assert.Equal(new object[] { 234, "Jay Smith", new DateTime(2017, 05, 21) }, actual2); Assert.False(parser.Read(), "There should not be any more records."); }
public void TestGetSchema_SchemaProvided_WrongNumberOfColumns_Throws() { const string text = @" 123 Bob"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); FixedLengthReader parser = new FixedLengthReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema); parser.Read(); }
public void TestGetSchema_SchemaProvided_WrongNumberOfColumns_Throws() { const string text = @" 123 Bob"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); StringReader stringReader = new StringReader(text); FixedLengthReader parser = new FixedLengthReader(stringReader, schema); Assert.Throws<FlatFileException>(() => parser.Read()); }
private static object[] AssertExtra(FixedLengthReader reader, String expected) { Assert.IsTrue(reader.Read(), "Could not read the next record."); object[] values = reader.GetValues(); FixedLengthSchema schema = reader.GetSchema(); Assert.AreEqual(schema.ColumnDefinitions.Count, values.Length, "The wrong number of values were parsed."); object value = values[schema.ColumnDefinitions.Count - 1]; Assert.AreEqual(expected, value, "The wrong extra value was found for the record."); return(values); }
public void TestFixedLengthReader_MetadataColumn_IgnoresLength() { var schema = new FixedLengthSchema() .AddColumn(new RecordNumberColumn("RecordNumber"), 10) .AddColumn(new Int16Column("RecordType"), 2) .AddColumn(new StringColumn("Data"), 6); const string output = @"30header 31detail 39footer "; var stringReader = new StringReader(output); var reader = new FixedLengthReader(stringReader, schema); Assert.IsTrue(reader.Read(), "The header record could not be read."); CollectionAssert.AreEqual(new object[] { 1, (short)30, "header" }, reader.GetValues(), "The header data is wrong."); Assert.IsTrue(reader.Read(), "The detail record could not be read."); CollectionAssert.AreEqual(new object[] { 2, (short)31, "detail" }, reader.GetValues(), "The detail data is wrong."); Assert.IsTrue(reader.Read(), "The footer record could not be read."); CollectionAssert.AreEqual(new object[] { 3, (short)39, "footer" }, reader.GetValues(), "The footer data is wrong."); Assert.IsFalse(reader.Read(), "Read too many records"); }
public void TestRead_InspectRawRecords() { FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), 10) .AddColumn(new StringColumn("name"), 25) .AddColumn(new DateTimeColumn("created"), 10); const string text = @" 123 Bob 1/19/2013"; StringReader stringReader = new StringReader(text); var reader = new FixedLengthReader(stringReader, schema); reader.RecordRead += (sender, e) => { Assert.AreEqual(@" 123 Bob 1/19/2013", e.Record); }; reader.RecordParsed += (sender, e) => { Assert.AreEqual(@" 123 Bob 1/19/2013", e.RecordContext.Record); CollectionAssert.AreEqual(new[] { " 123", " Bob", " 1/19/2013" }, e.RecordContext.Values); }; Assert.IsTrue(reader.Read()); Assert.IsFalse(reader.Read()); }
public void TestGetSchema_SchemaProvided_WrongNumberOfColumns_Throws() { const string text = @" 123 Bob"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); StringReader stringReader = new StringReader(text); FixedLengthReader parser = new FixedLengthReader(stringReader, schema); Assert.ThrowsException <RecordProcessingException>(() => parser.Read()); }
public void TestReaderWriter_TrailingText_RoundTripsExtra() { FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10) { Alignment = FixedAlignment.RightAligned }) .AddColumn(new StringColumn("name"), new Window(25) { Alignment = FixedAlignment.RightAligned }) .AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" }, new Window(10) { Alignment = FixedAlignment.RightAligned }) .AddColumn(new StringColumn("extra"), Window.Trailing); const string lines = @" 123 Bob Smith 4/21/2017This -1 Jay Smith 8/14/2017is 234 Jay Smith 5/21/2017extra "; StringReader stringReader = new StringReader(lines); FixedLengthReader parser = new FixedLengthReader(stringReader, schema); List <object[]> records = new List <object[]>() { AssertExtra(parser, "This"), AssertExtra(parser, "is"), AssertExtra(parser, "extra") }; Assert.IsFalse(parser.Read()); StringWriter stringWriter = new StringWriter(); FixedLengthWriter writer = new FixedLengthWriter(stringWriter, schema); foreach (object[] record in records) { writer.Write(record); } string formatted = stringWriter.ToString(); Assert.AreEqual(lines, formatted, "The records did not round-trip."); }
public void ShouldWriteSchemaIfExplicit() { StringWriter stringWriter = new StringWriter(); // Explicitly indicate that the first record is NOT the schema FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new StringColumn("Col1"), 10); FixedLengthWriter writer = new FixedLengthWriter(stringWriter, schema, new FixedLengthOptions() { IsFirstRecordHeader = false }); writer.WriteSchema(); // Explicitly write the schema writer.Write(new string[] { "a" }); StringReader stringReader = new StringReader(stringWriter.ToString()); var reader = new FixedLengthReader(stringReader, schema, new FixedLengthOptions() { IsFirstRecordHeader = true }); Assert.IsTrue(reader.Read(), "The record was not retrieved after the schema."); Assert.IsFalse(reader.Read(), "Encountered more than the expected number of records."); }
public void ShouldSubstituteBadValues_FixedLength() { const string data = @"ABC 2018-02-30{1234-5678-9123-000000} "; var stringReader = new StringReader(data); var schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("Int32"), 5); schema.AddColumn(new DateTimeColumn("DateTime"), 10); schema.AddColumn(new GuidColumn("Guid"), 32); var csvReader = new FixedLengthReader(stringReader, schema); csvReader.ColumnError += (sender, e) => { if (e.ColumnContext.ColumnDefinition.ColumnName == "Int32") { e.Substitution = 1; e.IsHandled = true; } else if (e.ColumnContext.ColumnDefinition.ColumnName == "DateTime") { e.Substitution = new DateTime(2018, 07, 08); e.IsHandled = true; } else if (e.ColumnContext.ColumnDefinition.ColumnName == "Guid") { e.Substitution = Guid.Empty; e.IsHandled = true; } }; Assert.IsTrue(csvReader.Read(), "Could not read the first record."); var values = csvReader.GetValues(); var expected = new object[] { 1, new DateTime(2018, 07, 08), Guid.Empty }; CollectionAssert.AreEqual(expected, values, "The wrong values were substituted."); Assert.IsFalse(csvReader.Read(), "Read too many records."); }
public void TestGetValues_CustomRecordSeparator_SplitsFile() { const string text = " 123 Bob 1/19/2013BOOM 234 Sam12/20/2013"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); FixedLengthOptions options = new FixedLengthOptions() { RecordSeparator = "BOOM" }; FixedLengthReader parser = new FixedLengthReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema, options); Assert.IsTrue(parser.Read(), "Could not read the first record."); object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) }; object[] actual = parser.GetValues(); CollectionAssert.AreEqual(expected, actual, "The values for the first record were wrong."); Assert.IsTrue(parser.Read(), "Could not read the second record."); expected = new object[] { 234, "Sam", new DateTime(2013, 12, 20) }; actual = parser.GetValues(); CollectionAssert.AreEqual(expected, actual, "The values for the second record were wrong."); }
public void TestRead_SkipRecord_NoParsingError() { const string text = "a b c"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("A"), 8); schema.AddColumn(new DateTimeColumn("B"), 23); schema.AddColumn(new GuidColumn("C"), 2); StringReader stringReader = new StringReader(text); FixedLengthReader parser = new FixedLengthReader(stringReader, schema); bool canRead = parser.Skip(); Assert.IsTrue(canRead, "Could not skip the record."); canRead = parser.Read(); Assert.IsFalse(canRead, "No more records should have been read."); }
public void TestRead_MultipleCallsToValues_ReturnsSameValues() { const string text = @" 123 Bob 1/19/2013"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); FixedLengthReader parser = new FixedLengthReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema); bool canRead = parser.Read(); Assert.IsTrue(canRead, "Could not read the record."); object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) }; object[] actual = parser.GetValues(); CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed."); actual = parser.GetValues(); CollectionAssert.AreEqual(expected, actual, "The same values were not returned multiple times."); }
public void TestRead_MultipleCallsToValues_ReturnsSameValues() { const string text = @" 123 Bob 1/19/2013"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); StringReader stringReader = new StringReader(text); FixedLengthReader parser = new FixedLengthReader(stringReader, schema); bool canRead = parser.Read(); Assert.IsTrue(canRead, "Could not read the record."); object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) }; object[] actual = parser.GetValues(); CollectionAssert.AreEqual(expected, actual); actual = parser.GetValues(); CollectionAssert.AreEqual(expected, actual); }
public void TestGetValues_CustomFillCharacter_TrimsFill() { FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10) { Alignment = FixedAlignment.LeftAligned }) .AddColumn(new StringColumn("name"), new Window(25) { Alignment = FixedAlignment.LeftAligned }) .AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" }, new Window(10) { Alignment = FixedAlignment.LeftAligned }); FixedLengthOptions options = new FixedLengthOptions() { FillCharacter = '@' }; object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19) }; using (MemoryStream stream = new MemoryStream()) { using (FixedLengthWriter builder = new FixedLengthWriter(stream, schema, options)) { builder.Write(sources); } stream.Position = 0; FixedLengthReader parser = new FixedLengthReader(stream, schema, options); Assert.IsTrue(parser.Read(), "Could not read the first record."); object[] actual = parser.GetValues(); CollectionAssert.AreEqual(sources, actual, "The values for the first record were wrong."); } }
public void TestGetValues_CustomFillCharacter_TrimsFill() { FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10) { Alignment = FixedAlignment.LeftAligned }) .AddColumn(new StringColumn("name"), new Window(25) { Alignment = FixedAlignment.LeftAligned }) .AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" }, new Window(10) { Alignment = FixedAlignment.LeftAligned }); FixedLengthOptions options = new FixedLengthOptions() { FillCharacter = '@' }; object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19) }; StringWriter stringWriter = new StringWriter(); FixedLengthWriter builder = new FixedLengthWriter(stringWriter, schema, options); builder.Write(sources); StringReader stringReader = new StringReader(stringWriter.ToString()); FixedLengthReader parser = new FixedLengthReader(stringReader, schema, options); Assert.True(parser.Read(), "Could not read the first record."); object[] actual = parser.GetValues(); Assert.Equal(sources, actual); }
public void TestReader_ReadThreeTypes() { StringWriter stringWriter = new StringWriter(); var injector = getSchemaInjector(); var options = new FixedLengthOptions() { Alignment = FixedAlignment.RightAligned }; var writer = new FixedLengthWriter(stringWriter, injector, options); writer.Write(new object[] { "First Batch", 2 }); writer.Write(new object[] { 1, "Bob Smith", new DateTime(2018, 06, 04), 12.34m }); writer.Write(new object[] { 2, "Jane Doe", new DateTime(2018, 06, 05), 34.56m }); writer.Write(new object[] { 46.9m, 23.45m, true }); string output = stringWriter.ToString(); Assert.AreEqual(@" First Batch 2 1 Bob Smith 20180604 12.34 2 Jane Doe 20180605 34.56 46.9 23.45 True ", output); var stringReader = new StringReader(output); var selector = getSchemaSelector(); var reader = new FixedLengthReader(stringReader, selector, options); Assert.IsTrue(reader.Read(), "The header record could not be read."); var headerValues = reader.GetValues(); Assert.AreEqual(2, headerValues.Length); Assert.AreEqual("First Batch", headerValues[0]); Assert.AreEqual(2, headerValues[1]); Assert.IsTrue(reader.Read(), "The first data record could not be read."); var dataValues1 = reader.GetValues(); Assert.AreEqual(4, dataValues1.Length); Assert.AreEqual(1, dataValues1[0]); Assert.AreEqual("Bob Smith", dataValues1[1]); Assert.AreEqual(new DateTime(2018, 6, 4), dataValues1[2]); Assert.AreEqual(12.34m, dataValues1[3]); Assert.IsTrue(reader.Read(), "The second data record could not be read."); var dataValues2 = reader.GetValues(); Assert.AreEqual(4, dataValues2.Length); Assert.AreEqual(2, dataValues2[0]); Assert.AreEqual("Jane Doe", dataValues2[1]); Assert.AreEqual(new DateTime(2018, 6, 5), dataValues2[2]); Assert.AreEqual(34.56m, dataValues2[3]); Assert.IsTrue(reader.Read(), "The footer record could not be read."); var footerValues = reader.GetValues(); Assert.AreEqual(3, footerValues.Length); Assert.AreEqual(46.9m, footerValues[0]); Assert.AreEqual(23.45m, footerValues[1]); Assert.AreEqual(true, footerValues[2]); Assert.IsFalse(reader.Read()); }
public void TestRead_RecordWithCP1252Characters_ReturnsCorrectCharacters() { //---- Arrange ----------------------------------------------------- // Need to convert the string to target encoding because otherwise a string declared in VS will always be encoded as UTF-8 var text = Encoding.Convert(Encoding.UTF8, Encoding.GetEncoding(1252), Encoding.UTF8.GetBytes(@" 123 Müller 1/17/2014")); var schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); var options = new FixedLengthOptions() { Encoding = Encoding.GetEncoding(1252) }; var testee = new FixedLengthReader(new MemoryStream(text), schema, options); //---- Act --------------------------------------------------------- var result = testee.Read(); //---- Assert ------------------------------------------------------ Assert.IsTrue(result, "Could not read the record."); object[] expected = { 123, "Müller", new DateTime(2014, 1, 17) }; object[] actual = testee.GetValues(); CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed."); }
public void TestGetValues_NoRecordSeparator_SplitsFile() { const string text = " 123 Bob 1/19/2013 234 Sam12/20/2013"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); FixedLengthOptions options = new FixedLengthOptions() { RecordSeparator = null }; StringReader stringReader = new StringReader(text); FixedLengthReader parser = new FixedLengthReader(stringReader, schema, options); Assert.True(parser.Read(), "Could not read the first record."); object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) }; object[] actual = parser.GetValues(); Assert.Equal(expected, actual); Assert.True(parser.Read(), "Could not read the second record."); expected = new object[] { 234, "Sam", new DateTime(2013, 12, 20) }; actual = parser.GetValues(); Assert.Equal(expected, actual); }
public void TestRead_MultipleCallsToValues_ReturnsSameValues() { const string text = @" 123 Bob 1/19/2013"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); StringReader stringReader = new StringReader(text); FixedLengthReader parser = new FixedLengthReader(stringReader, schema); bool canRead = parser.Read(); Assert.True(canRead, "Could not read the record."); object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) }; object[] actual = parser.GetValues(); Assert.Equal(expected, actual); actual = parser.GetValues(); Assert.Equal(expected, actual); }
public void TestRead_SkipRecord_NoParsingError() { const string text = "a b c"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("A"), 8); schema.AddColumn(new DateTimeColumn("B"), 23); schema.AddColumn(new GuidColumn("C"), 2); StringReader stringReader = new StringReader(text); FixedLengthReader parser = new FixedLengthReader(stringReader, schema); bool canRead = parser.Skip(); Assert.True(canRead, "Could not skip the record."); canRead = parser.Read(); Assert.False(canRead, "No more records should have been read."); }
public void TestRead_ValuesAfterEndOfFile_Throws() { const string text = @" 123 Bob 1/19/2013"; FixedLengthSchema schema = new FixedLengthSchema(); schema.AddColumn(new Int32Column("id"), new Window(10)) .AddColumn(new StringColumn("name"), new Window(25)) .AddColumn(new DateTimeColumn("created"), new Window(10)); StringReader stringReader = new StringReader(text); FixedLengthReader parser = new FixedLengthReader(stringReader, schema); Assert.True(parser.Read(), "Could not read the record."); Assert.False(parser.Read(), "We should have reached the end of the file."); Assert.Throws<InvalidOperationException>(() => parser.GetValues()); }
public Core.ResultDefinition ValidateFile() { var schemaData = string.Empty; var schemaParser = new SchemaParser(); var validationResults = new List <ValidationResult>(); var validator = new Validators.FixedLengthValidator(); if (this.RawSchema == null) { try { using (var sr = new StreamReader(File.OpenRead(this.SchemaFilePath))) { schemaData = sr.ReadToEnd(); } } catch (Exception ex) { var errorMsg = String.Format(null, SharedResources.SchemaError, ex.Message); Console.WriteLine($"{errorMsg}"); } this.RawSchema = schemaParser.ParseSchema(schemaData); } var fixedLengthFileSchemaBuilder = new FixedLengthSchemaBuilder(); var fixedFileSchema = fixedLengthFileSchemaBuilder.BuildSchema(this.RawSchema); var schemas = fixedFileSchema.FixedLengthRecordSchemas; var count = 0; foreach (var inputLine in File.ReadLines(this.FilePath)) { Console.WriteLine($"Started parsing line...{count}"); if (inputLine != null) { FixedLengthSchema actualSchema; TextReader stringReader = new StringReader(inputLine); var schema = schemas.FirstOrDefault(x => inputLine.StartsWith(x.RecordIdentifier)); count++; var innerResults = new List <ValidationResult>(); var line = inputLine; if (schema != null) { var parser = new FixedLengthReader(stringReader, schema.FixedLengthSchema); try { actualSchema = parser.GetSchema(); parser.Read(); var rawValues = GetRawValues(inputLine, schema.FixedLengthSchema); var values = parser.GetValues(); for (int i = 0; i < values.Length; i++) { var validationResult = new ValidationResult { Record = inputLine, ColumnName = actualSchema.ColumnDefinitions[i].ColumnName, ColumnType = actualSchema.ColumnDefinitions[i].ColumnType.Name, ActualRowLength = inputLine.Length, MaxRowLength = actualSchema.TotalWidth, ParsedValueLength = (values[i].ToString() ?? string.Empty).Length, RawValueLength = rawValues[i].ToString().Length, ColumnLength = actualSchema.Windows[i].Width, FillCharacter = actualSchema.Windows[i].FillCharacter != null ? actualSchema.Windows[i].FillCharacter.ToString() : "Not specified", TextAlignment = actualSchema.Windows[i].Alignment != null ? actualSchema.Windows[i].Alignment.ToString() : "Not specified", ParsedValue = values[i].ToString() ?? string.Empty, RawValue = rawValues[i].ToString() ?? string.Empty, RowNumber = count }; var failures = validator.Validate(validationResult); if (failures.Errors != null && failures.Errors.Any() && !failures.IsValid) { var errors = new List <string>(); foreach (var error in failures.Errors) { errors.Add(error.ErrorMessage); } validationResult.ErrorMessages = errors; validationResult.HasErrors = true; line = line.Replace(validationResult.RawValue, $"<strong style='color:{HtmlColors[i%HtmlColors.Length]}'>{validationResult.RawValue}</strong>"); validationResult.Record = line; validationResults.Add(validationResult); } } } catch (Exception ex) { var errorMsg = string.Empty; if (ex.InnerException != null && ex.InnerException.Message != null) { errorMsg = ex.InnerException.Message; Console.WriteLine(ex.InnerException.Message); } else { errorMsg = ex.Message; Console.WriteLine(ex.Message); } var error = new ValidationResult { RowNumber = count, Record = inputLine, ErrorMessages = new List <string> { $"Exception: {errorMsg}" }, HasErrors = true }; validationResults.Add(error); } } else { var error = new ValidationResult { RowNumber = count, Record = inputLine, ErrorMessages = new List <string> { $"Exception: The row does not contain a record identifier that match the schema." }, HasErrors = true }; validationResults.Add(error); } } } var results = new Core.ResultDefinition { Results = validationResults, TotalLinesProcessed = count, FileFormat = this.RawSchema.FileFormat, FilePath = this.FilePath, SchemaPath = this.SchemaFilePath }; return(results); }