public void ShouldTruncateOverflow()
{
MemoryStream stream = new MemoryStream();
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new StringColumn("Default"), new Window(5));
schema.AddColumn(new StringColumn("Leading"), new Window(5) { TruncationPolicy = OverflowTruncationPolicy.TruncateLeading });
schema.AddColumn(new StringColumn("Trailing"), new Window(5) { TruncationPolicy = OverflowTruncationPolicy.TruncateTrailing });
FixedLengthOptions options = new FixedLengthOptions();
options.TruncationPolicy = OverflowTruncationPolicy.TruncateLeading; // this is the default anyway
using (FixedLengthWriter writer = new FixedLengthWriter(stream, schema, options))
{
writer.Write(new object[] { "Pineapple", "Pineapple", "Pineapple" });
}
stream.Position = 0;
string output = Encoding.Default.GetString(stream.ToArray());
string expected = "appleapplePinea" + Environment.NewLine;
Assert.AreEqual(expected, output, "The values were not truncated properly.");
}
public void TestReader_WrappedWithIgnoredColumns()
{
var mapper = FixedLengthTypeMapper.Define(() => new ComplicatedPerson());
mapper.Property(x => x.PersonId, 10);
mapper.Ignored(1);
mapper.Property(x => x.Name, 10);
mapper.Ignored(1);
mapper.Property(x => x.CreatedOn, 10).OutputFormat("MM/dd/yyyy");
var people = new[]
{
new ComplicatedPerson()
{
PersonId = 1, Name = "Bob", CreatedOn = new DateTime(2018, 04, 25)
},
new ComplicatedPerson()
{
PersonId = 2, Name = "Tom", CreatedOn = new DateTime(2018, 04, 26)
},
new ComplicatedPerson()
{
PersonId = 3, Name = "Jane", CreatedOn = new DateTime(2018, 04, 27)
}
};
StringWriter writer = new StringWriter();
mapper.Write(writer, people, new FixedLengthOptions()
{
IsFirstRecordHeader = true
});
string output = writer.ToString();
mapper.CustomMapping(new RecordNumberColumn("RecordNumber"), 10)
.WithReader((p, v) => p.RecordNumber = (int)v)
.WithWriter(p => p.RecordNumber);
StringReader stringReader = new StringReader(output);
var options = new FixedLengthOptions()
{
IsFirstRecordHeader = true
};
var reader = mapper.GetReader(stringReader, options);
reader.RecordPartitioned += (sender, e) =>
{
e.IsSkipped = e.Values.Length >= 2 && e.Values[2] == "Tom";
};
var results = reader.ReadAll().ToArray();
Assert.AreEqual(2, results.Length);
Assert.AreEqual(1, results[0].PersonId);
Assert.AreEqual("Bob", results[0].Name);
Assert.AreEqual(new DateTime(2018, 04, 25), results[0].CreatedOn);
Assert.AreEqual(1, results[0].RecordNumber);
Assert.AreEqual(3, results[1].PersonId);
Assert.AreEqual("Jane", results[1].Name);
Assert.AreEqual(new DateTime(2018, 04, 27), results[1].CreatedOn);
Assert.AreEqual(2, results[1].RecordNumber);
}
public void TestTypeMapper_Roundtrip()
{
var mapper = FixedLengthTypeMapper.Define <Person>();
mapper.Property(p => p.Id, new Window(25)).ColumnName("id");
mapper.Property(p => p.Name, new Window(100)).ColumnName("name");
mapper.Property(p => p.Created, new Window(8)).ColumnName("created").InputFormat("yyyyMMdd").OutputFormat("yyyyMMdd");
mapper.Property(p => p.IsActive, new Window(5)).ColumnName("active");
var bob = new Person()
{
Id = 123, Name = "Bob", Created = new DateTime(2013, 1, 19), IsActive = true
};
var options = new FixedLengthOptions()
{
FillCharacter = '@'
};
StringWriter stringWriter = new StringWriter();
mapper.Write(stringWriter, new Person[] { bob }, options);
StringReader stringReader = new StringReader(stringWriter.ToString());
var people = mapper.Read(stringReader, options).ToArray();
Assert.AreEqual(1, people.Length);
var person = people.SingleOrDefault();
Assert.AreEqual(bob.Id, person.Id);
Assert.AreEqual(bob.Name, person.Name);
Assert.AreEqual(bob.Created, person.Created);
}
public void ShouldWriteHeader()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new StringColumn("First"), new Window(10)
{
FillCharacter = '@'
});
schema.AddColumn(new StringColumn("Second"), new Window(10)
{
FillCharacter = '!'
});
schema.AddColumn(new StringColumn("Third"), new Window(10)
{
FillCharacter = '$'
});
FixedLengthOptions options = new FixedLengthOptions()
{
IsFirstRecordHeader = true
};
StringWriter stringWriter = new StringWriter();
FixedLengthWriter writer = new FixedLengthWriter(stringWriter, schema, options);
writer.Write(new object[] { "Apple", "Grape", "Pear" });
string output = stringWriter.ToString();
string expected = "First@@@@@Second!!!!Third$$$$$"
+ Environment.NewLine
+ "Apple@@@@@Grape!!!!!Pear$$$$$$"
+ Environment.NewLine;
Assert.AreEqual(expected, output);
}
public void TestGetValues_NoRecordSeparator_SplitsFile()
{
const string text = " 123 Bob 1/19/2013 234 Sam12/20/2013";
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10))
.AddColumn(new StringColumn("name"), new Window(25))
.AddColumn(new DateTimeColumn("created"), new Window(10));
FixedLengthOptions options = new FixedLengthOptions()
{
RecordSeparator = null
};
StringReader stringReader = new StringReader(text);
FixedLengthReader parser = new FixedLengthReader(stringReader, schema, options);
Assert.True(parser.Read(), "Could not read the first record.");
object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
object[] actual = parser.GetValues();
Assert.Equal(expected, actual);
Assert.True(parser.Read(), "Could not read the second record.");
expected = new object[] { 234, "Sam", new DateTime(2013, 12, 20) };
actual = parser.GetValues();
Assert.Equal(expected, actual);
}
public void TestCtor_Options_TextNull_Throws()
{
TextReader reader = null;
FixedLengthSchema schema = new FixedLengthSchema();
FixedLengthOptions options = new FixedLengthOptions();
Assert.Throws<ArgumentNullException>(() => new FixedLengthReader(reader, schema, options));
}
public void TestTypeWriter_WriteAnonymous()
{
var people = from id in Enumerable.Range(0, 1)
select new
{
Id = id,
Name = "Bob " + id,
Created = new DateTime(2013, 1, 19)
};
var mapper = FixedLengthTypeMapper.DefineWriter(people);
mapper.Property(p => p.Id, 10).ColumnName("id");
mapper.Property(p => p.Name, 100).ColumnName("name");
mapper.Property(p => p.Created, 8).ColumnName("created").InputFormat("yyyyMMdd").OutputFormat("yyyyMMdd");
using (MemoryStream stream = new MemoryStream())
{
var options = new FixedLengthOptions() { FillCharacter = '@', RecordSeparator = "\n" };
mapper.Write(stream, options);
stream.Position = 0; // go back to the beginning of the stream
FixedLengthSchema schema = mapper.GetSchema();
FlatFileReader reader = new FlatFileReader(new FixedLengthReader(stream, schema, options));
Assert.IsTrue(reader.Read(), "The writer did not write the entities.");
int id = reader.GetInt32(0);
string name = reader.GetString(1);
DateTime created = reader.GetDateTime(2);
Assert.AreEqual(people.First().Id, id, "The ID value was not persisted.");
Assert.AreEqual(people.First().Name, name, "The Name value was not persisted.");
Assert.AreEqual(people.First().Created, created, "The Created value was not persisted.");
Assert.IsFalse(reader.Read(), "The writer wrote too many records.");
}
}
public void TestTypeMapper_RoundtripWithNull()
{
var mapper = FixedLengthTypeMapper.Define <Person>();
mapper.Property(p => p.Id, new Window(25)).ColumnName("id");
mapper.Property(p => p.Name, new Window(100)).ColumnName("name");
mapper.Property(p => p.Created, new Window(8)).ColumnName("created").InputFormat("yyyyMMdd").OutputFormat("yyyyMMdd");
using (MemoryStream stream = new MemoryStream())
{
var bob = new Person()
{
Id = 123, Name = null, Created = new DateTime(2013, 1, 19)
};
var options = new FixedLengthOptions()
{
FillCharacter = '@'
};
mapper.Write(stream, options, new Person[] { bob });
stream.Position = 0; // go back to the beginning of the stream
var people = mapper.Read(stream, options);
Assert.AreEqual(1, people.Count(), "The wrong number of people were returned.");
var person = people.SingleOrDefault();
Assert.AreEqual(bob.Id, person.Id, "The ID value was not persisted.");
Assert.AreEqual(bob.Name, person.Name, "The Name value was not persisted.");
Assert.AreEqual(bob.Created, person.Created, "The Created value was not persisted.");
}
}
public void TestRead_RecordWithCP1251Characters_ReturnsCorrectCharacters()
{
//---- Arrange -----------------------------------------------------
// Need to convert the string to target encoding because otherwise a string declared in VS will always be encoded as UTF-8
var text = Encoding.Convert(Encoding.UTF8, Encoding.GetEncoding(1251), Encoding.UTF8.GetBytes(@" 123 Лучиано 1/17/2014"));
var schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10))
.AddColumn(new StringColumn("name"), new Window(25))
.AddColumn(new DateTimeColumn("created"), new Window(10));
var options = new FixedLengthOptions()
{
Encoding = Encoding.GetEncoding(1251)
};
var testee = new FixedLengthReader(new MemoryStream(text), schema, options);
//---- Act ---------------------------------------------------------
var result = testee.Read();
//---- Assert ------------------------------------------------------
Assert.IsTrue(result, "Could not read the record.");
object[] expected = { 123, "Лучиано", new DateTime(2014, 1, 17) };
object[] actual = testee.GetValues();
CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed.");
}
public void ShouldTruncateOverflow()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new StringColumn("Default"), new Window(5));
schema.AddColumn(new StringColumn("Leading"), new Window(5)
{
TruncationPolicy = OverflowTruncationPolicy.TruncateLeading
});
schema.AddColumn(new StringColumn("Trailing"), new Window(5)
{
TruncationPolicy = OverflowTruncationPolicy.TruncateTrailing
});
FixedLengthOptions options = new FixedLengthOptions
{
TruncationPolicy = OverflowTruncationPolicy.TruncateLeading // this is the default anyway
};
StringWriter stringWriter = new StringWriter();
FixedLengthWriter writer = new FixedLengthWriter(stringWriter, schema, options);
writer.Write(new object[] { "Pineapple", "Pineapple", "Pineapple" });
string output = stringWriter.ToString();
string expected = "appleapplePinea" + Environment.NewLine;
Assert.AreEqual(expected, output);
}
public FixedLengthResult(IFixedLengthTypeMapper <T> mapper, IEnumerable <T> data, FixedLengthOptions options = null, string fileName = null, string contentType = null)
{
if (mapper == null)
{
throw new ArgumentNullException("mapper");
}
if (data == null)
{
throw new ArgumentNullException("data");
}
if (options == null)
{
options = new FixedLengthOptions();
}
if (String.IsNullOrWhiteSpace(fileName))
{
fileName = "file.txt";
}
if (contentType == null)
{
contentType = "text/plain";
}
this.writer = mapper.ToWriter(data);
this.options = options;
this.fileName = fileName;
this.contentType = contentType;
}
public void TestCtor_Options_TextNull_Throws()
{
Stream stream = null;
FixedLengthSchema schema = new FixedLengthSchema();
FixedLengthOptions options = new FixedLengthOptions();
new FixedLengthReader(stream, schema, options);
}
public void TestCtor_Options_SchemaNull_Throws()
{
string text = String.Empty;
FixedLengthSchema schema = null;
FixedLengthOptions options = new FixedLengthOptions();
new FixedLengthReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema, options);
}
public void TestCtor_OptionsNull_Throws()
{
string text = String.Empty;
FixedLengthSchema schema = new FixedLengthSchema();
FixedLengthOptions options = null;
new FixedLengthReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema, options);
}
public void TestCtor_Options_TextNull_Throws()
{
Stream stream = null;
FixedLengthSchema schema = new FixedLengthSchema();
FixedLengthOptions options = new FixedLengthOptions();
new FixedLengthReader(stream, schema, options);
}
public void TestCtor_Options_TextNull_Throws()
{
TextReader reader = null;
FixedLengthSchema schema = new FixedLengthSchema();
FixedLengthOptions options = new FixedLengthOptions();
Assert.ThrowsException <ArgumentNullException>(() => new FixedLengthReader(reader, schema, options));
}
public void TestReader_WithSchema_WithIgnoredColumn_NoRecordSeparator_WithFilter_LogicalRecordsOnly()
{
var mapper = FixedLengthTypeMapper.Define(() => new Person());
mapper.Property(x => x.Name, 10);
mapper.Ignored(1);
var people = new[]
{
new Person()
{
Name = "Bob"
},
new Person()
{
Name = "Tom"
},
new Person()
{
Name = "Jane"
}
};
StringWriter writer = new StringWriter();
mapper.Write(writer, people, new FixedLengthOptions()
{
IsFirstRecordHeader = true,
HasRecordSeparator = false
});
string output = writer.ToString();
mapper.CustomMapping(new RecordNumberColumn("RecordNumber"), 10)
.WithReader((p, v) => p.RecordNumber = (int)v)
.WithWriter(p => p.RecordNumber);
StringReader stringReader = new StringReader(output);
var options = new FixedLengthOptions()
{
IsFirstRecordHeader = true,
HasRecordSeparator = false
};
var reader = mapper.GetReader(stringReader, options);
reader.RecordPartitioned += (sender, e) =>
{
e.IsSkipped = e.Values.Length >= 1 && e.Values[0] == "Tom";
};
var results = reader.ReadAll().ToArray();
Assert.AreEqual(2, results.Length);
Assert.AreEqual("Bob", results[0].Name);
Assert.AreEqual(1, results[0].RecordNumber);
Assert.AreEqual("Jane", results[1].Name);
Assert.AreEqual(2, results[1].RecordNumber);
}
public void TestReader_WithSchema_SchemaNotCounted_WithFilter_LineCount()
{
var mapper = new FixedLengthTypeMapper <Person>(() => new Person());
mapper.Property(x => x.Name, 10);
var people = new[]
{
new Person()
{
Name = "Bob"
},
new Person()
{
Name = "Tom"
},
new Person()
{
Name = "Jane"
}
};
StringWriter writer = new StringWriter();
mapper.Write(writer, people, new FixedLengthOptions()
{
IsFirstRecordHeader = true
});
string output = writer.ToString();
mapper.CustomProperty(x => x.RecordNumber, new RecordNumberColumn("RecordNumber")
{
IncludeSchema = false,
IncludeFilteredRecords = true
}, 10);
StringReader stringReader = new StringReader(output);
var options = new FixedLengthOptions()
{
IsFirstRecordHeader = true
};
var reader = mapper.GetReader(stringReader, options);
reader.RecordPartitioned += (sender, e) =>
{
e.IsSkipped = e.Values.Length == 1 && e.Values[0] == "Tom";
};
var results = reader.ReadAll().ToArray();
Assert.AreEqual(2, results.Length);
Assert.AreEqual("Bob", results[0].Name);
Assert.AreEqual(1, results[0].RecordNumber);
Assert.AreEqual("Jane", results[1].Name);
Assert.AreEqual(3, results[1].RecordNumber);
}
public void TestReader_WrappedWithIgnoredColumns()
{
var mapper = new FixedLengthTypeMapper <ComplicatedPerson>(() => new ComplicatedPerson());
mapper.Property(x => x.PersonId, 10);
mapper.Ignored(1);
mapper.Property(x => x.Name, 10);
mapper.Ignored(1);
mapper.Property(x => x.CreatedOn, 10).OutputFormat("MM/dd/yyyy");
var people = new[]
{
new ComplicatedPerson()
{
PersonId = 1, Name = "Bob", CreatedOn = new DateTime(2018, 04, 25)
},
new ComplicatedPerson()
{
PersonId = 2, Name = "Tom", CreatedOn = new DateTime(2018, 04, 26)
},
new ComplicatedPerson()
{
PersonId = 3, Name = "Jane", CreatedOn = new DateTime(2018, 04, 27)
}
};
StringWriter writer = new StringWriter();
mapper.Write(writer, people, new FixedLengthOptions()
{
IsFirstRecordHeader = true
});
string output = writer.ToString();
mapper.CustomProperty(x => x.RecordNumber, new RecordNumberColumn("RecordNumber"), 10);
StringReader reader = new StringReader(output);
var options = new FixedLengthOptions()
{
IsFirstRecordHeader = true,
PartitionedRecordFilter = (values) => values.Length >= 2 && values[2] == "Tom"
};
var results = mapper.Read(reader, options).ToArray();
Assert.Equal(2, results.Length);
Assert.Equal(1, results[0].PersonId);
Assert.Equal("Bob", results[0].Name);
Assert.Equal(new DateTime(2018, 04, 25), results[0].CreatedOn);
Assert.Equal(1, results[0].RecordNumber);
Assert.Equal(3, results[1].PersonId);
Assert.Equal("Jane", results[1].Name);
Assert.Equal(new DateTime(2018, 04, 27), results[1].CreatedOn);
Assert.Equal(2, results[1].RecordNumber);
}
public void TestReader_WithSchema_WithFilter_LineCount()
{
var mapper = new FixedLengthTypeMapper <Person>(() => new Person());
mapper.Property(x => x.Name, 10);
var people = new[]
{
new Person()
{
Name = "Bob"
},
new Person()
{
Name = "Tom"
},
new Person()
{
Name = "Jane"
}
};
StringWriter writer = new StringWriter();
mapper.Write(writer, people, new FixedLengthOptions()
{
IsFirstRecordHeader = true
});
string output = writer.ToString();
mapper.CustomProperty(x => x.RecordNumber, new RecordNumberColumn("RecordNumber")
{
IncludeSchema = true,
IncludeFilteredRecords = true
}, 10);
StringReader reader = new StringReader(output);
var options = new FixedLengthOptions()
{
IsFirstRecordHeader = true,
PartitionedRecordFilter = (values) => values.Length == 1 && values[0] == "Tom"
};
var results = mapper.Read(reader, options).ToArray();
Assert.Equal(2, results.Length);
Assert.Equal("Bob", results[0].Name);
Assert.Equal(2, results[0].RecordNumber);
Assert.Equal("Jane", results[1].Name);
Assert.Equal(4, results[1].RecordNumber);
}
public void TestReader_WithSchema_WithIgnoredColumn_NoRecordSeparator_WithFilter_LogicalRecordsOnly()
{
var mapper = new FixedLengthTypeMapper <Person>(() => new Person());
mapper.Property(x => x.Name, 10);
mapper.Ignored(1);
var people = new[]
{
new Person()
{
Name = "Bob"
},
new Person()
{
Name = "Tom"
},
new Person()
{
Name = "Jane"
}
};
StringWriter writer = new StringWriter();
mapper.Write(writer, people, new FixedLengthOptions()
{
IsFirstRecordHeader = true,
HasRecordSeparator = false
});
string output = writer.ToString();
mapper.CustomProperty(x => x.RecordNumber, new RecordNumberColumn("RecordNumber"), 10);
StringReader reader = new StringReader(output);
var options = new FixedLengthOptions()
{
IsFirstRecordHeader = true,
HasRecordSeparator = false,
PartitionedRecordFilter = (values) => values.Length >= 1 && values[0] == "Tom"
};
var results = mapper.Read(reader, options).ToArray();
Assert.Equal(2, results.Length);
Assert.Equal("Bob", results[0].Name);
Assert.Equal(1, results[0].RecordNumber);
Assert.Equal("Jane", results[1].Name);
Assert.Equal(2, results[1].RecordNumber);
}
/// <summary>
/// Gets a typed writer for writing the objects to the file.
/// </summary>
/// <param name="writer">The writer to use.</param>
/// <param name="options">The separate value options to use.</param>
/// <returns>The typed writer.</returns>
public ITypedWriter <object> GetWriter(TextWriter writer, FixedLengthOptions options = null)
{
var injector = new FixedLengthSchemaInjector();
var valueWriter = new FixedLengthWriter(writer, injector, options);
var multiWriter = new MultiplexingTypedWriter(valueWriter, this);
foreach (var matcher in matchers)
{
injector.When((values) => matcher.IsMatch).Use(matcher.TypeMapper.GetSchema());
}
if (defaultMatcher != nonMatcher)
{
injector.WithDefault(defaultMatcher.TypeMapper.GetSchema());
}
return(multiWriter);
}
/// <summary>
/// Gets a typed reader for reading the objects from the file.
/// </summary>
/// <param name="reader">The reader to use.</param>
/// <param name="options">The separate value options to use.</param>
/// <returns>The typed reader.</returns>
public IFixedLengthTypedReader <object> GetReader(TextReader reader, FixedLengthOptions options = null)
{
var selector = new FixedLengthSchemaSelector();
var valueReader = new FixedLengthReader(reader, selector, options);
var multiReader = new MultiplexingFixedLengthTypedReader(valueReader);
foreach (var matcher in matchers)
{
var typedReader = new Lazy <Func <IRecordContext, object[], object> >(GetReader(matcher.TypeMapper));
selector.When(matcher.Predicate).Use(matcher.TypeMapper.GetSchema()).OnMatch(() => multiReader.Deserializer = typedReader.Value);
}
if (defaultMapper != null)
{
var typeReader = new Lazy <Func <IRecordContext, object[], object> >(GetReader(defaultMapper));
selector.WithDefault(defaultMapper.GetSchema()).OnMatch(() => multiReader.Deserializer = typeReader.Value);
}
return(multiReader);
}
public void TestTypeMapper_DefaultRecordSeparator_Intermixed()
{
var mapper = FixedLengthTypeMapper.Define <Person>();
mapper.Property(p => p.Id, new Window(25)).ColumnName("id");
mapper.Property(p => p.Name, new Window(100)).ColumnName("name");
mapper.Property(p => p.Created, new Window(8)).ColumnName("created").InputFormat("yyyyMMdd").OutputFormat("yyyyMMdd");
string rawData = "123 Bob 20130119\r\n234 Sam 20130119\r345 Ron 20130119\n456 Carl 20130119\r\n";
StringReader stringReader = new StringReader(rawData);
var options = new FixedLengthOptions()
{
HasRecordSeparator = true, RecordSeparator = null
};
var people = mapper.Read(stringReader, options).ToArray();
Assert.AreEqual(4, people.Count());
}
public void TestGetValues_WithPartitionedRecordFilter_SkipRecordsMatchingCriteria()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10)
{
Alignment = FixedAlignment.RightAligned
})
.AddColumn(new StringColumn("name"), new Window(25)
{
Alignment = FixedAlignment.RightAligned
})
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "M/d/yyyy"
}, new Window(10)
{
Alignment = FixedAlignment.RightAligned
});
FixedLengthOptions options = new FixedLengthOptions()
{
PartitionedRecordFilter = (parts) => parts.Length == 3 && parts[0].StartsWith("-")
};
const string lines = @" 123 Bob Smith 4/21/2017
-1 Jay Smith 8/14/2017
234 Jay Smith 5/21/2017";
StringReader stringReader = new StringReader(lines);
FixedLengthReader parser = new FixedLengthReader(stringReader, schema, options);
Assert.True(parser.Read(), "Could not read the first record.");
object[] actual1 = parser.GetValues();
Assert.Equal(new object[] { 123, "Bob Smith", new DateTime(2017, 04, 21) }, actual1);
Assert.True(parser.Read(), "Could not read the second record.");
object[] actual2 = parser.GetValues();
Assert.Equal(new object[] { 234, "Jay Smith", new DateTime(2017, 05, 21) }, actual2);
Assert.False(parser.Read(), "There should not be any more records.");
}
public void TestGetValues_CustomFillCharacter_TrimsFill()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10)
{
Alignment = FixedAlignment.LeftAligned
})
.AddColumn(new StringColumn("name"), new Window(25)
{
Alignment = FixedAlignment.LeftAligned
})
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy"
}, new Window(10)
{
Alignment = FixedAlignment.LeftAligned
});
FixedLengthOptions options = new FixedLengthOptions()
{
FillCharacter = '@'
};
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
using (MemoryStream stream = new MemoryStream())
{
using (FixedLengthWriter builder = new FixedLengthWriter(stream, schema, options))
{
builder.Write(sources);
}
stream.Position = 0;
FixedLengthReader parser = new FixedLengthReader(stream, schema, options);
Assert.IsTrue(parser.Read(), "Could not read the first record.");
object[] actual = parser.GetValues();
CollectionAssert.AreEqual(sources, actual, "The values for the first record were wrong.");
}
}
public void TestTypeMapper_IgnoredSeparators_RoundTrip()
{
var mapper = FixedLengthTypeMapper.Define <Person>();
mapper.Property(p => p.Id, new Window(25)).ColumnName("id");
mapper.Ignored(new Window(1)
{
FillCharacter = '|'
});
mapper.Property(p => p.Name, new Window(100)).ColumnName("name");
mapper.Ignored(new Window(1)
{
FillCharacter = '|'
});
mapper.Property(p => p.Created, new Window(8)).ColumnName("created").InputFormat("yyyyMMdd").OutputFormat("yyyyMMdd");
var bob = new Person()
{
Id = 123, Name = "Bob Smith", Created = new DateTime(2013, 1, 19)
};
var options = new FixedLengthOptions()
{
FillCharacter = ' '
};
StringWriter stringWriter = new StringWriter();
mapper.Write(stringWriter, new Person[] { bob }, options);
StringReader stringReader = new StringReader(stringWriter.ToString());
var people = mapper.Read(stringReader, options).ToArray();
Assert.Single(people);
var person = people.SingleOrDefault();
Assert.Equal(bob.Id, person.Id);
Assert.Equal(bob.Name, person.Name);
Assert.Equal(bob.Created, person.Created);
}
public void TestGetValues_CustomFillCharacter_TrimsFill()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10)
{
Alignment = FixedAlignment.LeftAligned
})
.AddColumn(new StringColumn("name"), new Window(25)
{
Alignment = FixedAlignment.LeftAligned
})
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy"
}, new Window(10)
{
Alignment = FixedAlignment.LeftAligned
});
FixedLengthOptions options = new FixedLengthOptions()
{
FillCharacter = '@'
};
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
StringWriter stringWriter = new StringWriter();
FixedLengthWriter builder = new FixedLengthWriter(stringWriter, schema, options);
builder.Write(sources);
StringReader stringReader = new StringReader(stringWriter.ToString());
FixedLengthReader parser = new FixedLengthReader(stringReader, schema, options);
Assert.True(parser.Read(), "Could not read the first record.");
object[] actual = parser.GetValues();
Assert.Equal(sources, actual);
}
public FixedLengthResult(IFixedLengthTypeWriter <T> writer, FixedLengthOptions options = null, string fileName = null, string contentType = null)
{
if (writer == null)
{
throw new ArgumentNullException("writer");
}
if (options == null)
{
options = new FixedLengthOptions();
}
if (String.IsNullOrWhiteSpace(fileName))
{
fileName = "file.txt";
}
if (contentType == null)
{
contentType = "text/plain";
}
this.writer = writer;
this.options = options;
this.fileName = fileName;
this.contentType = contentType;
}
public void TestTypeWriter_WriteAnonymous()
{
var people = from id in Enumerable.Range(0, 1)
select new
{
Id = id,
Name = "Bob " + id,
Created = new DateTime(2013, 1, 19)
};
var mapper = FixedLengthTypeMapper.DefineWriter(people);
mapper.Property(p => p.Id, 10).ColumnName("id");
mapper.Property(p => p.Name, 100).ColumnName("name");
mapper.Property(p => p.Created, 8).ColumnName("created").InputFormat("yyyyMMdd").OutputFormat("yyyyMMdd");
using (MemoryStream stream = new MemoryStream())
{
var options = new FixedLengthOptions()
{
FillCharacter = '@', RecordSeparator = "\n"
};
mapper.Write(stream, options);
stream.Position = 0; // go back to the beginning of the stream
FixedLengthSchema schema = mapper.GetSchema();
FlatFileReader reader = new FlatFileReader(new FixedLengthReader(stream, schema, options));
Assert.IsTrue(reader.Read(), "The writer did not write the entities.");
int id = reader.GetInt32(0);
string name = reader.GetString(1);
DateTime created = reader.GetDateTime(2);
Assert.AreEqual(people.First().Id, id, "The ID value was not persisted.");
Assert.AreEqual(people.First().Name, name, "The Name value was not persisted.");
Assert.AreEqual(people.First().Created, created, "The Created value was not persisted.");
Assert.IsFalse(reader.Read(), "The writer wrote too many records.");
}
}
public async Task TestTypeMapper_Roundtrip_NoSeparator()
{
var mapper = FixedLengthTypeMapper.Define <Person>();
mapper.Property(p => p.Id, new Window(25)).ColumnName("id");
mapper.Property(p => p.Name, new Window(100)).ColumnName("name");
mapper.Property(p => p.Created, new Window(8)).ColumnName("created").InputFormat("yyyyMMdd").OutputFormat("yyyyMMdd");
mapper.Property(p => p.IsActive, new Window(5)).ColumnName("active");
var bob = new Person()
{
Id = 123, Name = "Bob", Created = new DateTime(2013, 1, 19), IsActive = true
};
var options = new FixedLengthOptions()
{
HasRecordSeparator = false
};
StringWriter stringWriter = new StringWriter();
await mapper.WriteAsync(stringWriter, new Person[] { bob, bob }, options);
StringReader stringReader = new StringReader(stringWriter.ToString());
var reader = mapper.GetReader(stringReader, options);
var people = new List <Person>();
while (await reader.ReadAsync())
{
people.Add(reader.Current);
}
Assert.Equal(2, people.Count());
var person1 = people.First();
Assert.Equal(bob.Id, person1.Id);
Assert.Equal(bob.Name, person1.Name);
Assert.Equal(bob.Created, person1.Created);
Assert.True(person1.IsActive);
}
public void TestGetValues_CustomRecordSeparator_SplitsFile()
{
const string text = " 123 Bob 1/19/2013BOOM 234 Sam12/20/2013";
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10))
.AddColumn(new StringColumn("name"), new Window(25))
.AddColumn(new DateTimeColumn("created"), new Window(10));
FixedLengthOptions options = new FixedLengthOptions()
{
RecordSeparator = "BOOM"
};
FixedLengthReader parser = new FixedLengthReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema, options);
Assert.IsTrue(parser.Read(), "Could not read the first record.");
object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
object[] actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The values for the first record were wrong.");
Assert.IsTrue(parser.Read(), "Could not read the second record.");
expected = new object[] { 234, "Sam", new DateTime(2013, 12, 20) };
actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The values for the second record were wrong.");
}
public void TestReader_ReadThreeTypes()
{
StringWriter stringWriter = new StringWriter();
var injector = getSchemaInjector();
var options = new FixedLengthOptions()
{
Alignment = FixedAlignment.RightAligned
};
var writer = new FixedLengthWriter(stringWriter, injector, options);
writer.Write(new object[] { "First Batch", 2 });
writer.Write(new object[] { 1, "Bob Smith", new DateTime(2018, 06, 04), 12.34m });
writer.Write(new object[] { 2, "Jane Doe", new DateTime(2018, 06, 05), 34.56m });
writer.Write(new object[] { 46.9m, 23.45m, true });
string output = stringWriter.ToString();
Assert.AreEqual(@" First Batch 2
1 Bob Smith 20180604 12.34
2 Jane Doe 20180605 34.56
46.9 23.45 True
", output);
var stringReader = new StringReader(output);
var selector = getSchemaSelector();
var reader = new FixedLengthReader(stringReader, selector, options);
Assert.IsTrue(reader.Read(), "The header record could not be read.");
var headerValues = reader.GetValues();
Assert.AreEqual(2, headerValues.Length);
Assert.AreEqual("First Batch", headerValues[0]);
Assert.AreEqual(2, headerValues[1]);
Assert.IsTrue(reader.Read(), "The first data record could not be read.");
var dataValues1 = reader.GetValues();
Assert.AreEqual(4, dataValues1.Length);
Assert.AreEqual(1, dataValues1[0]);
Assert.AreEqual("Bob Smith", dataValues1[1]);
Assert.AreEqual(new DateTime(2018, 6, 4), dataValues1[2]);
Assert.AreEqual(12.34m, dataValues1[3]);
Assert.IsTrue(reader.Read(), "The second data record could not be read.");
var dataValues2 = reader.GetValues();
Assert.AreEqual(4, dataValues2.Length);
Assert.AreEqual(2, dataValues2[0]);
Assert.AreEqual("Jane Doe", dataValues2[1]);
Assert.AreEqual(new DateTime(2018, 6, 5), dataValues2[2]);
Assert.AreEqual(34.56m, dataValues2[3]);
Assert.IsTrue(reader.Read(), "The footer record could not be read.");
var footerValues = reader.GetValues();
Assert.AreEqual(3, footerValues.Length);
Assert.AreEqual(46.9m, footerValues[0]);
Assert.AreEqual(23.45m, footerValues[1]);
Assert.AreEqual(true, footerValues[2]);
Assert.IsFalse(reader.Read());
}
public void TestRead_RecordWithCP1252Characters_ReturnsCorrectCharacters()
{
//---- Arrange -----------------------------------------------------
// Need to convert the string to target encoding because otherwise a string declared in VS will always be encoded as UTF-8
var text = Encoding.Convert(Encoding.UTF8, Encoding.GetEncoding(1252), Encoding.UTF8.GetBytes(@" 123 Müller 1/17/2014"));
var schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10))
.AddColumn(new StringColumn("name"), new Window(25))
.AddColumn(new DateTimeColumn("created"), new Window(10));
var options = new FixedLengthOptions() { Encoding = Encoding.GetEncoding(1252) };
var testee = new FixedLengthReader(new MemoryStream(text), schema, options);
//---- Act ---------------------------------------------------------
var result = testee.Read();
//---- Assert ------------------------------------------------------
Assert.IsTrue(result, "Could not read the record.");
object[] expected = { 123, "Müller", new DateTime(2014, 1, 17) };
object[] actual = testee.GetValues();
CollectionAssert.AreEqual(expected, actual, "The wrong values were parsed.");
}
public void TestTypeMapper_RoundtripWithNull()
{
var mapper = FixedLengthTypeMapper.Define<Person>();
mapper.Property(p => p.Id, new Window(25)).ColumnName("id");
mapper.Property(p => p.Name, new Window(100)).ColumnName("name");
mapper.Property(p => p.Created, new Window(8)).ColumnName("created").InputFormat("yyyyMMdd").OutputFormat("yyyyMMdd");
using (MemoryStream stream = new MemoryStream())
{
var bob = new Person() { Id = 123, Name = null, Created = new DateTime(2013, 1, 19) };
var options = new FixedLengthOptions() { FillCharacter = '@' };
mapper.Write(stream, options, new Person[] { bob });
stream.Position = 0; // go back to the beginning of the stream
var people = mapper.Read(stream, options);
Assert.AreEqual(1, people.Count(), "The wrong number of people were returned.");
var person = people.SingleOrDefault();
Assert.AreEqual(bob.Id, person.Id, "The ID value was not persisted.");
Assert.AreEqual(bob.Name, person.Name, "The Name value was not persisted.");
Assert.AreEqual(bob.Created, person.Created, "The Created value was not persisted.");
}
}
public void TestGetValues_CustomRecordSeparator_SplitsFile()
{
const string text = " 123 Bob 1/19/2013BOOM 234 Sam12/20/2013";
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10))
.AddColumn(new StringColumn("name"), new Window(25))
.AddColumn(new DateTimeColumn("created"), new Window(10));
FixedLengthOptions options = new FixedLengthOptions() { RecordSeparator = "BOOM" };
FixedLengthReader parser = new FixedLengthReader(new MemoryStream(Encoding.Default.GetBytes(text)), schema, options);
Assert.IsTrue(parser.Read(), "Could not read the first record.");
object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
object[] actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The values for the first record were wrong.");
Assert.IsTrue(parser.Read(), "Could not read the second record.");
expected = new object[] { 234, "Sam", new DateTime(2013, 12, 20) };
actual = parser.GetValues();
CollectionAssert.AreEqual(expected, actual, "The values for the second record were wrong.");
}
public void ShouldUseLeadingTruncationByDefault()
{
FixedLengthOptions options = new FixedLengthOptions();
Assert.AreEqual(OverflowTruncationPolicy.TruncateLeading, options.TruncationPolicy, "TruncateLeading not the default policy.");
}
public void TestGetValues_NoRecordSeparator_SplitsFile()
{
const string text = " 123 Bob 1/19/2013 234 Sam12/20/2013";
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10))
.AddColumn(new StringColumn("name"), new Window(25))
.AddColumn(new DateTimeColumn("created"), new Window(10));
FixedLengthOptions options = new FixedLengthOptions() { RecordSeparator = null };
StringReader stringReader = new StringReader(text);
FixedLengthReader parser = new FixedLengthReader(stringReader, schema, options);
Assert.True(parser.Read(), "Could not read the first record.");
object[] expected = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
object[] actual = parser.GetValues();
Assert.Equal(expected, actual);
Assert.True(parser.Read(), "Could not read the second record.");
expected = new object[] { 234, "Sam", new DateTime(2013, 12, 20) };
actual = parser.GetValues();
Assert.Equal(expected, actual);
}
public void TestTypeMapper_RoundTrip_SkipHeaderRow()
{
var mapper = FixedLengthTypeMapper.Define<Person>();
mapper.Property(p => p.Id, new Window(25)).ColumnName("id");
mapper.Property(p => p.Name, new Window(100)).ColumnName("name");
mapper.Property(p => p.Created, new Window(8)).ColumnName("created").InputFormat("yyyyMMdd").OutputFormat("yyyyMMdd");
var bob = new Person() { Id = 123, Name = "Bob", Created = new DateTime(2013, 1, 19) };
var options = new FixedLengthOptions() { IsFirstRecordHeader = true, FillCharacter = '@' };
StringWriter stringWriter = new StringWriter();
mapper.Write(stringWriter, new Person[] { bob }, options);
StringReader stringReader = new StringReader(stringWriter.ToString());
var people = mapper.Read(stringReader, options).ToArray();
Assert.Equal(1, people.Count());
var person = people.SingleOrDefault();
Assert.Equal(bob.Id, person.Id);
Assert.Equal(bob.Name, person.Name);
Assert.Equal(bob.Created, person.Created);
}
public void TestGetValues_CustomFillCharacter_TrimsFill()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10) { Alignment = FixedAlignment.LeftAligned })
.AddColumn(new StringColumn("name"), new Window(25) { Alignment = FixedAlignment.LeftAligned })
.AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" }, new Window(10) { Alignment = FixedAlignment.LeftAligned });
FixedLengthOptions options = new FixedLengthOptions() { FillCharacter = '@' };
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
StringWriter stringWriter = new StringWriter();
FixedLengthWriter builder = new FixedLengthWriter(stringWriter, schema, options);
builder.Write(sources);
StringReader stringReader = new StringReader(stringWriter.ToString());
FixedLengthReader parser = new FixedLengthReader(stringReader, schema, options);
Assert.True(parser.Read(), "Could not read the first record.");
object[] actual = parser.GetValues();
Assert.Equal(sources, actual);
}
public void ShouldUseLeadingTruncationByDefault()
{
FixedLengthOptions options = new FixedLengthOptions();
Assert.AreEqual(OverflowTruncationPolicy.TruncateLeading, options.TruncationPolicy, "TruncateLeading not the default policy.");
}
public void TestGetValues_CustomFillCharacter_TrimsFill()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10) { Alignment = FixedAlignment.LeftAligned })
.AddColumn(new StringColumn("name"), new Window(25) { Alignment = FixedAlignment.LeftAligned })
.AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" }, new Window(10) { Alignment = FixedAlignment.LeftAligned });
FixedLengthOptions options = new FixedLengthOptions() { FillCharacter = '@' };
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
using (MemoryStream stream = new MemoryStream())
{
using (FixedLengthWriter builder = new FixedLengthWriter(stream, schema, options))
{
builder.Write(sources);
}
stream.Position = 0;
FixedLengthReader parser = new FixedLengthReader(stream, schema, options);
Assert.IsTrue(parser.Read(), "Could not read the first record.");
object[] actual = parser.GetValues();
CollectionAssert.AreEqual(sources, actual, "The values for the first record were wrong.");
}
}
public void TestTypeMapper_BadRecordColumn_SkipError()
{
const string data = @" 12017-06-11 John Smith
22017-12-32 Tom Stallon
32017-08-13 Walter Kay";
var mapper = FixedLengthTypeMapper.Define <Person>();
mapper.Property(x => x.Id, 10);
mapper.Property(x => x.Created, 10);
mapper.Property(x => x.Name, 15);
StringReader stringReader = new StringReader(data);
List <int> errorRecords = new List <int>();
var options = new FixedLengthOptions()
{
ErrorHandler = (sender, e) =>
{
errorRecords.Add(e.RecordNumber);
e.IsHandled = true;
}
};
var people = mapper.Read(stringReader, options).ToArray();
Assert.Equal(2, people.Count());
Assert.Single(errorRecords);
Assert.Equal(2, errorRecords[0]);
}