public void ShouldTruncateOverflow()
{
MemoryStream stream = new MemoryStream();
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new StringColumn("Default"), new Window(5));
schema.AddColumn(new StringColumn("Leading"), new Window(5) { TruncationPolicy = OverflowTruncationPolicy.TruncateLeading });
schema.AddColumn(new StringColumn("Trailing"), new Window(5) { TruncationPolicy = OverflowTruncationPolicy.TruncateTrailing });
FixedLengthOptions options = new FixedLengthOptions();
options.TruncationPolicy = OverflowTruncationPolicy.TruncateLeading; // this is the default anyway
using (FixedLengthWriter writer = new FixedLengthWriter(stream, schema, options))
{
writer.Write(new object[] { "Pineapple", "Pineapple", "Pineapple" });
}
stream.Position = 0;
string output = Encoding.Default.GetString(stream.ToArray());
string expected = "appleapplePinea" + Environment.NewLine;
Assert.AreEqual(expected, output, "The values were not truncated properly.");
}
public void ShouldTruncateOverflow()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new StringColumn("Default"), new Window(5));
schema.AddColumn(new StringColumn("Leading"), new Window(5)
{
TruncationPolicy = OverflowTruncationPolicy.TruncateLeading
});
schema.AddColumn(new StringColumn("Trailing"), new Window(5)
{
TruncationPolicy = OverflowTruncationPolicy.TruncateTrailing
});
FixedLengthOptions options = new FixedLengthOptions();
options.TruncationPolicy = OverflowTruncationPolicy.TruncateLeading; // this is the default anyway
StringWriter stringWriter = new StringWriter();
FixedLengthWriter writer = new FixedLengthWriter(stringWriter, schema, options);
writer.Write(new object[] { "Pineapple", "Pineapple", "Pineapple" });
string output = stringWriter.ToString();
string expected = "appleapplePinea" + Environment.NewLine;
Assert.AreEqual(expected, output);
}
public void ShouldWriteHeader()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new StringColumn("First"), new Window(10)
{
FillCharacter = '@'
});
schema.AddColumn(new StringColumn("Second"), new Window(10)
{
FillCharacter = '!'
});
schema.AddColumn(new StringColumn("Third"), new Window(10)
{
FillCharacter = '$'
});
FixedLengthOptions options = new FixedLengthOptions()
{
IsFirstRecordHeader = true
};
StringWriter stringWriter = new StringWriter();
FixedLengthWriter writer = new FixedLengthWriter(stringWriter, schema, options);
writer.Write(new object[] { "Apple", "Grape", "Pear" });
string output = stringWriter.ToString();
string expected = "First@@@@@Second!!!!Third$$$$$"
+ Environment.NewLine
+ "Apple@@@@@Grape!!!!!Pear$$$$$$"
+ Environment.NewLine;
Assert.AreEqual(expected, output);
}
public void ShouldTruncateOverflow()
{
MemoryStream stream = new MemoryStream();
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new StringColumn("Default"), new Window(5));
schema.AddColumn(new StringColumn("Leading"), new Window(5)
{
TruncationPolicy = OverflowTruncationPolicy.TruncateLeading
});
schema.AddColumn(new StringColumn("Trailing"), new Window(5)
{
TruncationPolicy = OverflowTruncationPolicy.TruncateTrailing
});
FixedLengthOptions options = new FixedLengthOptions();
options.TruncationPolicy = OverflowTruncationPolicy.TruncateLeading; // this is the default anyway
using (FixedLengthWriter writer = new FixedLengthWriter(stream, schema, options))
{
writer.Write(new object[] { "Pineapple", "Pineapple", "Pineapple" });
}
stream.Position = 0;
string output = Encoding.Default.GetString(stream.ToArray());
string expected = "appleapplePinea" + Environment.NewLine;
Assert.AreEqual(expected, output, "The values were not truncated properly.");
}
public void TestReaderWriter_TrailingText_RoundTripsExtra()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10)
{
Alignment = FixedAlignment.RightAligned
})
.AddColumn(new StringColumn("name"), new Window(25)
{
Alignment = FixedAlignment.RightAligned
})
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy"
}, new Window(10)
{
Alignment = FixedAlignment.RightAligned
})
.AddColumn(new StringColumn("extra"), Window.Trailing);
const string lines = @" 123 Bob Smith 4/21/2017This
-1 Jay Smith 8/14/2017is
234 Jay Smith 5/21/2017extra
";
StringReader stringReader = new StringReader(lines);
FixedLengthReader parser = new FixedLengthReader(stringReader, schema);
List <object[]> records = new List <object[]>()
{
AssertExtra(parser, "This"),
AssertExtra(parser, "is"),
AssertExtra(parser, "extra")
};
Assert.IsFalse(parser.Read());
StringWriter stringWriter = new StringWriter();
FixedLengthWriter writer = new FixedLengthWriter(stringWriter, schema);
foreach (object[] record in records)
{
writer.Write(record);
}
string formatted = stringWriter.ToString();
Assert.AreEqual(lines, formatted, "The records did not round-trip.");
}
/// <summary>
/// Gets a typed writer for writing the objects to the file.
/// </summary>
/// <param name="writer">The writer to use.</param>
/// <param name="options">The separate value options to use.</param>
/// <returns>The typed writer.</returns>
public ITypedWriter <object> GetWriter(TextWriter writer, FixedLengthOptions options = null)
{
var injector = new FixedLengthSchemaInjector();
var valueWriter = new FixedLengthWriter(writer, injector, options);
var multiWriter = new MultiplexingTypedWriter(valueWriter, this);
foreach (var matcher in matchers)
{
injector.When((values) => matcher.IsMatch).Use(matcher.TypeMapper.GetSchema());
}
if (defaultMatcher != nonMatcher)
{
injector.WithDefault(defaultMatcher.TypeMapper.GetSchema());
}
return(multiWriter);
}
public void ShouldHandleNullValues()
{
MemoryStream stream = new MemoryStream();
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("NullableInt32"), new Window(5));
StringWriter stringWriter = new StringWriter();
FixedLengthWriter writer = new FixedLengthWriter(stringWriter, schema);
writer.Write(new object[] { null });
string output = stringWriter.ToString();
string expected = " " + Environment.NewLine;
Assert.AreEqual(expected, output);
}
public void ShouldHandleNullValues()
{
MemoryStream stream = new MemoryStream();
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("NullableInt32"), new Window(5));
using (FixedLengthWriter writer = new FixedLengthWriter(stream, schema))
{
writer.Write(new object[] { null });
}
stream.Position = 0;
string output = Encoding.Default.GetString(stream.ToArray());
string expected = " " + Environment.NewLine;
Assert.AreEqual(expected, output, "The values were not truncated properly.");
}
public void Write(string outputFilePath, DataSource dataSource, FileSchemaBase schema)
{
using (var fileStreamWriter = new StreamWriter(outputFilePath, true))
{
switch (schema)
{
case FixedLengthFileSchema fixedLengthFileSchema:
foreach (var rowData in dataSource.Data)
{
var rowSchema = fixedLengthFileSchema.FixedLengthRecordSchemas.FirstOrDefault(x => x.RecordIdentifier.Equals(rowData.Identifier));
if (rowSchema == null)
{
throw new Exception($"No such identifier defined: '{rowData.Identifier}'");
}
var fixedLengthWriter = new FixedLengthWriter(fileStreamWriter, rowSchema.FixedLengthSchema);
fixedLengthWriter.Write(rowData.DataArray);
}
break;
case SeparatedFileSchema separatedFileSchema:
foreach (var rowData in dataSource.Data)
{
var rowSchema = separatedFileSchema.SeparatedRecordSchemas.FirstOrDefault(x => x.RecordIdentifier.Equals(rowData.Identifier));
if (rowSchema == null)
{
throw new Exception($"No such identifier defined: '{rowData.Identifier}'");
}
var separatedWriter = new SeparatedValueWriter(fileStreamWriter, rowSchema.SeparatedValueSchema, new SeparatedValueOptions
{
Separator = separatedFileSchema.Delimeter
});
separatedWriter.Write(rowData.DataArray);
}
break;
}
}
}
public void ShouldHandleNullValues()
{
MemoryStream stream = new MemoryStream();
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("NullableInt32"), new Window(5));
using (FixedLengthWriter writer = new FixedLengthWriter(stream, schema))
{
writer.Write(new object[] { null });
}
stream.Position = 0;
string output = Encoding.Default.GetString(stream.ToArray());
string expected = " " + Environment.NewLine;
Assert.AreEqual(expected, output, "The values were not truncated properly.");
}
public void TestGetValues_CustomFillCharacter_TrimsFill()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10)
{
Alignment = FixedAlignment.LeftAligned
})
.AddColumn(new StringColumn("name"), new Window(25)
{
Alignment = FixedAlignment.LeftAligned
})
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy"
}, new Window(10)
{
Alignment = FixedAlignment.LeftAligned
});
FixedLengthOptions options = new FixedLengthOptions()
{
FillCharacter = '@'
};
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
using (MemoryStream stream = new MemoryStream())
{
using (FixedLengthWriter builder = new FixedLengthWriter(stream, schema, options))
{
builder.Write(sources);
}
stream.Position = 0;
FixedLengthReader parser = new FixedLengthReader(stream, schema, options);
Assert.IsTrue(parser.Read(), "Could not read the first record.");
object[] actual = parser.GetValues();
CollectionAssert.AreEqual(sources, actual, "The values for the first record were wrong.");
}
}
public void TestGetValues_CustomFillCharacter_TrimsFill()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10)
{
Alignment = FixedAlignment.LeftAligned
})
.AddColumn(new StringColumn("name"), new Window(25)
{
Alignment = FixedAlignment.LeftAligned
})
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy"
}, new Window(10)
{
Alignment = FixedAlignment.LeftAligned
});
FixedLengthOptions options = new FixedLengthOptions()
{
FillCharacter = '@'
};
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
StringWriter stringWriter = new StringWriter();
FixedLengthWriter builder = new FixedLengthWriter(stringWriter, schema, options);
builder.Write(sources);
StringReader stringReader = new StringReader(stringWriter.ToString());
FixedLengthReader parser = new FixedLengthReader(stringReader, schema, options);
Assert.True(parser.Read(), "Could not read the first record.");
object[] actual = parser.GetValues();
Assert.Equal(sources, actual);
}
public void ShouldWriteSchemaIfExplicit()
{
StringWriter stringWriter = new StringWriter();
// Explicitly indicate that the first record is NOT the schema
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new StringColumn("Col1"), 10);
FixedLengthWriter writer = new FixedLengthWriter(stringWriter, schema, new FixedLengthOptions()
{
IsFirstRecordHeader = false
});
writer.WriteSchema(); // Explicitly write the schema
writer.Write(new string[] { "a" });
StringReader stringReader = new StringReader(stringWriter.ToString());
var reader = new FixedLengthReader(stringReader, schema, new FixedLengthOptions()
{
IsFirstRecordHeader = true
});
Assert.IsTrue(reader.Read(), "The record was not retrieved after the schema.");
Assert.IsFalse(reader.Read(), "Encountered more than the expected number of records.");
}
private void ExportText()
{
var fileName = Project.Current.MapPath(FileName);
if (string.IsNullOrWhiteSpace(fileName))
{
throw new ArgumentNullException("Filename is not provided.");
}
if (!Overwrite && File.Exists(fileName))
{
throw new Exception($"File '{FileName}' already exists.");
}
if (Overwrite && File.Exists(fileName))
{
File.Delete(fileName);
}
using var dataReader = GetDataSourceReader(_Output, DataSource,
new DataSourceParameters()
{
IgnoreErrors = this.IgnoreErrors, Columns = this.SelectColumns, SkipColumns = this.SkipColumns
});
var schema = new FixedLengthSchema();
if (Columns == null)
{
throw new ArgumentNullException(nameof(Columns), "Fixed-length import/export requires column schema.");
}
int[] columnIndexes = new int[Columns.Length];
for (int i = 0; i < Columns.Length; i++)
{
var column = Columns[i];
int columnIndex = dataReader.GetOrdinal(column.ColumnName);
if (columnIndex < 0)
{
throw new Exception($"Cannot find column '{column.ColumnName}'.");
}
columnIndexes[i] = columnIndex;
var definition = CreateColumnDefinition(column);
var window = new Window(column.ColumnLength);
if (column.FillCharacter.HasValue)
{
window.FillCharacter = column.FillCharacter.Value;
}
if (column.Alignment.HasValue)
{
window.Alignment = column.Alignment.Value;
}
if (column.TruncationPolicy.HasValue)
{
window.TruncationPolicy = column.TruncationPolicy.Value;
}
schema.AddColumn(definition, window);
}
#pragma warning disable IDE0017 // Simplify object initialization
var options = new FixedLengthOptions();
#pragma warning restore IDE0017 // Simplify object initialization
options.FillCharacter = FillCharacter;
options.HasRecordSeparator = !NoRecordSeparator;
if (RecordSeparator != null)
{
options.RecordSeparator = RecordSeparator;
}
options.IsFirstRecordHeader = !NoHeaderRow;
options.Alignment = Alignment;
options.TruncationPolicy = TruncationPolicy;
if (!string.IsNullOrWhiteSpace(Culture))
{
options.FormatProvider = new CultureInfo(Culture);
}
using var writer = new StreamWriter(File.OpenWrite(fileName));
var csvWriter = new FixedLengthWriter(writer, schema, options);
int columnCount = schema.ColumnDefinitions.Count;
object[] values = new object[columnCount];
bool isHeader = options.IsFirstRecordHeader;
while (dataReader.Read())
{
for (int i = 0; i < columnCount; i++)
{
object value = dataReader.GetValue(columnIndexes[i]);
if (value == DBNull.Value)
{
value = null;
}
if (!isHeader)
{
value = Utils.ChangeType(schema.ColumnDefinitions[i].ColumnType, value, null);
}
values[i] = value;
isHeader = false;
}
csvWriter.Write(values);
}
}
public void TestReader_ReadThreeTypes()
{
StringWriter stringWriter = new StringWriter();
var injector = getSchemaInjector();
var options = new FixedLengthOptions()
{
Alignment = FixedAlignment.RightAligned
};
var writer = new FixedLengthWriter(stringWriter, injector, options);
writer.Write(new object[] { "First Batch", 2 });
writer.Write(new object[] { 1, "Bob Smith", new DateTime(2018, 06, 04), 12.34m });
writer.Write(new object[] { 2, "Jane Doe", new DateTime(2018, 06, 05), 34.56m });
writer.Write(new object[] { 46.9m, 23.45m, true });
string output = stringWriter.ToString();
Assert.AreEqual(@" First Batch 2
1 Bob Smith 20180604 12.34
2 Jane Doe 20180605 34.56
46.9 23.45 True
", output);
var stringReader = new StringReader(output);
var selector = getSchemaSelector();
var reader = new FixedLengthReader(stringReader, selector, options);
Assert.IsTrue(reader.Read(), "The header record could not be read.");
var headerValues = reader.GetValues();
Assert.AreEqual(2, headerValues.Length);
Assert.AreEqual("First Batch", headerValues[0]);
Assert.AreEqual(2, headerValues[1]);
Assert.IsTrue(reader.Read(), "The first data record could not be read.");
var dataValues1 = reader.GetValues();
Assert.AreEqual(4, dataValues1.Length);
Assert.AreEqual(1, dataValues1[0]);
Assert.AreEqual("Bob Smith", dataValues1[1]);
Assert.AreEqual(new DateTime(2018, 6, 4), dataValues1[2]);
Assert.AreEqual(12.34m, dataValues1[3]);
Assert.IsTrue(reader.Read(), "The second data record could not be read.");
var dataValues2 = reader.GetValues();
Assert.AreEqual(4, dataValues2.Length);
Assert.AreEqual(2, dataValues2[0]);
Assert.AreEqual("Jane Doe", dataValues2[1]);
Assert.AreEqual(new DateTime(2018, 6, 5), dataValues2[2]);
Assert.AreEqual(34.56m, dataValues2[3]);
Assert.IsTrue(reader.Read(), "The footer record could not be read.");
var footerValues = reader.GetValues();
Assert.AreEqual(3, footerValues.Length);
Assert.AreEqual(46.9m, footerValues[0]);
Assert.AreEqual(23.45m, footerValues[1]);
Assert.AreEqual(true, footerValues[2]);
Assert.IsFalse(reader.Read());
}
public void TestGetValues_CustomFillCharacter_TrimsFill()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10) { Alignment = FixedAlignment.LeftAligned })
.AddColumn(new StringColumn("name"), new Window(25) { Alignment = FixedAlignment.LeftAligned })
.AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" }, new Window(10) { Alignment = FixedAlignment.LeftAligned });
FixedLengthOptions options = new FixedLengthOptions() { FillCharacter = '@' };
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
StringWriter stringWriter = new StringWriter();
FixedLengthWriter builder = new FixedLengthWriter(stringWriter, schema, options);
builder.Write(sources);
StringReader stringReader = new StringReader(stringWriter.ToString());
FixedLengthReader parser = new FixedLengthReader(stringReader, schema, options);
Assert.True(parser.Read(), "Could not read the first record.");
object[] actual = parser.GetValues();
Assert.Equal(sources, actual);
}
public void TestGetValues_CustomFillCharacter_TrimsFill()
{
FixedLengthSchema schema = new FixedLengthSchema();
schema.AddColumn(new Int32Column("id"), new Window(10) { Alignment = FixedAlignment.LeftAligned })
.AddColumn(new StringColumn("name"), new Window(25) { Alignment = FixedAlignment.LeftAligned })
.AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" }, new Window(10) { Alignment = FixedAlignment.LeftAligned });
FixedLengthOptions options = new FixedLengthOptions() { FillCharacter = '@' };
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19) };
using (MemoryStream stream = new MemoryStream())
{
using (FixedLengthWriter builder = new FixedLengthWriter(stream, schema, options))
{
builder.Write(sources);
}
stream.Position = 0;
FixedLengthReader parser = new FixedLengthReader(stream, schema, options);
Assert.IsTrue(parser.Read(), "Could not read the first record.");
object[] actual = parser.GetValues();
CollectionAssert.AreEqual(sources, actual, "The values for the first record were wrong.");
}
}
