public void TestReader_ReadThreeTypes()
{
StringWriter stringWriter = new StringWriter();
var injector = getSchemaInjector();
var writer = new SeparatedValueWriter(stringWriter, injector);
writer.Write(new object[] { "First Batch", 2 });
writer.Write(new object[] { 1, "Bob Smith", new DateTime(2018, 06, 04), 12.34m });
writer.Write(new object[] { 2, "Jane Doe", new DateTime(2018, 06, 05), 34.56m });
writer.Write(new object[] { 46.9m, 23.45m, true });
string output = stringWriter.ToString();
Assert.AreEqual(@"First Batch,2
1,Bob Smith,20180604,12.34
2,Jane Doe,20180605,34.56
46.9,23.45,True
", output);
var stringReader = new StringReader(output);
var selector = getSchemaSelector();
var reader = new SeparatedValueReader(stringReader, selector);
Assert.IsTrue(reader.Read(), "The header record could not be read.");
var headerValues = reader.GetValues();
Assert.AreEqual(2, headerValues.Length);
Assert.AreEqual("First Batch", headerValues[0]);
Assert.AreEqual(2, headerValues[1]);
Assert.IsTrue(reader.Read(), "The first data record could not be read.");
var dataValues1 = reader.GetValues();
Assert.AreEqual(4, dataValues1.Length);
Assert.AreEqual(1, dataValues1[0]);
Assert.AreEqual("Bob Smith", dataValues1[1]);
Assert.AreEqual(new DateTime(2018, 6, 4), dataValues1[2]);
Assert.AreEqual(12.34m, dataValues1[3]);
Assert.IsTrue(reader.Read(), "The second data record could not be read.");
var dataValues2 = reader.GetValues();
Assert.AreEqual(4, dataValues2.Length);
Assert.AreEqual(2, dataValues2[0]);
Assert.AreEqual("Jane Doe", dataValues2[1]);
Assert.AreEqual(new DateTime(2018, 6, 5), dataValues2[2]);
Assert.AreEqual(34.56m, dataValues2[3]);
Assert.IsTrue(reader.Read(), "The footer record could not be read.");
var footerValues = reader.GetValues();
Assert.AreEqual(3, footerValues.Length);
Assert.AreEqual(46.9m, footerValues[0]);
Assert.AreEqual(23.45m, footerValues[1]);
Assert.AreEqual(true, footerValues[2]);
Assert.IsFalse(reader.Read());
}
public void TestGetValues_BlankLeadingSection_ReturnsNull()
{
using (MemoryStream stream = new MemoryStream())
{
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new StringColumn("leading"))
.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy"
});
object[] sources = new object[] { "", 123, "Bob", new DateTime(2013, 1, 19) };
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter builder = new SeparatedValueWriter(stringWriter, schema, options);
builder.Write(sources);
StringReader stringReader = new StringReader(stringWriter.ToString());
SeparatedValueReader parser = new SeparatedValueReader(stringReader, schema, options);
Assert.True(parser.Read(), "No records were found.");
object[] values = parser.GetValues();
Assert.Equal(schema.ColumnDefinitions.Count, values.Length);
Assert.Equal(null, values[0]);
Assert.Equal(sources[1], values[1]);
Assert.Equal(sources[2], values[2]);
Assert.Equal(sources[3], values[3]);
Assert.False(parser.Read(), "Too many records were found.");
}
}
public void TestReadFlatFile_DataTableNull_Throws()
{
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "MM/dd/yyyy"
})
.AddColumn(new DecimalColumn("avg"));
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
using (MemoryStream stream = new MemoryStream())
{
using (SeparatedValueWriter builder = new SeparatedValueWriter(stream, schema, options))
{
builder.Write(new object[] { 123, "Bob", new DateTime(2012, 12, 31), 3.14159m });
}
stream.Position = 0;
DataTable table = null;
IReader parser = new SeparatedValueReader(stream, options);
DataTableExtensions.ReadFlatFile(table, parser);
}
}
public void TestGetValues_BlankLeadingSection_ReturnsNull()
{
using (MemoryStream stream = new MemoryStream())
{
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new StringColumn("leading"))
.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy"
});
object[] sources = new object[] { "", 123, "Bob", new DateTime(2013, 1, 19) };
using (SeparatedValueWriter builder = new SeparatedValueWriter(stream, schema, options))
{
builder.Write(sources);
}
stream.Position = 0;
SeparatedValueReader parser = new SeparatedValueReader(stream, schema, options);
Assert.IsTrue(parser.Read(), "No records were found.");
object[] values = parser.GetValues();
Assert.AreEqual(schema.ColumnDefinitions.Count, values.Length, "The wrong number of values were read.");
Assert.AreEqual(null, values[0], "The first column was not interpreted as null.");
Assert.AreEqual(sources[1], values[1], "The second column was not parsed correctly.");
Assert.AreEqual(sources[2], values[2], "The third column was not parsed correctly.");
Assert.AreEqual(sources[3], values[3], "The forth column was not parsed correctly.");
Assert.IsFalse(parser.Read(), "Too many records were found.");
}
}
public void ShouldWriteSchemaIfExplicit()
{
StringWriter stringWriter = new StringWriter();
// Explicitly indicate that the first record is NOT the schema
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new StringColumn("Col1"));
SeparatedValueWriter writer = new SeparatedValueWriter(stringWriter, schema, new SeparatedValueOptions()
{
IsFirstRecordSchema = false
});
writer.WriteSchema(); // Explicitly write the schema
writer.Write(new string[] { "a" });
StringReader stringReader = new StringReader(stringWriter.ToString());
var reader = new SeparatedValueReader(stringReader, new SeparatedValueOptions()
{
IsFirstRecordSchema = true
});
var parsedSchema = reader.GetSchema();
Assert.Equal(schema.ColumnDefinitions.Count, parsedSchema.ColumnDefinitions.Count);
Assert.Equal(schema.ColumnDefinitions[0].ColumnName, parsedSchema.ColumnDefinitions[0].ColumnName);
Assert.True(reader.Read(), "The record was not retrieved after the schema.");
Assert.False(reader.Read(), "Encountered more than the expected number of records.");
}
private static string writeValues(object[] values)
{
var schema = getSchema();
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter writer = new SeparatedValueWriter(stringWriter, schema);
writer.Write(values);
return(stringWriter.ToString());
}
public void TestReadWrite_Comments()
{
StringWriter output = new StringWriter();
SeparatedValueWriter writer = new SeparatedValueWriter(output);
writer.Write(new[] { "a", "b", "c" });
writer.WriteRaw("# Hello, world!!!", true);
writer.Write(new[] { "d", "e", "f" });
StringReader input = new StringReader(output.ToString());
SeparatedValueReader reader = new SeparatedValueReader(input);
reader.RecordRead += (sender, e) =>
{
e.IsSkipped = e.Values.Length > 0 && e.Values[0].StartsWith("#");
};
Assert.IsTrue(reader.Read());
CollectionAssert.AreEqual(new[] { "a", "b", "c" }, reader.GetValues());
Assert.IsTrue(reader.Read());
CollectionAssert.AreEqual(new[] { "d", "e", "f" }, reader.GetValues());
Assert.IsFalse(reader.Read());
}
public void ShouldNotWriteSchemaIfNoSchemaProvided()
{
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter writer = new SeparatedValueWriter(stringWriter, new SeparatedValueOptions()
{
IsFirstRecordSchema = true
});
writer.Write(new string[] { "a" });
string output = stringWriter.ToString();
string expected = "a" + Environment.NewLine;
Assert.Equal(expected, output);
}
private static string writeValues(object[] values)
{
var schema = getSchema();
using (MemoryStream stream = new MemoryStream())
{
using (SeparatedValueWriter writer = new SeparatedValueWriter(stream, schema))
{
writer.Write(values);
}
stream.Position = 0;
byte[] encoded = stream.ToArray();
return(Encoding.Default.GetString(encoded));
}
}
public void ShouldRoundTrip()
{
const string message = @"Tom,Hanselman,2016-06-0426 Walking Ice,Ace
";
StringReader stringReader = new StringReader(message);
SeparatedValueSchema outerSchema = new SeparatedValueSchema();
outerSchema.AddColumn(new StringColumn("FirstName"));
outerSchema.AddColumn(new StringColumn("LastName"));
FixedLengthSchema innerSchema = new FixedLengthSchema();
innerSchema.AddColumn(new DateTimeColumn("StartDate")
{
InputFormat = "yyyy-MM-dd", OutputFormat = "yyyy-MM-dd"
}, 10);
innerSchema.AddColumn(new Int32Column("Age"), 2);
innerSchema.AddColumn(new StringColumn("StageName"), new Window(20)
{
Alignment = FixedAlignment.RightAligned
});
outerSchema.AddColumn(new FixedLengthComplexColumn("PlayerStats", innerSchema));
outerSchema.AddColumn(new StringColumn("Nickname"));
SeparatedValueReader reader = new SeparatedValueReader(stringReader, outerSchema);
Assert.True(reader.Read(), "A record should have been read.");
object[] values = reader.GetValues();
Assert.Equal("Tom", values[0]);
Assert.Equal("Hanselman", values[1]);
Assert.IsType <object[]>(values[2]);
object[] playerValues = (object[])values[2];
Assert.Equal(new DateTime(2016, 06, 04), playerValues[0]);
Assert.Equal(26, playerValues[1]);
Assert.Equal("Walking Ice", playerValues[2]);
Assert.Equal("Ace", values[3]);
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter writer = new SeparatedValueWriter(stringWriter, outerSchema);
writer.Write(values);
string output = stringWriter.GetStringBuilder().ToString();
Assert.Equal(message, output);
}
public void Write(string outputFilePath, DataSource dataSource, FileSchemaBase schema)
{
using (var fileStreamWriter = new StreamWriter(outputFilePath, true))
{
switch (schema)
{
case FixedLengthFileSchema fixedLengthFileSchema:
foreach (var rowData in dataSource.Data)
{
var rowSchema = fixedLengthFileSchema.FixedLengthRecordSchemas.FirstOrDefault(x => x.RecordIdentifier.Equals(rowData.Identifier));
if (rowSchema == null)
{
throw new Exception($"No such identifier defined: '{rowData.Identifier}'");
}
var fixedLengthWriter = new FixedLengthWriter(fileStreamWriter, rowSchema.FixedLengthSchema);
fixedLengthWriter.Write(rowData.DataArray);
}
break;
case SeparatedFileSchema separatedFileSchema:
foreach (var rowData in dataSource.Data)
{
var rowSchema = separatedFileSchema.SeparatedRecordSchemas.FirstOrDefault(x => x.RecordIdentifier.Equals(rowData.Identifier));
if (rowSchema == null)
{
throw new Exception($"No such identifier defined: '{rowData.Identifier}'");
}
var separatedWriter = new SeparatedValueWriter(fileStreamWriter, rowSchema.SeparatedValueSchema, new SeparatedValueOptions
{
Separator = separatedFileSchema.Delimeter
});
separatedWriter.Write(rowData.DataArray);
}
break;
}
}
}
public void TestReadFlatFile_ExtractsSchema_PopulatesTable()
{
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "MM/dd/yyyy", OutputFormat = "MM/dd/yyyy"
})
.AddColumn(new DecimalColumn("avg"));
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
using (MemoryStream stream = new MemoryStream())
{
using (SeparatedValueWriter builder = new SeparatedValueWriter(stream, schema, options))
{
builder.Write(new object[] { 123, "Bob", new DateTime(2012, 12, 31), 3.14159m });
}
stream.Position = 0;
DataTable table = new DataTable();
IReader parser = new SeparatedValueReader(stream, options);
table.ReadFlatFile(parser);
Assert.AreEqual(4, table.Columns.Count, "The wrong number of columns were extracted.");
Assert.IsTrue(table.Columns.Contains("id"), "The ID column was not extracted.");
Assert.IsTrue(table.Columns.Contains("name"), "The name column was not extracted.");
Assert.IsTrue(table.Columns.Contains("created"), "The created column was not extracted.");
Assert.IsTrue(table.Columns.Contains("avg"), "The AVG column was not extracted.");
Assert.AreEqual(1, table.Rows.Count, "Not all of the records were extracted.");
DataRow row = table.Rows[0];
object[] expected = new object[] { "123", "Bob", "12/31/2012", "3.14159" };
object[] values = row.ItemArray;
CollectionAssert.AreEqual(expected, values, "The wrong values were extracted");
}
}
public void TestReadFlatFile_ExtractsSchema_PopulatesTable()
{
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created")
{
InputFormat = "MM/dd/yyyy", OutputFormat = "MM/dd/yyyy"
})
.AddColumn(new DecimalColumn("avg"));
SeparatedValueOptions options = new SeparatedValueOptions()
{
IsFirstRecordSchema = true
};
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter builder = new SeparatedValueWriter(stringWriter, schema, options);
builder.Write(new object[] { 123, "Bob", new DateTime(2012, 12, 31), 3.14159m });
StringReader stringReader = new StringReader(stringWriter.ToString());
DataTable table = new DataTable();
IReader parser = new SeparatedValueReader(stringReader, options);
table.ReadFlatFile(parser);
Assert.Equal(4, table.Columns.Count);
Assert.True(table.Columns.Contains("id"), "The ID column was not extracted.");
Assert.True(table.Columns.Contains("name"), "The name column was not extracted.");
Assert.True(table.Columns.Contains("created"), "The created column was not extracted.");
Assert.True(table.Columns.Contains("avg"), "The AVG column was not extracted.");
Assert.Equal(1, table.Rows.Count);
DataRow row = table.Rows[0];
object[] expected = new object[] { "123", "Bob", "12/31/2012", "3.14159" };
object[] values = row.ItemArray;
Assert.Equal(expected, values);
}
public void TestGetValues_BlankTrailingSection_ReturnsNull()
{
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" })
.AddColumn(new StringColumn("trailing"));
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19), "" };
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter builder = new SeparatedValueWriter(stringWriter, schema, options);
builder.Write(sources);
StringReader stringReader = new StringReader(stringWriter.ToString());
SeparatedValueReader parser = new SeparatedValueReader(stringReader, schema, options);
Assert.True(parser.Read(), "No records were found.");
object[] values = parser.GetValues();
Assert.Equal(schema.ColumnDefinitions.Count, values.Length);
Assert.Equal(sources[0], values[0]);
Assert.Equal(sources[1], values[1]);
Assert.Equal(sources[2], values[2]);
Assert.Equal(null, values[3]);
Assert.False(parser.Read(), "Too many records were found.");
}
public void TestReadFlatFile_ExtractsSchema_PopulatesTable()
{
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created") { InputFormat = "MM/dd/yyyy", OutputFormat = "MM/dd/yyyy" })
.AddColumn(new DecimalColumn("avg"));
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter builder = new SeparatedValueWriter(stringWriter, schema, options);
builder.Write(new object[] { 123, "Bob", new DateTime(2012, 12, 31), 3.14159m });
StringReader stringReader = new StringReader(stringWriter.ToString());
DataTable table = new DataTable();
IReader parser = new SeparatedValueReader(stringReader, options);
table.ReadFlatFile(parser);
Assert.Equal(4, table.Columns.Count);
Assert.True(table.Columns.Contains("id"), "The ID column was not extracted.");
Assert.True(table.Columns.Contains("name"), "The name column was not extracted.");
Assert.True(table.Columns.Contains("created"), "The created column was not extracted.");
Assert.True(table.Columns.Contains("avg"), "The AVG column was not extracted.");
Assert.Equal(1, table.Rows.Count);
DataRow row = table.Rows[0];
object[] expected = new object[] { "123", "Bob", "12/31/2012", "3.14159" };
object[] values = row.ItemArray;
Assert.Equal(expected, values);
}
private static string writeValues(object[] values)
{
var schema = getSchema();
using (MemoryStream stream = new MemoryStream())
{
using (SeparatedValueWriter writer = new SeparatedValueWriter(stream, schema))
{
writer.Write(values);
}
stream.Position = 0;
byte[] encoded = stream.ToArray();
return Encoding.Default.GetString(encoded);
}
}
private void ExportText()
{
var fileName = Project.Current.MapPath(FileName);
if (string.IsNullOrWhiteSpace(fileName))
{
throw new ArgumentNullException("Filename is not provided.");
}
if (!Overwrite && File.Exists(fileName))
{
throw new Exception($"File '{FileName}' already exists.");
}
if (Overwrite && File.Exists(fileName))
{
File.Delete(fileName);
}
using var dataReader = GetDataSourceReader(_Output, DataSource,
new DataSourceParameters()
{
IgnoreErrors = this.IgnoreErrors, Columns = this.SelectColumns, SkipColumns = this.SkipColumns
});
var schema = new SeparatedValueSchema();
int colCount = Columns != null ? Columns.Length : dataReader.FieldCount;
int[] columnIndexes = new int[colCount];
if (Columns != null)
{
for (int i = 0; i < Columns.Length; i++)
{
var column = Columns[i];
int columnIndex = dataReader.GetOrdinal(column.ColumnName);
if (columnIndex < 0)
{
throw new Exception($"Cannot find column '{column.ColumnName}'.");
}
columnIndexes[i] = columnIndex;
var definition = CreateColumnDefinition(column);
schema.AddColumn(definition);
}
}
else
{
for (int i = 0; i < dataReader.FieldCount; i++)
{
columnIndexes[i] = i;
var column = new TextColumnDefinition()
{
ColumnName = dataReader.GetName(i),
ColumnType = dataReader.GetFieldType(i)
};
var definition = CreateColumnDefinition(column);
schema.AddColumn(definition);
}
}
var options = new SeparatedValueOptions();
if (Separator != null)
{
options.Separator = Separator;
}
if (RecordSeparator != null)
{
options.RecordSeparator = RecordSeparator;
}
options.Quote = Quote;
options.IsFirstRecordSchema = !NoHeaderRow;
options.PreserveWhiteSpace = PreserveWhiteSpace;
if (!string.IsNullOrWhiteSpace(Culture))
{
options.FormatProvider = new CultureInfo(Culture);
}
options.QuoteBehavior = QuoteBehavior;
using var writer = new StreamWriter(File.OpenWrite(fileName));
var csvWriter = new SeparatedValueWriter(writer, schema, options);
int columnCount = schema.ColumnDefinitions.Count;
object[] values = new object[columnCount];
bool isHeader = options.IsFirstRecordSchema;
while (dataReader.Read())
{
for (int i = 0; i < columnCount; i++)
{
object value = dataReader.GetValue(columnIndexes[i]);
if (value == DBNull.Value)
{
value = null;
}
if (!isHeader)
{
value = Utils.ChangeType(schema.ColumnDefinitions[i].ColumnType, value, null);
}
values[i] = value;
isHeader = false;
}
csvWriter.Write(values);
}
}
private static string writeValues(object[] values)
{
var schema = getSchema();
StringWriter stringWriter = new StringWriter();
SeparatedValueWriter writer = new SeparatedValueWriter(stringWriter, schema);
writer.Write(values);
return stringWriter.ToString();
}
public void TestGetValues_BlankTrailingSection_ReturnsNull()
{
using (MemoryStream stream = new MemoryStream())
{
SeparatedValueOptions options = new SeparatedValueOptions() { IsFirstRecordSchema = true };
SeparatedValueSchema schema = new SeparatedValueSchema();
schema.AddColumn(new Int32Column("id"))
.AddColumn(new StringColumn("name"))
.AddColumn(new DateTimeColumn("created") { InputFormat = "M/d/yyyy", OutputFormat = "M/d/yyyy" })
.AddColumn(new StringColumn("trailing"));
object[] sources = new object[] { 123, "Bob", new DateTime(2013, 1, 19), "" };
using (SeparatedValueWriter builder = new SeparatedValueWriter(stream, schema, options))
{
builder.Write(sources);
}
stream.Position = 0;
SeparatedValueReader parser = new SeparatedValueReader(stream, schema, options);
Assert.IsTrue(parser.Read(), "No records were found.");
object[] values = parser.GetValues();
Assert.AreEqual(schema.ColumnDefinitions.Count, values.Length, "The wrong number of values were read.");
Assert.AreEqual(sources[0], values[0], "The first column was not parsed correctly.");
Assert.AreEqual(sources[1], values[1], "The second column was not parsed correctly.");
Assert.AreEqual(sources[2], values[2], "The third column was not parsed correctly.");
Assert.AreEqual(null, values[3], "The forth column was not interpreted as null.");
Assert.IsFalse(parser.Read(), "Too many records were found.");
}
}
