public void EndsWithNewLine()
{
string file = $"Column1,Column2,Column3{Environment.NewLine}Data1,Data2,Data3{Environment.NewLine}";
var data = new DsvReader(file, System.Text.Encoding.UTF8, DsvOptions.DefaultCsvOptions);
List <ReadOnlyMemory <char> > columns = null;
List <List <ReadOnlyMemory <char> > > rows = new List <List <ReadOnlyMemory <char> > >();
while (data.MoveNext())
{
if (!data.ColumnsFilled)
{
columns = data.ReadLine().ToList();
}
else
{
rows.Add(data.ReadLine().ToList());
}
}
Assert.Equal(3, columns.Count);
Assert.Single(rows);
Assert.Equal("Column1", columns[0].ToString());
Assert.Equal("Column2", columns[1].ToString());
Assert.Equal("Column3", columns[2].ToString());
Assert.Equal("Data1", rows[0][0].ToString());
Assert.Equal("Data2", rows[0][1].ToString());
Assert.Equal("Data3", rows[0][2].ToString());
}
public void Constructor_ReadOnlySpan()
{
string file = FileGenerator("Column", "Data", 1, 3);
var span = file.AsSpan();
var data = new DsvReader(span, System.Text.Encoding.UTF8, DsvOptions.DefaultCsvOptions);
List <ReadOnlyMemory <char> > columns = null;
List <List <ReadOnlyMemory <char> > > rows = new List <List <ReadOnlyMemory <char> > >();
while (data.MoveNext())
{
if (!data.ColumnsFilled)
{
columns = data.ReadLine().ToList();
}
else
{
rows.Add(data.ReadLine().ToList());
}
}
Assert.Equal(3, columns.Count);
Assert.Single(rows);
Assert.Equal("Column1", columns[0].ToString());
Assert.Equal("Column2", columns[1].ToString());
Assert.Equal("Column3", columns[2].ToString());
Assert.Equal("Data1", rows[0][0].ToString());
Assert.Equal("Data2", rows[0][1].ToString());
Assert.Equal("Data3", rows[0][2].ToString());
}
public void SpecialCharacters()
{
string file = $"Column1,Column2,Column3{Environment.NewLine}Düsseldorf,Datüa2,ü123{Environment.NewLine}";
var bytes = Encoding.UTF8.GetBytes(file);
var data = new DsvReader(bytes, System.Text.Encoding.UTF8, DsvOptions.DefaultCsvOptions);
List <ReadOnlyMemory <char> > columns = null;
List <List <ReadOnlyMemory <char> > > rows = new List <List <ReadOnlyMemory <char> > >();
while (data.MoveNext())
{
if (!data.ColumnsFilled)
{
columns = data.ReadLine().ToList();
}
else
{
rows.Add(data.ReadLine().ToList());
}
}
Assert.Equal(3, columns.Count);
Assert.Single(rows);
Assert.Equal("Column1", columns[0].ToString());
Assert.Equal("Column2", columns[1].ToString());
Assert.Equal("Column3", columns[2].ToString());
Assert.Equal("Düsseldorf", rows[0][0].ToString());
Assert.Equal("Datüa2", rows[0][1].ToString());
Assert.Equal("ü123", rows[0][2].ToString());
}
public void MalformedData_Escapes()
{
string file = $"\"\"Column1\",Column2{Environment.NewLine}Data1,Data2";
Assert.Throws <FormatException>(() =>
{
var data = new DsvReader(file, System.Text.Encoding.UTF8, DsvOptions.DefaultCsvOptions);
List <ReadOnlyMemory <char> > columns = null;
List <List <ReadOnlyMemory <char> > > rows = new List <List <ReadOnlyMemory <char> > >();
while (data.MoveNext())
{
if (!data.ColumnsFilled)
{
columns = data.ReadLine().ToList();
}
else
{
rows.Add(data.ReadLine().ToList());
}
}
});
}
public void Beffyman_DsvReader()
{
var reader = new DsvReader(MemoryFile, Encoding.UTF8, DsvOptions.DefaultCsvOptions);
while (reader.MoveNext())
{
}
}
public void JaggedTable()
{
const string ValueDelimeter = "|";
const string RecordDelimeter = "\r\n";
const int NumRecords = 4;
const int NumValuesOddRow = 3;
const int NumValuesEvenRow = 5;
var builder = new StringBuilder(128);
for (var recordNum = 0; recordNum < NumRecords; recordNum++)
{
if (recordNum > 0)
{
builder.Append(RecordDelimeter);
}
var numValues = recordNum % 2 == 0 ? NumValuesEvenRow : NumValuesOddRow;
AddRecord(builder, ValueDelimeter, recordNum, numValues);
}
var reader = new DsvReader(builder.ToString(), ValueDelimeter, RecordDelimeter);
Assert.IsTrue(reader.Current.IsEmpty);
Assert.AreEqual(-1, reader.CurrentIndex);
for (var recordNum = 0; recordNum < NumRecords; recordNum++)
{
Assert.IsTrue(reader.MoveNextRecord());
Assert.IsTrue(reader.Current.IsEmpty);
Assert.AreEqual(-1, reader.CurrentIndex);
var numValues = recordNum % 2 == 0 ? NumValuesEvenRow : NumValuesOddRow;
for (var valueNum = 0; valueNum < numValues; valueNum++)
{
Assert.IsTrue(reader.MoveNextValue());
var value = reader.Current;
Assert.AreEqual($"Value {recordNum}-{valueNum}", value.ToString());
}
Assert.IsFalse(reader.MoveNextValue());
Assert.IsTrue(reader.Current.IsEmpty);
Assert.AreEqual(-1, reader.CurrentIndex);
}
Assert.IsFalse(reader.MoveNextRecord());
Assert.IsTrue(reader.Current.IsEmpty);
Assert.AreEqual(-1, reader.CurrentIndex);
}
public int Beffyman_DsvReaderWithValue()
{
var reader = new DsvReader(MemoryFile, Encoding.UTF8, DsvOptions.DefaultCsvOptions);
int totalLength = 0;
while (reader.MoveNext())
{
var val = reader.ReadNextAsSpan();
totalLength += val.Length;
}
return(totalLength);
}
public int DelimiterSeparatedTextReader()
{
var totalLength = 0;
var reader = new DsvReader(this.str, CsvReader.ValueDelimeter, CsvReader.RecordDelimeter);
while (reader.MoveNextRecord())
{
while (reader.MoveNextValue())
{
var value = reader.Current;
totalLength += value.Length;
}
}
return(totalLength);
}
public void Constructor_Span()
{
string val = "Data1,Data2,Data3";
Span <char> span = new Span <char>(val.ToCharArray());
var data = new DsvReader(span, System.Text.Encoding.UTF8, new DsvOptions(',', '"', false));
List <ReadOnlyMemory <char> > values = new List <ReadOnlyMemory <char> >();
while (data.MoveNext())
{
values.Add(data.ReadNextAsMemory());
}
Assert.Equal(val, string.Join(",", values));
}
public void ReadLine()
{
string file = $"Column1,Column2,Column3{Environment.NewLine}Data1,Data2,Data3";
var data = new DsvReader(file, System.Text.Encoding.UTF8, DsvOptions.DefaultCsvOptions);
int rows = 0;
Assert.Equal(rows, data.RowCount);
while (data.MoveNextLine())
{
rows++;
Assert.Equal(rows, data.RowCount);
}
Assert.Equal(rows, data.RowCount);
}
public void ReadNextSequential()
{
Span <string> span = new Span <string>(new string[10]);
span.Fill("Data");
var file = string.Join(",", span.ToArray());
var data = new DsvReader(file, System.Text.Encoding.UTF8, DsvOptions.DefaultCsvOptions);
List <ReadOnlyMemory <char> > values = new List <ReadOnlyMemory <char> >();
while (data.MoveNext())
{
values.Add(data.ReadNextAsMemory());
}
Assert.Equal(span.ToArray(), values.Select(x => x.ToString()).ToArray());
}
private void EscapeBomWithEncoding(Encoding encoding, string bomPrefix = null)
{
var BOM = bomPrefix ?? encoding.GetString(encoding.GetPreamble());
string file = $"{BOM}\"Column1\",\"Column2\",\"Column3\"{Environment.NewLine}\"\",Data2,Data3{Environment.NewLine},\"\",";
var data = new DsvReader(file, encoding, DsvOptions.DefaultCsvOptions);
List <ReadOnlyMemory <char> > columns = null;
List <List <ReadOnlyMemory <char> > > rows = new List <List <ReadOnlyMemory <char> > >();
while (data.MoveNext())
{
if (!data.ColumnsFilled)
{
columns = data.ReadLine().ToList();
}
else
{
rows.Add(data.ReadLine().ToList());
}
}
Assert.Equal(3, columns.Count);
Assert.Equal(2, rows.Count);
Assert.Equal("Column1", columns[0].ToString());
Assert.Equal("Column2", columns[1].ToString());
Assert.Equal("Column3", columns[2].ToString());
Assert.Equal("", rows[0][0].ToString());
Assert.Equal("Data2", rows[0][1].ToString());
Assert.Equal("Data3", rows[0][2].ToString());
Assert.Equal("", rows[1][0].ToString());
Assert.Equal("", rows[1][1].ToString());
Assert.Equal("", rows[1][2].ToString());
}
public void ColumnResize()
{
string file = FileGenerator("Column", "Data", 3, 5);
var data = new DsvReader(file, System.Text.Encoding.UTF8, DsvOptions.DefaultCsvOptions);
List <ReadOnlyMemory <char> > columns = null;
List <List <ReadOnlyMemory <char> > > rows = new List <List <ReadOnlyMemory <char> > >();
while (data.MoveNext())
{
if (!data.ColumnsFilled)
{
columns = data.ReadLine().ToList();
}
else
{
rows.Add(data.ReadLine().ToList());
}
}
Assert.Equal(5, columns.Count);
Assert.Equal(3, rows.Count);
Assert.Equal("Column1", columns[0].ToString());
Assert.Equal("Column2", columns[1].ToString());
Assert.Equal("Column3", columns[2].ToString());
Assert.Equal("Column4", columns[3].ToString());
Assert.Equal("Column5", columns[4].ToString());
for (int i = 0; i < 3; i++)
{
Assert.Equal("Data1", rows[i][0].ToString());
Assert.Equal("Data2", rows[i][1].ToString());
Assert.Equal("Data3", rows[i][2].ToString());
}
}
public void LongDelimeters()
{
const string ValueDelimeter = "This is a very long and terrible value delimeter";
const string RecordDelimeter = "This is a very long and terrible record delimeter";
const int NumRecords = 3;
const int NumValues = 4;
var builder = CreateTable(ValueDelimeter, RecordDelimeter, NumRecords, NumValues);
var reader = new DsvReader(builder.ToString(), ValueDelimeter, RecordDelimeter);
Assert.IsTrue(reader.Current.IsEmpty);
Assert.AreEqual(-1, reader.CurrentIndex);
for (var recordNum = 0; recordNum < NumRecords; recordNum++)
{
Assert.IsTrue(reader.MoveNextRecord());
Assert.IsTrue(reader.Current.IsEmpty);
Assert.AreEqual(-1, reader.CurrentIndex);
for (var valueNum = 0; valueNum < NumValues; valueNum++)
{
Assert.IsTrue(reader.MoveNextValue());
var value = reader.Current;
Assert.AreEqual($"Value {recordNum}-{valueNum}", value.ToString());
}
Assert.IsFalse(reader.MoveNextValue());
Assert.IsTrue(reader.Current.IsEmpty);
Assert.AreEqual(-1, reader.CurrentIndex);
}
Assert.IsFalse(reader.MoveNextRecord());
Assert.IsTrue(reader.Current.IsEmpty);
Assert.AreEqual(-1, reader.CurrentIndex);
}
public void LargeTable()
{
const string ValueDelimeter = "|";
const string RecordDelimeter = "\r\n";
const int NumRecords = 10_000;
const int NumValues = 100;
var builder = CreateTable(ValueDelimeter, RecordDelimeter, NumRecords, NumValues);
var reader = new DsvReader(builder.ToString(), ValueDelimeter, RecordDelimeter);
Assert.IsTrue(reader.Current.IsEmpty);
Assert.AreEqual(-1, reader.CurrentIndex);
for (var recordNum = 0; recordNum < NumRecords; recordNum++)
{
Assert.IsTrue(reader.MoveNextRecord());
Assert.IsTrue(reader.Current.IsEmpty);
Assert.AreEqual(-1, reader.CurrentIndex);
for (var valueNum = 0; valueNum < NumValues; valueNum++)
{
Assert.IsTrue(reader.MoveNextValue());
var value = reader.Current;
Assert.AreEqual($"Value {recordNum}-{valueNum}", value.ToString());
}
Assert.IsFalse(reader.MoveNextValue());
Assert.IsTrue(reader.Current.IsEmpty);
Assert.AreEqual(-1, reader.CurrentIndex);
}
Assert.IsFalse(reader.MoveNextRecord());
Assert.IsTrue(reader.Current.IsEmpty);
Assert.AreEqual(-1, reader.CurrentIndex);
}
public void Escaped_LastValue()
{
string file = $"\"Column1\",\"Column2\",\"Column3\"{Environment.NewLine}\"\",Data2,Data3{Environment.NewLine},\"\",\"HELLO\"{Environment.NewLine}";
var data = new DsvReader(file, System.Text.Encoding.UTF8, DsvOptions.DefaultCsvOptions);
List <ReadOnlyMemory <char> > columns = null;
List <List <ReadOnlyMemory <char> > > rows = new List <List <ReadOnlyMemory <char> > >();
while (data.MoveNext())
{
if (!data.ColumnsFilled)
{
columns = data.ReadLine().ToList();
}
else
{
rows.Add(data.ReadLine().ToList());
}
}
Assert.Equal(3, columns.Count);
Assert.Equal(2, rows.Count);
Assert.Equal("Column1", columns[0].ToString());
Assert.Equal("Column2", columns[1].ToString());
Assert.Equal("Column3", columns[2].ToString());
Assert.Equal("", rows[0][0].ToString());
Assert.Equal("Data2", rows[0][1].ToString());
Assert.Equal("Data3", rows[0][2].ToString());
Assert.Equal("", rows[1][0].ToString());
Assert.Equal("", rows[1][1].ToString());
Assert.Equal("HELLO", rows[1][2].ToString());
}
public void PublicPropertyState()
{
string file = FileGenerator("Column", "Data", 1, 3);
var data = new DsvReader(file, System.Text.Encoding.UTF8, DsvOptions.DefaultCsvOptions);
Assert.True(data.MoveNext());
var column1 = data.ReadNextAsMemory().ToString();
Assert.Equal("Column1", column1);
Assert.Empty(data.ReadNextAsMemory().ToString());
Assert.Equal(1, data.Column);
Assert.Equal(1, data.ColumnCount);
Assert.Equal(0, data.RowCount);
Assert.False(data.ColumnsFilled);
Assert.False(data.NewRowNextRead);
Assert.True(data.MoveNext());
var column2 = data.ReadNextAsMemory().ToString();
Assert.Equal("Column2", column2);
Assert.Empty(data.ReadNextAsMemory().ToString());
Assert.Equal(2, data.Column);
Assert.Equal(2, data.ColumnCount);
Assert.Equal(0, data.RowCount);
Assert.False(data.ColumnsFilled);
Assert.False(data.NewRowNextRead);
Assert.True(data.MoveNext());
var column3 = data.ReadNextAsMemory().ToString();
Assert.Equal("Column3", column3);
Assert.Empty(data.ReadNextAsMemory().ToString());
Assert.Equal(3, data.Column);
Assert.Equal(3, data.ColumnCount);
Assert.Equal(1, data.RowCount);
Assert.True(data.ColumnsFilled);
Assert.True(data.NewRowNextRead);
Assert.True(data.MoveNext());
var data1 = data.ReadNextAsMemory().ToString();
Assert.Equal("Data1", data1);
Assert.Empty(data.ReadNextAsMemory().ToString());
Assert.Equal(1, data.Column);
Assert.Equal(3, data.ColumnCount);
Assert.Equal(1, data.RowCount);
Assert.True(data.ColumnsFilled);
Assert.False(data.NewRowNextRead);
Assert.True(data.MoveNext());
var data2 = data.ReadNextAsMemory().ToString();
Assert.Equal("Data2", data2);
Assert.Empty(data.ReadNextAsMemory().ToString());
Assert.Equal(2, data.Column);
Assert.Equal(3, data.ColumnCount);
Assert.Equal(1, data.RowCount);
Assert.True(data.ColumnsFilled);
Assert.False(data.NewRowNextRead);
Assert.True(data.MoveNext());
var data3 = data.ReadNextAsMemory().ToString();
Assert.Equal("Data3", data3);
Assert.Empty(data.ReadNextAsMemory().ToString());
Assert.Equal(3, data.Column);
Assert.Equal(3, data.ColumnCount);
Assert.Equal(2, data.RowCount);
Assert.True(data.ColumnsFilled);
Assert.False(data.NewRowNextRead);
Assert.False(data.MoveNext());
}