public void Datetime_as_null_writes()
{
var schemaElements = new List <Data.SchemaElement>();
schemaElements.Add(new SchemaElement <string>("primary-key"));
schemaElements.Add(new SchemaElement <DateTime>("as-at-date"));
var ds = new DataSet(schemaElements);
// row 1
var row1 = new List <object>(schemaElements.Count);
row1.Add(Guid.NewGuid().ToString());
row1.Add(DateTime.UtcNow.AddDays(-5));
ds.Add(new Row(row1));
// row 2
var row2 = new List <object>(schemaElements.Count);
row2.Add(Guid.NewGuid().ToString());
row2.Add(DateTime.UtcNow);
ds.Add(new Row(row2));
// row 3
var row3 = new List <object>(schemaElements.Count);
row3.Add(Guid.NewGuid().ToString());
row3.Add(null);
//objData3.Add(DateTime.UtcNow);
ds.Add(new Row(row3));
DataSet dsRead = DataSetGenerator.WriteRead(ds);
Assert.Equal(3, dsRead.RowCount);
}
public void Structure_in_a_list_in_a_structure_of_lists_writes_reads()
{
var ds = new DataSet(
new DataField <string>("name"),
new ListField("addresses",
new StructField("address",
new ListField("lines",
new StructField("first", new DataField <int>("one"))))));
ds.Add(
"Ivan", // name
new Row[] // addresses
{
new Row // addresses.address
(
true,
new Row[] // addresses.address.lines
{
new Row // addresses.address.lines.first
(
1 // addresses.address.lines.first.one
)
}
)
});
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal("{Ivan;[{[{1}]}]}", ds1[0].ToString());
}
public void Repeatable_field_writes_reads()
{
var ds = new DataSet(new DataField <int>("id"), new DataField <IEnumerable <string> >("repeats"));
ds.Add(1, new[] { "one", "two", "three" });
Assert.Equal("{1;[one;two;three]}", DataSetGenerator.WriteRead(ds)[0].ToString());
}
public void Simplest_write_read()
{
var ds = new DataSet(new SchemaElement <int>("id"));
ds.Add(1);
ds.Add(2);
DataSetGenerator.WriteRead(ds);
}
public void Writing_no_values_is_possible()
{
var ds = new DataSet(new DataField("id", DataType.Int32));
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal(0, ds1.TotalRowCount);
Assert.Equal(0, ds1.RowCount);
}
public void Setting_custom_metadata_keys_reads_them_back()
{
var ds = new DataSet(new DataField <int>("id"));
ds.Metadata.Custom["ivan"] = "is cool";
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal("is cool", ds1.Metadata.Custom["ivan"]);
}
public void Read_column_with_all_nulls()
{
var ds = new DataSet(new SchemaElement <int?>("id"))
{
new object[] { null },
new object[] { null }
};
DataSet ds1 = DataSetGenerator.WriteRead(ds);
}
public void Run_perfect_expressive_boolean_column()
{
var schema = new Schema(new DataField("id", DataType.Boolean, false, false));
var ds = new DataSet(schema);
ds.Add(true);
ds.Add(false);
ds.Add(true);
DataSet ds1 = DataSetGenerator.WriteRead(ds);
}
public void Reads_created_by_metadata()
{
DataSet ds = DataSetGenerator.Generate(10);
var ms = new MemoryStream();
ParquetWriter.Write(ds, ms);
ms.Position = 0;
DataSet ds1 = ParquetReader.Read(ms);
Assert.StartsWith("parquet-dotnet", ds1.Metadata.CreatedBy);
}
public void Simple_repeated_field_write_read()
{
var ds = new DataSet(
new SchemaElement <int>("id"),
new SchemaElement <IEnumerable <string> >("items"));
ds.Add(1, new[] { "one", "two" });
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal(1, ds1[0][0]);
Assert.Equal(new[] { "one", "two" }, ds1[0][1]);
}
public void Write_simple_repeated_field()
{
var ds = new DataSet(
new SchemaElement <int>("id"),
new SchemaElement <IEnumerable <string> >("cities"));
ds.Add(1, new[] { "London", "Derby" });
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal("id", ds.Schema[0].Name);
Assert.Equal("cities", ds.Schema[1].Name);
}
public void Column_with_all_null_decimals_has_type_length()
{
var ds = new DataSet(new SchemaElement <int>("id"), new SchemaElement <decimal>("nulls"))
{
{ 1, null },
{ 2, null }
};
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Null(ds1[0][1]);
Assert.Null(ds1[1][1]);
}
public void Write_supposably_in_dictionary_encoding()
{
var ds = new DataSet(new SchemaElement <int>("id"), new SchemaElement <string>("dic_col"));
ds.Add(1, "one");
ds.Add(2, "one");
ds.Add(3, "one");
ds.Add(4, "one");
ds.Add(5, "one");
ds.Add(6, "two");
ds.Add(7, "two");
ds = DataSetGenerator.WriteRead(ds);
}
public void Structure_with_repeated_field_writes_reads()
{
var ds = new DataSet(
new DataField <string>("name"),
new StructField("address",
new DataField <string>("name"),
new DataField <IEnumerable <string> >("lines")));
ds.Add("Ivan", new Row("Primary", new[] { "line1", "line2" }));
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal("{Ivan;{Primary;[line1;line2]}}", ds1[0].ToString());
}
public void Simple_nested_structure_write_read()
{
var ds = new DataSet(
new SchemaElement <string>("name"),
new SchemaElement <Row>("address",
new SchemaElement <string>("line1"),
new SchemaElement <string>("postcode")
));
ds.Add("Ivan", new Row("Woods", "postcode"));
Assert.Equal(1, ds.RowCount);
Assert.Equal(2, ds.ColumnCount);
DataSet ds1 = DataSetGenerator.WriteRead(ds);
}
public void Interval_tests()
{
var ds = new DataSet(
new DataField <int>("id"),
new DataField <Interval>("interval"));
ds.Add(1000, new Interval(1, 2, 3));
DataSet ds1 = DataSetGenerator.WriteRead(ds);
var retInterval = (Interval)ds1[0][1];
Assert.Equal(1, retInterval.Months);
Assert.Equal(2, retInterval.Days);
Assert.Equal(3, retInterval.Millis);
}
public void List_of_structures_writes_reads()
{
var ds = new DataSet(
new DataField <int>("id"),
new ListField("cities",
new StructField("element",
new DataField <string>("name"),
new DataField <string>("country"))));
ds.Add(1, new[] { new Row("London", "UK"), new Row("New York", "US") });
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal("{1;[{London;UK};{New York;US}]}", ds1[0].ToString());
}
public void Structure_nested_into_structure_write_read()
{
var ds = new DataSet(
new DataField <string>("name"),
new StructField("address",
new DataField <string>("name"),
new StructField("lines",
new DataField <string>("line1"),
new DataField <string>("line2"))));
ds.Add("Ivan", new Row("Primary", new Row("line1", "line2")));
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal("{Ivan;{Primary;{line1;line2}}}", ds1[0].ToString());
}
public void Write_negative_decimals_in_legacy_mode()
{
var ds = new DataSet(new DecimalDataField("dec", 10, 2, true));
ds.Add(-1.0m);
ds.Add(-1.23m);
ds.Add(-23233.12m);
ds.Add(-999999.999m);
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal(-1.0m, (decimal)(ds1[0][0]), 2);
Assert.Equal(-1.23m, (decimal)(ds1[1][0]), 2);
Assert.Equal(-23233.12m, (decimal)(ds1[2][0]), 2);
Assert.Equal(-999999.99m, (decimal)(ds1[3][0]), 2);
}
public void Simple_first_level_map_int_to_string()
{
var ds = new DataSet(
new MapField("names", new DataField("key", DataType.Int32), new DataField("value", DataType.String)),
new DataField <int>("id"));
ds.Add(new Dictionary <int, string>
{
[1] = "one",
[2] = "two",
[3] = "three"
}, 1);
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal("{[1=>one;2=>two;3=>three];1}", ds1[0].ToString());
}
public void Simple_nested_structure_write_read()
{
var ds = new DataSet(
new SchemaElement <int>("id"),
new SchemaElement <Row>("address",
new SchemaElement <string>("line1"),
new SchemaElement <string>("postcode")
));
ds.Add(1, new Row("flat 4", "se11"));
Assert.Equal(1, ds.RowCount);
Assert.Equal(2, ds.ColumnCount);
DataSet ds1 = DataSetGenerator.WriteRead(ds);
//ParquetWriter.WriteFile(ds, "c:\\tmp\\firstnested.parquet");
}
public void Read_from_negative_offset_fails()
{
DataSet ds = DataSetGenerator.Generate(15);
var wo = new WriterOptions {
RowGroupsSize = 5
};
var ro = new ReaderOptions {
Offset = -4, Count = 2
};
var ms = new MemoryStream();
ParquetWriter.Write(ds, ms, CompressionMethod.None, null, wo);
ms.Position = 0;
Assert.Throws <ParquetException>(() => ParquetReader.Read(ms, null, ro));
}
public void Simple_structure_write_read()
{
var ds = new DataSet(
new DataField <string>("name"),
new StructField("address",
new DataField <string>("line1"),
new DataField <string>("postcode")
));
ds.Add("Ivan", new Row("Woods", "postcode"));
Assert.Equal(1, ds.RowCount);
Assert.Equal(2, ds.FieldCount);
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal("{Ivan;{Woods;postcode}}", ds1[0].ToString());
}
public void Repeated_structure_write_read()
{
var ds = new DataSet(
new SchemaElement <int>("id"),
new SchemaElement <IEnumerable <Row> >("cities",
new SchemaElement <string>("name"),
new SchemaElement <string>("country")));
Assert.Equal(4, ds.Schema["cities"].Children[1].MaxDefinitionLevel);
Assert.Equal(1, ds.Schema["cities"].Children[1].MaxRepetitionLevel);
ds.Add(1, new[] { new Row("London", "UK"), new Row("New York", "US") });
//ParquetWriter.WriteFile(ds, "c:\\tmp\\rep.parquet", CompressionMethod.None);
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal("{1;[{London;UK};{New York;US}]}", ds1[0].ToString());
}
public void Simple_first_level_map_int_to_string()
{
var ds = new DataSet(
new SchemaElement <IDictionary <int, string> >("names"),
new SchemaElement <int>("id"));
ds.Add(new Dictionary <int, string>
{
[1] = "one",
[2] = "two",
[3] = "three"
}, 1);
//ParquetWriter.WriteFile(ds, "c:\\tmp\\pmap2.parquet");
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal("{[1=>one;2=>two;3=>three];1}", ds1[0].ToString());
}
public void Write_loads_of_booleans_all_true(int count)
{
var ds = new DataSet(
new DataField <int>("id"),
new DataField <bool>("enabled"));
//generate data
for (int i = 0; i < count; i++)
{
ds.Add(i, true);
}
DataSet ds1 = DataSetGenerator.WriteRead(ds);
for (int i = 0; i < count; i++)
{
Assert.True(ds1[i].GetBoolean(1), $"got FALSE at position {i}");
}
}
public void List_of_elements_with_some_items_empty_writes_reads()
{
var ds = new DataSet(
new DataField <int>("id"),
new ListField("strings",
new DataField <string>("item")
));
ds.Add(1, new string[] { "1", "2", "3" });
ds.Add(2, new string[] { });
ds.Add(3, new string[] { "1", "2", "3" });
ds.Add(4, new string[] { });
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal(4, ds1.RowCount);
Assert.Equal("{1;[1;2;3]}", ds1[0].ToString());
Assert.Equal("{2;[]}", ds1[1].ToString());
Assert.Equal("{3;[1;2;3]}", ds1[2].ToString());
Assert.Equal("{4;[]}", ds1[3].ToString());
}
public void Read_from_offset_in_second_chunk()
{
DataSet ds = DataSetGenerator.Generate(15);
var wo = new WriterOptions {
RowGroupsSize = 5
};
var ro = new ReaderOptions {
Offset = 5, Count = 2
};
var ms = new MemoryStream();
ParquetWriter.Write(ds, ms, CompressionMethod.None, null, wo);
ms.Position = 0;
DataSet ds1 = ParquetReader.Read(ms, null, ro);
Assert.Equal(15, ds1.TotalRowCount);
Assert.Equal(2, ds1.RowCount);
Assert.Equal(5, ds1[0][0]);
Assert.Equal(6, ds1[1][0]);
}
public void Simple_repeated_field_write_read()
{
//v2
var ds = new DataSet(
new DataField <int>("id"),
new DataField <IEnumerable <string> >("items"));
ds.Add(1, new[] { "one", "two" });
DataSet ds1 = DataSetGenerator.WriteRead(ds);
Assert.Equal(1, ds1[0][0]);
Assert.Equal(new[] { "one", "two" }, ds1[0][1]);
//v3
// arrange
var field = new DataField <IEnumerable <int> >("items");
var column = new DataColumn(field);
column.IncrementLevel();
column.Add(1);
column.Add(2);
column.DecrementLevel();
column.IncrementLevel();
column.Add(3);
column.Add(4);
column.DecrementLevel();
// act
DataColumn rc = WriteReadSingleColumn(field, 2, column);
// assert
Assert.Equal(new int[] { 1, 2, 3, 4 }, rc.DefinedData);
Assert.Equal(new int[] { 1, 1, 1, 1 }, rc.DefinitionLevels);
Assert.Equal(new int[] { 0, 1, 0, 1 }, rc.RepetitionLevels);
}