protected object WriteReadSingle(DataField field, object value)
{
using (var ms = new MemoryStream())
{
// write single value
using (var writer = new ParquetWriter3(new Schema(field), ms))
{
writer.CompressionMethod = CompressionMethod.None;
using (ParquetRowGroupWriter rg = writer.CreateRowGroup(1))
{
var column = new DataColumn(field);
column.Add(value);
rg.Write(column);
}
}
// read back single value
ms.Position = 0;
using (var reader = new ParquetReader3(ms))
{
foreach (ParquetRowGroupReader rowGroupReader in reader)
{
DataColumn column = rowGroupReader.ReadColumn(field);
return(column.DefinedData.OfType <object>().FirstOrDefault());
}
return(null);
}
}
}
public void Write_simplest_int_and_string_columns_in_one_row_group()
{
var schema = new Schema(new DataField <int>("id"), new DataField <string>("name"));
using (var ms = new MemoryStream())
{
using (var writer = new ParquetWriter3(schema, ms))
{
writer.CompressionMethod = CompressionMethod.None;
using (ParquetRowGroupWriter group = writer.CreateRowGroup(3))
{
group.Write(CreateColumn(schema[0], 1, 2, 3));
group.Write(CreateColumn(schema[1], "first", "second", "third"));
}
}
}
}
protected object WriteReadSingle(DataField field, object value, CompressionMethod compressionMethod = CompressionMethod.None)
{
//for sanity, use disconnected streams
byte[] data;
using (var ms = new MemoryStream())
{
// write single value
using (var writer = new ParquetWriter3(new Schema(field), ms))
{
writer.CompressionMethod = compressionMethod;
using (ParquetRowGroupWriter rg = writer.CreateRowGroup(1))
{
var column = new DataColumn(field);
column.Add(value);
rg.Write(column);
}
}
data = ms.ToArray();
//F.WriteAllBytes($"c:\\tmp\\{compressionMethod}.parquet", data);
}
using (var ms = new MemoryStream(data))
{
// read back single value
ms.Position = 0;
using (var reader = new ParquetReader3(ms))
{
using (ParquetRowGroupReader rowGroupReader = reader.OpenRowGroupReader(0))
{
DataColumn column = rowGroupReader.ReadColumn(field);
return(column.DefinedData.OfType <object>().FirstOrDefault());
}
}
}
}
protected object WriteReadSingle(
DataField field, object value,
CompressionMethod compressionMethod = CompressionMethod.None,
bool flushToDisk = false)
{
using (var ms = new MemoryStream())
{
// write single value
using (var writer = new ParquetWriter3(new Schema(field), ms))
{
writer.CompressionMethod = compressionMethod;
using (ParquetRowGroupWriter rg = writer.CreateRowGroup(1))
{
var column = new DataColumn(field);
column.Add(value);
rg.Write(column);
}
}
if (flushToDisk)
{
FlushTempFile(ms);
}
// read back single value
ms.Position = 0;
using (var reader = new ParquetReader3(ms))
{
using (ParquetRowGroupReader rowGroupReader = reader.OpenRowGroupReader(0))
{
DataColumn column = rowGroupReader.ReadColumn(field);
return(column.DefinedData.OfType <object>().FirstOrDefault());
}
}
}
}
