protected object WriteReadSingle(DataField field, object value)
{
using (var ms = new MemoryStream())
{
// write single value
using (var writer = new ParquetWriter3(new Schema(field), ms))
{
writer.CompressionMethod = CompressionMethod.None;
using (ParquetRowGroupWriter rg = writer.CreateRowGroup(1))
{
var column = new DataColumn(field);
column.Add(value);
rg.Write(column);
}
}
// read back single value
ms.Position = 0;
using (var reader = new ParquetReader3(ms))
{
foreach (ParquetRowGroupReader rowGroupReader in reader)
{
DataColumn column = rowGroupReader.ReadColumn(field);
return(column.DefinedData.OfType <object>().FirstOrDefault());
}
return(null);
}
}
}
internal DataColumn WriteReadSingleColumn(DataField field, int rowCount, DataColumn dataColumn, bool flushToDisk = false)
{
using (var ms = new MemoryStream())
{
// write with built-in extension method
ms.WriteSingleRowGroup(new Schema(field), rowCount, dataColumn);
ms.Position = 0;
if (flushToDisk)
{
FlushTempFile(ms);
}
// read first gow group and first column
using (var reader = new ParquetReader3(ms))
{
if (reader.RowGroupCount == 0)
{
return(null);
}
ParquetRowGroupReader rgReader = reader.OpenRowGroupReader(0);
return(rgReader.ReadColumn(field));
}
}
}
public void Serialise_single_row_group()
{
List <SimpleStructure> structures = Enumerable
.Range(0, 10)
.Select(i => new SimpleStructure {
Id = i, Name = $"row {i}"
})
.ToList();
using (var ms = new MemoryStream())
{
Schema schema = ParquetConvert.Serialize(structures, ms, compressionMethod: CompressionMethod.None);
ms.Position = 0;
using (var reader = new ParquetReader3(ms))
{
Assert.Equal(1, reader.RowGroupCount);
using (ParquetRowGroupReader rgr = reader.OpenRowGroupReader(0))
{
DataColumn ids = rgr.ReadColumn(schema.DataFieldAt(0));
DataColumn names = rgr.ReadColumn(schema.DataFieldAt(1));
Assert.Equal(10, ids.TotalCount);
Assert.Equal(10, names.TotalCount);
}
}
}
}
protected object WriteReadSingle(DataField field, object value, CompressionMethod compressionMethod = CompressionMethod.None)
{
//for sanity, use disconnected streams
byte[] data;
using (var ms = new MemoryStream())
{
// write single value
using (var writer = new ParquetWriter3(new Schema(field), ms))
{
writer.CompressionMethod = compressionMethod;
using (ParquetRowGroupWriter rg = writer.CreateRowGroup(1))
{
var column = new DataColumn(field);
column.Add(value);
rg.Write(column);
}
}
data = ms.ToArray();
//F.WriteAllBytes($"c:\\tmp\\{compressionMethod}.parquet", data);
}
using (var ms = new MemoryStream(data))
{
// read back single value
ms.Position = 0;
using (var reader = new ParquetReader3(ms))
{
using (ParquetRowGroupReader rowGroupReader = reader.OpenRowGroupReader(0))
{
DataColumn column = rowGroupReader.ReadColumn(field);
return(column.DefinedData.OfType <object>().FirstOrDefault());
}
}
}
}
protected object WriteReadSingle(
DataField field, object value,
CompressionMethod compressionMethod = CompressionMethod.None,
bool flushToDisk = false)
{
using (var ms = new MemoryStream())
{
// write single value
using (var writer = new ParquetWriter3(new Schema(field), ms))
{
writer.CompressionMethod = compressionMethod;
using (ParquetRowGroupWriter rg = writer.CreateRowGroup(1))
{
var column = new DataColumn(field);
column.Add(value);
rg.Write(column);
}
}
if (flushToDisk)
{
FlushTempFile(ms);
}
// read back single value
ms.Position = 0;
using (var reader = new ParquetReader3(ms))
{
using (ParquetRowGroupReader rowGroupReader = reader.OpenRowGroupReader(0))
{
DataColumn column = rowGroupReader.ReadColumn(field);
return(column.DefinedData.OfType <object>().FirstOrDefault());
}
}
}
}
