protected object WriteReadSingle(DataField field, object value) { using (var ms = new MemoryStream()) { // write single value using (var writer = new ParquetWriter3(new Schema(field), ms)) { writer.CompressionMethod = CompressionMethod.None; using (ParquetRowGroupWriter rg = writer.CreateRowGroup(1)) { var column = new DataColumn(field); column.Add(value); rg.Write(column); } } // read back single value ms.Position = 0; using (var reader = new ParquetReader3(ms)) { foreach (ParquetRowGroupReader rowGroupReader in reader) { DataColumn column = rowGroupReader.ReadColumn(field); return(column.DefinedData.OfType <object>().FirstOrDefault()); } return(null); } } }
public void Write_simplest_int_and_string_columns_in_one_row_group() { var schema = new Schema(new DataField <int>("id"), new DataField <string>("name")); using (var ms = new MemoryStream()) { using (var writer = new ParquetWriter3(schema, ms)) { writer.CompressionMethod = CompressionMethod.None; using (ParquetRowGroupWriter group = writer.CreateRowGroup(3)) { group.Write(CreateColumn(schema[0], 1, 2, 3)); group.Write(CreateColumn(schema[1], "first", "second", "third")); } } } }
protected object WriteReadSingle(DataField field, object value, CompressionMethod compressionMethod = CompressionMethod.None) { //for sanity, use disconnected streams byte[] data; using (var ms = new MemoryStream()) { // write single value using (var writer = new ParquetWriter3(new Schema(field), ms)) { writer.CompressionMethod = compressionMethod; using (ParquetRowGroupWriter rg = writer.CreateRowGroup(1)) { var column = new DataColumn(field); column.Add(value); rg.Write(column); } } data = ms.ToArray(); //F.WriteAllBytes($"c:\\tmp\\{compressionMethod}.parquet", data); } using (var ms = new MemoryStream(data)) { // read back single value ms.Position = 0; using (var reader = new ParquetReader3(ms)) { using (ParquetRowGroupReader rowGroupReader = reader.OpenRowGroupReader(0)) { DataColumn column = rowGroupReader.ReadColumn(field); return(column.DefinedData.OfType <object>().FirstOrDefault()); } } } }
protected object WriteReadSingle( DataField field, object value, CompressionMethod compressionMethod = CompressionMethod.None, bool flushToDisk = false) { using (var ms = new MemoryStream()) { // write single value using (var writer = new ParquetWriter3(new Schema(field), ms)) { writer.CompressionMethod = compressionMethod; using (ParquetRowGroupWriter rg = writer.CreateRowGroup(1)) { var column = new DataColumn(field); column.Add(value); rg.Write(column); } } if (flushToDisk) { FlushTempFile(ms); } // read back single value ms.Position = 0; using (var reader = new ParquetReader3(ms)) { using (ParquetRowGroupReader rowGroupReader = reader.OpenRowGroupReader(0)) { DataColumn column = rowGroupReader.ReadColumn(field); return(column.DefinedData.OfType <object>().FirstOrDefault()); } } } }