private List <PageTag> WriteColumn(DataColumn column, Thrift.SchemaElement tse, IDataTypeHandler dataTypeHandler, int maxRepetitionLevel, int maxDefinitionLevel) { var pages = new List <PageTag>(); /* * Page header must preceeed actual data (compressed or not) however it contains both * the uncompressed and compressed data size which we don't know! This somehow limits * the write efficiency. */ using (var ms = new MemoryStream()) { Thrift.PageHeader dataPageHeader = _footer.CreateDataPage(column.TotalCount); //chain streams together so we have real streaming instead of wasting undefraggable LOH memory using (GapStream pageStream = DataStreamFactory.CreateWriter(ms, _compressionMethod, true)) { using (var writer = new BinaryWriter(pageStream, Encoding.UTF8, true)) { if (maxRepetitionLevel > 0) { WriteLevels(writer, column.RepetitionLevels, maxRepetitionLevel); } if (maxDefinitionLevel > 0) { WriteLevels(writer, column.DefinitionLevels, maxDefinitionLevel); } dataTypeHandler.Write(tse, writer, column.DefinedData); writer.Flush(); } pageStream.Flush(); //extremely important to flush the stream as some compression algorithms don't finish writing dataPageHeader.Uncompressed_page_size = (int)pageStream.Position; } dataPageHeader.Compressed_page_size = (int)ms.Position; //write the header in int headerSize = _thriftStream.Write(dataPageHeader); ms.Position = 0; ms.CopyTo(_stream); var dataTag = new PageTag { HeaderMeta = dataPageHeader, HeaderSize = headerSize }; pages.Add(dataTag); } return(pages); }
private List <PageTag> WriteColumn(DataColumn column, Thrift.SchemaElement tse, IDataTypeHandler dataTypeHandler, int maxRepetitionLevel, int maxDefinitionLevel) { var pages = new List <PageTag>(); /* * Page header must preceeed actual data (compressed or not) however it contains both * the uncompressed and compressed data size which we don't know! This somehow limits * the write efficiency. */ using (var ms = new MemoryStream()) { Thrift.PageHeader dataPageHeader = _footer.CreateDataPage(column.TotalCount); //chain streams together so we have real streaming instead of wasting undefraggable LOH memory using (PositionTrackingStream pps = DataStreamFactory.CreateWriter(ms, _compressionMethod)) { using (var writer = new BinaryWriter(pps)) { if (column.HasRepetitions) { throw new NotImplementedException(); } if (column.HasDefinitions) { WriteLevels(writer, column.DefinitionLevels, maxDefinitionLevel); } dataTypeHandler.Write(tse, writer, column.DefinedData); } dataPageHeader.Uncompressed_page_size = (int)pps.Position; } dataPageHeader.Compressed_page_size = (int)ms.Position; //write the hader in int headerSize = _thriftStream.Write(dataPageHeader); ms.Position = 0; ms.CopyTo(_stream); var dataTag = new PageTag { HeaderMeta = dataPageHeader, HeaderSize = headerSize }; pages.Add(dataTag); } return(pages); }
private List <PageTag> WriteColumn(DataColumn column, Thrift.SchemaElement tse, IDataTypeHandler dataTypeHandler, int maxRepetitionLevel, int maxDefinitionLevel) { var pages = new List <PageTag>(); /* * Page header must preceeed actual data (compressed or not) however it contains both * the uncompressed and compressed data size which we don't know! This somehow limits * the write efficiency. */ using (var ms = new MemoryStream()) { Thrift.PageHeader dataPageHeader = _footer.CreateDataPage(column.Data.Length); //chain streams together so we have real streaming instead of wasting undefraggable LOH memory using (GapStream pageStream = DataStreamFactory.CreateWriter(ms, _compressionMethod, _compressionLevel, true)) { using (var writer = new BinaryWriter(pageStream, Encoding.UTF8, true)) { if (column.RepetitionLevels != null) { WriteLevels(writer, column.RepetitionLevels, column.RepetitionLevels.Length, maxRepetitionLevel); } Array data = column.Data; if (maxDefinitionLevel > 0) { data = column.PackDefinitions(maxDefinitionLevel, out int[] definitionLevels, out int definitionLevelsLength, out int nullCount); //last chance to capture null count as null data is compressed now column.Statistics.NullCount = nullCount; try { WriteLevels(writer, definitionLevels, definitionLevelsLength, maxDefinitionLevel); } finally { if (definitionLevels != null) { ArrayPool <int> .Shared.Return(definitionLevels); } } } else { //no defitions means no nulls column.Statistics.NullCount = 0; } dataTypeHandler.Write(tse, writer, data, column.Statistics); writer.Flush(); } pageStream.Flush(); //extremely important to flush the stream as some compression algorithms don't finish writing pageStream.MarkWriteFinished(); dataPageHeader.Uncompressed_page_size = (int)pageStream.Position; } dataPageHeader.Compressed_page_size = (int)ms.Position; //write the header in dataPageHeader.Data_page_header.Statistics = column.Statistics.ToThriftStatistics(dataTypeHandler, _schemaElement); int headerSize = _thriftStream.Write(dataPageHeader); ms.Position = 0; ms.CopyTo(_stream); var dataTag = new PageTag { HeaderMeta = dataPageHeader, HeaderSize = headerSize }; pages.Add(dataTag); } return(pages); }