public void Array_of_booleans_writes_and_reads() { var bools = new List <bool>(); bools.AddRange(new[] { true, false, true }); var schema = new SchemaElement <bool>("bool"); _writer.Write(_bw, schema, bools, out IList extra); _ms.Position = 0; var boolsRead = new List <bool?>(); _reader.Read(_br, schema, boolsRead, 3); Assert.Equal(bools, boolsRead.Cast <bool>().ToList()); }
private void WriteValues(SchemaElement schema, IList values, Thrift.PageHeader ph, CompressionMethod compression, ColumnStats stats) { byte[] data; using (var ms = new MemoryStream()) { using (var writer = new BinaryWriter(ms)) { if (stats.NullCount > 0) { CreateDefinitions(values, schema, out IList newValues, out List <int> definitions); values = newValues; _rleWriter.Write(writer, schema, definitions); } _plainWriter.Write(writer, schema, values); data = ms.ToArray(); } } ph.Uncompressed_page_size = data.Length; if (compression != CompressionMethod.None) { IDataWriter writer = DataFactory.GetWriter(compression); using (var ms = new MemoryStream()) { writer.Write(data, ms); data = ms.ToArray(); } ph.Compressed_page_size = data.Length; } else { ph.Compressed_page_size = ph.Uncompressed_page_size; } _thrift.Write(ph); _output.Write(data, 0, data.Length); }
private List <PageTag> WriteValues(SchemaElement schema, IList values, Thrift.PageHeader ph, CompressionMethod compression) { var result = new List <PageTag>(); byte[] dictionaryPageBytes = null; int dictionaryPageCount = 0; byte[] dataPageBytes; List <int> repetitions = null; List <int> definitions = null; //flatten values and create repetitions list if the field is repeatable if (schema.MaxRepetitionLevel > 0) { var rpack = new RepetitionPack(); values = rpack.Unpack(_schema, values, out repetitions); ph.Data_page_header.Num_values = values.Count; } if (schema.IsNullable || schema.MaxDefinitionLevel > 0) { var dpack = new DefinitionPack(); values = dpack.Unpack(values, _schema, out definitions); } using (var ms = new MemoryStream()) { using (var writer = new BinaryWriter(ms)) { //write repetitions if (repetitions != null) { int bitWidth = PEncoding.GetWidthFromMaxInt(_schema.MaxRepetitionLevel); RunLengthBitPackingHybridValuesWriter.Write(writer, bitWidth, repetitions); } //write definitions if (definitions != null) { int bitWidth = PEncoding.GetWidthFromMaxInt(_schema.MaxDefinitionLevel); RunLengthBitPackingHybridValuesWriter.Write(writer, bitWidth, definitions); } //write data if (!_writerOptions.UseDictionaryEncoding || !_dicWriter.Write(writer, schema, values, out IList dicValues)) { _plainWriter.Write(writer, schema, values, out IList plainExtra); } else { dictionaryPageCount = dicValues.Count; ph.Data_page_header.Encoding = Thrift.Encoding.PLAIN_DICTIONARY; using (var dms = new MemoryStream()) using (var dwriter = new BinaryWriter(dms)) { _plainWriter.Write(dwriter, schema, dicValues, out IList t0); dictionaryPageBytes = dms.ToArray(); } } dataPageBytes = ms.ToArray(); } } if (dictionaryPageBytes != null) { Thrift.PageHeader dph = _meta.CreateDictionaryPage(dictionaryPageCount); dictionaryPageBytes = Compress(dph, dictionaryPageBytes, compression); int dictionaryHeaderSize = Write(dph, dictionaryPageBytes); result.Add(new PageTag { HeaderSize = dictionaryHeaderSize, HeaderMeta = dph }); } dataPageBytes = Compress(ph, dataPageBytes, compression); int dataHeaderSize = Write(ph, dataPageBytes); result.Add(new PageTag { HeaderSize = dataHeaderSize, HeaderMeta = ph }); return(result); }
private List <PageTag> WriteValues(SchemaElement schema, IList values, Thrift.PageHeader ph, CompressionMethod compression, ColumnStats stats) { var result = new List <PageTag>(); byte[] dictionaryPageBytes = null; int dictionaryPageCount = 0; byte[] dataPageBytes; //flatten values if the field is repeatable if (schema.IsRepeated) { values = FlattenRepeatables(values, schema); } using (var ms = new MemoryStream()) { using (var writer = new BinaryWriter(ms)) { //write repetitions if (schema.IsRepeated) { List <int> repetitions = CreateRepetitions(values, schema); _rleWriter.Write(writer, _definitionsSchema, repetitions, out IList nullExtra); } //write definitions if (schema.HasNulls || schema.IsRepeated) { CreateDefinitions(values, schema, out IList newValues, out List <int> definitions); values = newValues; _rleWriter.Write(writer, _definitionsSchema, definitions, out IList nullExtra); } //write data if (!_writerOptions.UseDictionaryEncoding || !_dicWriter.Write(writer, schema, values, out IList dicValues)) { _plainWriter.Write(writer, schema, values, out IList plainExtra); } else { dictionaryPageCount = dicValues.Count; ph.Data_page_header.Encoding = Thrift.Encoding.PLAIN_DICTIONARY; using (var dms = new MemoryStream()) using (var dwriter = new BinaryWriter(dms)) { _plainWriter.Write(dwriter, schema, dicValues, out IList t0); dictionaryPageBytes = dms.ToArray(); } } dataPageBytes = ms.ToArray(); } } if (dictionaryPageBytes != null) { Thrift.PageHeader dph = _meta.CreateDictionaryPage(dictionaryPageCount); dictionaryPageBytes = Compress(dph, dictionaryPageBytes, compression); int dictionaryHeaderSize = Write(dph, dictionaryPageBytes); result.Add(new PageTag { HeaderSize = dictionaryHeaderSize, HeaderMeta = dph }); } dataPageBytes = Compress(ph, dataPageBytes, compression); int dataHeaderSize = Write(ph, dataPageBytes); result.Add(new PageTag { HeaderSize = dataHeaderSize, HeaderMeta = ph }); return(result); }