public RunLengthIntegerWriterV2(PositionedOutputStream output, bool signed, bool alignedBitpacking = true) { this.output = output; this.signed = signed; this.alignedBitpacking = alignedBitpacking; this.utils = new SerializationUtils(); clear(); }
public IntegerWriter createIntegerWriter(PositionedOutputStream output, bool signed, bool isDirectV2, StreamFactory writer) { if (isDirectV2) { bool alignedBitpacking = false; if (writer.getEncodingStrategy() == OrcFile.EncodingStrategy.SPEED) { alignedBitpacking = true; } return new RunLengthIntegerWriterV2(output, signed, alignedBitpacking); } else { return new RunLengthIntegerWriter(output, signed); } }
public BinaryTreeWriter( int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory writer, bool nullable) : base(columnId, inspector, schema, writer, nullable) { this.stream = writer.createStream(id, OrcProto.Stream.Types.Kind.DATA); this.isDirectV2 = isNewWriteFormat(writer); this.length = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer); recordPosition(rowIndexPosition); }
/** * Create a tree writer. * @param columnId the column id of the column to write * @param inspector the object inspector to use * @param schema the row schema * @param streamFactory limited access to the Writer's data. * @param nullable can the value be null? * @ */ protected TreeWriter( int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory streamFactory, bool nullable) { this.streamFactory = streamFactory; this.isCompressed = streamFactory.isCompressed(); this.id = columnId; this.inspector = inspector; if (nullable) { isPresentOutStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.PRESENT); isPresent = new BitFieldWriter(isPresentOutStream, 1); } else { isPresent = null; } this.foundNulls = false; createBloomFilter = streamFactory.getBloomFilterColumns()[columnId]; indexStatistics = ColumnStatisticsImpl.create(schema); stripeColStatistics = ColumnStatisticsImpl.create(schema); fileStatistics = ColumnStatisticsImpl.create(schema); childrenWriters = new TreeWriter[0]; rowIndex = OrcProto.RowIndex.CreateBuilder(); rowIndexEntry = OrcProto.RowIndexEntry.CreateBuilder(); rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry); stripeStatsBuilders = new List<OrcProto.StripeStatistics.Builder>(); if (streamFactory.buildIndex()) { rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.ROW_INDEX); } else { rowIndexStream = null; } if (createBloomFilter) { bloomFilterEntry = OrcProto.BloomFilter.CreateBuilder(); bloomFilterIndex = OrcProto.BloomFilterIndex.CreateBuilder(); bloomFilterStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.BLOOM_FILTER); bloomFilter = new BloomFilter(streamFactory.getRowIndexStride(), streamFactory.getBloomFilterFPP()); } else { bloomFilterEntry = null; bloomFilterIndex = null; bloomFilterStream = null; bloomFilter = null; } }
public StringBaseTreeWriter(int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory writer, bool nullable) : base(columnId, inspector, schema, writer, nullable) { this.isDirectV2 = isNewWriteFormat(writer); stringOutput = writer.createStream(id, OrcProto.Stream.Types.Kind.DICTIONARY_DATA); lengthOutput = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer); rowOutput = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Types.Kind.DATA), false, isDirectV2, writer); recordPosition(rowIndexPosition); rowIndexValueCount.Add(0L); buildIndex = writer.buildIndex(); directStreamOutput = writer.createStream(id, OrcProto.Stream.Types.Kind.DATA); directLengthOutput = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer); OrcFile.WriterOptions options = writer.getOptions(); dictionaryKeySizeThreshold = options.getDictionaryKeySizeThreshold(); strideDictionaryCheck = options.getStrideDictionaryCheck(); doneDictionaryCheck = false; }
public FloatTreeWriter(int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory writer, bool nullable) : base(columnId, inspector, schema, writer, nullable) { this.stream = writer.createStream(id, OrcProto.Stream.Types.Kind.DATA); this.utils = new SerializationUtils(); recordPosition(rowIndexPosition); }
public DecimalTreeWriter(int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory writer, bool nullable) : base(columnId, inspector, schema, writer, nullable) { this.isDirectV2 = isNewWriteFormat(writer); valueStream = writer.createStream(id, OrcProto.Stream.Types.Kind.DATA); this.scaleStream = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Types.Kind.SECONDARY), true, isDirectV2, writer); recordPosition(rowIndexPosition); }
public RunLengthIntegerWriterV2(PositionedOutputStream output, bool signed, bool alignedBitpacking = true) { this.output = output; this.signed = signed; this.alignedBitpacking = alignedBitpacking; this.utils = new SerializationUtils(); clear(); }