/** * Create a tree writer. * @param columnId the column id of the column to write * @param inspector the object inspector to use * @param schema the row schema * @param streamFactory limited access to the Writer's data. * @param nullable can the value be null? * @ */ protected TreeWriter( int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory streamFactory, bool nullable) { this.streamFactory = streamFactory; this.isCompressed = streamFactory.isCompressed(); this.id = columnId; this.inspector = inspector; if (nullable) { isPresentOutStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.PRESENT); isPresent = new BitFieldWriter(isPresentOutStream, 1); } else { isPresent = null; } this.foundNulls = false; createBloomFilter = streamFactory.getBloomFilterColumns()[columnId]; indexStatistics = ColumnStatisticsImpl.create(schema); stripeColStatistics = ColumnStatisticsImpl.create(schema); fileStatistics = ColumnStatisticsImpl.create(schema); childrenWriters = new TreeWriter[0]; rowIndex = OrcProto.RowIndex.CreateBuilder(); rowIndexEntry = OrcProto.RowIndexEntry.CreateBuilder(); rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry); stripeStatsBuilders = new List<OrcProto.StripeStatistics.Builder>(); if (streamFactory.buildIndex()) { rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.ROW_INDEX); } else { rowIndexStream = null; } if (createBloomFilter) { bloomFilterEntry = OrcProto.BloomFilter.CreateBuilder(); bloomFilterIndex = OrcProto.BloomFilterIndex.CreateBuilder(); bloomFilterStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.BLOOM_FILTER); bloomFilter = new BloomFilter(streamFactory.getRowIndexStride(), streamFactory.getBloomFilterFPP()); } else { bloomFilterEntry = null; bloomFilterIndex = null; bloomFilterStream = null; bloomFilter = null; } }
public StringBaseTreeWriter(int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory writer, bool nullable) : base(columnId, inspector, schema, writer, nullable) { this.isDirectV2 = isNewWriteFormat(writer); stringOutput = writer.createStream(id, OrcProto.Stream.Types.Kind.DICTIONARY_DATA); lengthOutput = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer); rowOutput = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Types.Kind.DATA), false, isDirectV2, writer); recordPosition(rowIndexPosition); rowIndexValueCount.Add(0L); buildIndex = writer.buildIndex(); directStreamOutput = writer.createStream(id, OrcProto.Stream.Types.Kind.DATA); directLengthOutput = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer); OrcFile.WriterOptions options = writer.getOptions(); dictionaryKeySizeThreshold = options.getDictionaryKeySizeThreshold(); strideDictionaryCheck = options.getStrideDictionaryCheck(); doneDictionaryCheck = false; }