Esempio n. 1
0
 /**
  * Create a tree writer.
  * @param columnId the column id of the column to write
  * @param inspector the object inspector to use
  * @param schema the row schema
  * @param streamFactory limited access to the Writer's data.
  * @param nullable can the value be null?
  * @
  */
 protected TreeWriter(
     int columnId,
     ObjectInspector inspector,
     TypeDescription schema,
     StreamFactory streamFactory,
     bool nullable)
 {
     this.streamFactory = streamFactory;
     this.isCompressed = streamFactory.isCompressed();
     this.id = columnId;
     this.inspector = inspector;
     if (nullable)
     {
         isPresentOutStream = streamFactory.createStream(id,
             OrcProto.Stream.Types.Kind.PRESENT);
         isPresent = new BitFieldWriter(isPresentOutStream, 1);
     }
     else
     {
         isPresent = null;
     }
     this.foundNulls = false;
     createBloomFilter = streamFactory.getBloomFilterColumns()[columnId];
     indexStatistics = ColumnStatisticsImpl.create(schema);
     stripeColStatistics = ColumnStatisticsImpl.create(schema);
     fileStatistics = ColumnStatisticsImpl.create(schema);
     childrenWriters = new TreeWriter[0];
     rowIndex = OrcProto.RowIndex.CreateBuilder();
     rowIndexEntry = OrcProto.RowIndexEntry.CreateBuilder();
     rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry);
     stripeStatsBuilders = new List<OrcProto.StripeStatistics.Builder>();
     if (streamFactory.buildIndex())
     {
         rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.ROW_INDEX);
     }
     else
     {
         rowIndexStream = null;
     }
     if (createBloomFilter)
     {
         bloomFilterEntry = OrcProto.BloomFilter.CreateBuilder();
         bloomFilterIndex = OrcProto.BloomFilterIndex.CreateBuilder();
         bloomFilterStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.BLOOM_FILTER);
         bloomFilter = new BloomFilter(streamFactory.getRowIndexStride(), streamFactory.getBloomFilterFPP());
     }
     else
     {
         bloomFilterEntry = null;
         bloomFilterIndex = null;
         bloomFilterStream = null;
         bloomFilter = null;
     }
 }
Esempio n. 2
0
 public StringBaseTreeWriter(int columnId,
                  ObjectInspector inspector,
                  TypeDescription schema,
                  StreamFactory writer,
                  bool nullable)
     : base(columnId, inspector, schema, writer, nullable)
 {
     this.isDirectV2 = isNewWriteFormat(writer);
     stringOutput = writer.createStream(id,
         OrcProto.Stream.Types.Kind.DICTIONARY_DATA);
     lengthOutput = createIntegerWriter(writer.createStream(id,
         OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer);
     rowOutput = createIntegerWriter(writer.createStream(id,
         OrcProto.Stream.Types.Kind.DATA), false, isDirectV2, writer);
     recordPosition(rowIndexPosition);
     rowIndexValueCount.Add(0L);
     buildIndex = writer.buildIndex();
     directStreamOutput = writer.createStream(id, OrcProto.Stream.Types.Kind.DATA);
     directLengthOutput = createIntegerWriter(writer.createStream(id,
         OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer);
     OrcFile.WriterOptions options = writer.getOptions();
     dictionaryKeySizeThreshold = options.getDictionaryKeySizeThreshold();
     strideDictionaryCheck = options.getStrideDictionaryCheck();
     doneDictionaryCheck = false;
 }