public override void merge(ColumnStatisticsImpl other) { if (other is DateStatisticsImpl) { DateStatisticsImpl dateStats = (DateStatisticsImpl)other; if (minimum == null) { minimum = dateStats.minimum; maximum = dateStats.maximum; } else if (dateStats.minimum != null) { if (minimum > dateStats.minimum) { minimum = dateStats.minimum; } if (maximum < dateStats.maximum) { maximum = dateStats.maximum; } } } else { if (isStatsExists() && minimum != null) { throw new ArgumentException("Incompatible merging of date column statistics"); } } base.merge(other); }
public override void merge(ColumnStatisticsImpl other) { if (other is DoubleStatisticsImpl) { DoubleStatisticsImpl dbl = (DoubleStatisticsImpl)other; if (!hasMinimum) { hasMinimum = dbl.hasMinimum; minimum = dbl.minimum; maximum = dbl.maximum; } else if (dbl.hasMinimum) { if (dbl.minimum < minimum) { minimum = dbl.minimum; } if (dbl.maximum > maximum) { maximum = dbl.maximum; } } sum += dbl.sum; } else { if (isStatsExists() && hasMinimum) { throw new ArgumentException("Incompatible merging of double column statistics"); } } base.merge(other); }
private static void writeRowGroupIndexes(JsonWriter writer, int col, OrcProto.RowIndex[] rowGroupIndex) { OrcProto.RowIndex index; if (rowGroupIndex == null || (col >= rowGroupIndex.Length) || ((index = rowGroupIndex[col]) == null)) { return; } writer.key("rowGroupIndexes").array(); for (int entryIx = 0; entryIx < index.EntryCount; ++entryIx) { writer.newObject(); writer.key("entryId").value(entryIx); OrcProto.RowIndexEntry entry = index.EntryList[entryIx]; if (entry == null) { continue; } OrcProto.ColumnStatistics colStats = entry.Statistics; writeColumnStatistics(writer, ColumnStatisticsImpl.deserialize(colStats)); writer.key("positions").array(); for (int posIx = 0; posIx < entry.PositionsCount; ++posIx) { writer.value(entry.PositionsList[posIx]); } writer.endArray(); writer.endObject(); } writer.endArray(); }
public override void merge(ColumnStatisticsImpl other) { if (other is TimestampStatisticsImpl) { TimestampStatisticsImpl timestampStats = (TimestampStatisticsImpl)other; if (minimum == null) { minimum = timestampStats.minimum; maximum = timestampStats.maximum; } else if (timestampStats.minimum != null) { if (minimum > timestampStats.minimum) { minimum = timestampStats.minimum; } if (maximum < timestampStats.maximum) { maximum = timestampStats.maximum; } } } else { if (isStatsExists() && minimum != null) { throw new ArgumentException("Incompatible merging of timestamp column statistics"); } } base.merge(other); }
public ColumnStatistics[] getStatistics() { ColumnStatistics[] result = new ColumnStatistics[types.Count]; for (int i = 0; i < result.Length; ++i) { result[i] = ColumnStatisticsImpl.deserialize(fileStats[i]); } return(result); }
public override void merge(ColumnStatisticsImpl other) { if (other is BinaryColumnStatistics) { BinaryStatisticsImpl bin = (BinaryStatisticsImpl)other; sum += bin.sum; } else { if (isStatsExists() && sum != 0) { throw new ArgumentException("Incompatible merging of binary column statistics"); } } base.merge(other); }
public override void merge(ColumnStatisticsImpl other) { if (other is BooleanStatisticsImpl) { BooleanStatisticsImpl bkt = (BooleanStatisticsImpl)other; trueCount += bkt.trueCount; } else { if (isStatsExists() && trueCount != 0) { throw new ArgumentException("Incompatible merging of bool column statistics"); } } base.merge(other); }
public override void merge(ColumnStatisticsImpl other) { if (other is IntegerStatisticsImpl) { IntegerStatisticsImpl otherInt = (IntegerStatisticsImpl)other; if (!hasMinimum) { hasMinimum = otherInt.hasMinimum; minimum = otherInt.minimum; maximum = otherInt.maximum; } else if (otherInt.hasMinimum) { if (otherInt.minimum < minimum) { minimum = otherInt.minimum; } if (otherInt.maximum > maximum) { maximum = otherInt.maximum; } } overflow |= otherInt.overflow; if (!overflow) { bool wasPositive = sum >= 0; sum += otherInt.sum; if ((otherInt.sum >= 0) == wasPositive) { overflow = (sum >= 0) != wasPositive; } } } else { if (isStatsExists() && hasMinimum) { throw new ArgumentException("Incompatible merging of integer column statistics"); } } base.merge(other); }
public override void merge(ColumnStatisticsImpl other) { if (other is DecimalStatisticsImpl) { DecimalStatisticsImpl dec = (DecimalStatisticsImpl)other; if (minimum == null) { minimum = dec.minimum; maximum = dec.maximum; sum = dec.sum; } else if (dec.minimum != null) { if (minimum.CompareTo(dec.minimum) > 0) { minimum = dec.minimum; } if (maximum.CompareTo(dec.maximum) < 0) { maximum = dec.maximum; } if (sum == null || dec.sum == null) { sum = null; } else { sum += dec.sum; } } } else { if (isStatsExists() && minimum != null) { throw new ArgumentException("Incompatible merging of decimal column statistics"); } } base.merge(other); }
public override void merge(ColumnStatisticsImpl other) { if (other is StringStatisticsImpl) { StringStatisticsImpl str = (StringStatisticsImpl)other; if (minimum == null) { if (str.minimum != null) { maximum = str.getMaximum(); minimum = str.getMinimum(); } else { /* both are empty */ maximum = minimum = null; } } else if (str.minimum != null) { if (string.CompareOrdinal(minimum, str.minimum) > 0) { minimum = str.minimum; } if (string.CompareOrdinal(maximum, str.maximum) < 0) { maximum = str.maximum; } } sum += str.sum; } else { if (isStatsExists() && minimum != null) { throw new ArgumentException("Incompatible merging of string column statistics"); } } base.merge(other); }
/** * Create a tree writer. * @param columnId the column id of the column to write * @param inspector the object inspector to use * @param schema the row schema * @param streamFactory limited access to the Writer's data. * @param nullable can the value be null? * @ */ protected TreeWriter( int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory streamFactory, bool nullable) { this.streamFactory = streamFactory; this.isCompressed = streamFactory.isCompressed(); this.id = columnId; this.inspector = inspector; if (nullable) { isPresentOutStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.PRESENT); isPresent = new BitFieldWriter(isPresentOutStream, 1); } else { isPresent = null; } this.foundNulls = false; createBloomFilter = streamFactory.getBloomFilterColumns()[columnId]; indexStatistics = ColumnStatisticsImpl.create(schema); stripeColStatistics = ColumnStatisticsImpl.create(schema); fileStatistics = ColumnStatisticsImpl.create(schema); childrenWriters = new TreeWriter[0]; rowIndex = OrcProto.RowIndex.CreateBuilder(); rowIndexEntry = OrcProto.RowIndexEntry.CreateBuilder(); rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry); stripeStatsBuilders = new List<OrcProto.StripeStatistics.Builder>(); if (streamFactory.buildIndex()) { rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.ROW_INDEX); } else { rowIndexStream = null; } if (createBloomFilter) { bloomFilterEntry = OrcProto.BloomFilter.CreateBuilder(); bloomFilterIndex = OrcProto.BloomFilterIndex.CreateBuilder(); bloomFilterStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.BLOOM_FILTER); bloomFilter = new BloomFilter(streamFactory.getRowIndexStride(), streamFactory.getBloomFilterFPP()); } else { bloomFilterEntry = null; bloomFilterIndex = null; bloomFilterStream = null; bloomFilter = null; } }
public virtual void merge(ColumnStatisticsImpl stats) { count += stats.count; _hasNull |= stats._hasNull; }