public override void merge(ColumnStatisticsImpl other)
{
if (other is DateStatisticsImpl)
{
DateStatisticsImpl dateStats = (DateStatisticsImpl)other;
if (minimum == null)
{
minimum = dateStats.minimum;
maximum = dateStats.maximum;
}
else if (dateStats.minimum != null)
{
if (minimum > dateStats.minimum)
{
minimum = dateStats.minimum;
}
if (maximum < dateStats.maximum)
{
maximum = dateStats.maximum;
}
}
}
else
{
if (isStatsExists() && minimum != null)
{
throw new ArgumentException("Incompatible merging of date column statistics");
}
}
base.merge(other);
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is DoubleStatisticsImpl)
{
DoubleStatisticsImpl dbl = (DoubleStatisticsImpl)other;
if (!hasMinimum)
{
hasMinimum = dbl.hasMinimum;
minimum = dbl.minimum;
maximum = dbl.maximum;
}
else if (dbl.hasMinimum)
{
if (dbl.minimum < minimum)
{
minimum = dbl.minimum;
}
if (dbl.maximum > maximum)
{
maximum = dbl.maximum;
}
}
sum += dbl.sum;
}
else
{
if (isStatsExists() && hasMinimum)
{
throw new ArgumentException("Incompatible merging of double column statistics");
}
}
base.merge(other);
}
private static void writeRowGroupIndexes(JsonWriter writer, int col,
OrcProto.RowIndex[] rowGroupIndex)
{
OrcProto.RowIndex index;
if (rowGroupIndex == null || (col >= rowGroupIndex.Length) ||
((index = rowGroupIndex[col]) == null))
{
return;
}
writer.key("rowGroupIndexes").array();
for (int entryIx = 0; entryIx < index.EntryCount; ++entryIx)
{
writer.newObject();
writer.key("entryId").value(entryIx);
OrcProto.RowIndexEntry entry = index.EntryList[entryIx];
if (entry == null)
{
continue;
}
OrcProto.ColumnStatistics colStats = entry.Statistics;
writeColumnStatistics(writer, ColumnStatisticsImpl.deserialize(colStats));
writer.key("positions").array();
for (int posIx = 0; posIx < entry.PositionsCount; ++posIx)
{
writer.value(entry.PositionsList[posIx]);
}
writer.endArray();
writer.endObject();
}
writer.endArray();
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is TimestampStatisticsImpl)
{
TimestampStatisticsImpl timestampStats = (TimestampStatisticsImpl)other;
if (minimum == null)
{
minimum = timestampStats.minimum;
maximum = timestampStats.maximum;
}
else if (timestampStats.minimum != null)
{
if (minimum > timestampStats.minimum)
{
minimum = timestampStats.minimum;
}
if (maximum < timestampStats.maximum)
{
maximum = timestampStats.maximum;
}
}
}
else
{
if (isStatsExists() && minimum != null)
{
throw new ArgumentException("Incompatible merging of timestamp column statistics");
}
}
base.merge(other);
}
public ColumnStatistics[] getStatistics()
{
ColumnStatistics[] result = new ColumnStatistics[types.Count];
for (int i = 0; i < result.Length; ++i)
{
result[i] = ColumnStatisticsImpl.deserialize(fileStats[i]);
}
return(result);
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is BinaryColumnStatistics)
{
BinaryStatisticsImpl bin = (BinaryStatisticsImpl)other;
sum += bin.sum;
}
else
{
if (isStatsExists() && sum != 0)
{
throw new ArgumentException("Incompatible merging of binary column statistics");
}
}
base.merge(other);
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is BooleanStatisticsImpl)
{
BooleanStatisticsImpl bkt = (BooleanStatisticsImpl)other;
trueCount += bkt.trueCount;
}
else
{
if (isStatsExists() && trueCount != 0)
{
throw new ArgumentException("Incompatible merging of bool column statistics");
}
}
base.merge(other);
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is IntegerStatisticsImpl)
{
IntegerStatisticsImpl otherInt = (IntegerStatisticsImpl)other;
if (!hasMinimum)
{
hasMinimum = otherInt.hasMinimum;
minimum = otherInt.minimum;
maximum = otherInt.maximum;
}
else if (otherInt.hasMinimum)
{
if (otherInt.minimum < minimum)
{
minimum = otherInt.minimum;
}
if (otherInt.maximum > maximum)
{
maximum = otherInt.maximum;
}
}
overflow |= otherInt.overflow;
if (!overflow)
{
bool wasPositive = sum >= 0;
sum += otherInt.sum;
if ((otherInt.sum >= 0) == wasPositive)
{
overflow = (sum >= 0) != wasPositive;
}
}
}
else
{
if (isStatsExists() && hasMinimum)
{
throw new ArgumentException("Incompatible merging of integer column statistics");
}
}
base.merge(other);
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is DecimalStatisticsImpl)
{
DecimalStatisticsImpl dec = (DecimalStatisticsImpl)other;
if (minimum == null)
{
minimum = dec.minimum;
maximum = dec.maximum;
sum = dec.sum;
}
else if (dec.minimum != null)
{
if (minimum.CompareTo(dec.minimum) > 0)
{
minimum = dec.minimum;
}
if (maximum.CompareTo(dec.maximum) < 0)
{
maximum = dec.maximum;
}
if (sum == null || dec.sum == null)
{
sum = null;
}
else
{
sum += dec.sum;
}
}
}
else
{
if (isStatsExists() && minimum != null)
{
throw new ArgumentException("Incompatible merging of decimal column statistics");
}
}
base.merge(other);
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is StringStatisticsImpl)
{
StringStatisticsImpl str = (StringStatisticsImpl)other;
if (minimum == null)
{
if (str.minimum != null)
{
maximum = str.getMaximum();
minimum = str.getMinimum();
}
else
{
/* both are empty */
maximum = minimum = null;
}
}
else if (str.minimum != null)
{
if (string.CompareOrdinal(minimum, str.minimum) > 0)
{
minimum = str.minimum;
}
if (string.CompareOrdinal(maximum, str.maximum) < 0)
{
maximum = str.maximum;
}
}
sum += str.sum;
}
else
{
if (isStatsExists() && minimum != null)
{
throw new ArgumentException("Incompatible merging of string column statistics");
}
}
base.merge(other);
}
/**
* Create a tree writer.
* @param columnId the column id of the column to write
* @param inspector the object inspector to use
* @param schema the row schema
* @param streamFactory limited access to the Writer's data.
* @param nullable can the value be null?
* @
*/
protected TreeWriter(
int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory streamFactory,
bool nullable)
{
this.streamFactory = streamFactory;
this.isCompressed = streamFactory.isCompressed();
this.id = columnId;
this.inspector = inspector;
if (nullable)
{
isPresentOutStream = streamFactory.createStream(id,
OrcProto.Stream.Types.Kind.PRESENT);
isPresent = new BitFieldWriter(isPresentOutStream, 1);
}
else
{
isPresent = null;
}
this.foundNulls = false;
createBloomFilter = streamFactory.getBloomFilterColumns()[columnId];
indexStatistics = ColumnStatisticsImpl.create(schema);
stripeColStatistics = ColumnStatisticsImpl.create(schema);
fileStatistics = ColumnStatisticsImpl.create(schema);
childrenWriters = new TreeWriter[0];
rowIndex = OrcProto.RowIndex.CreateBuilder();
rowIndexEntry = OrcProto.RowIndexEntry.CreateBuilder();
rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry);
stripeStatsBuilders = new List<OrcProto.StripeStatistics.Builder>();
if (streamFactory.buildIndex())
{
rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.ROW_INDEX);
}
else
{
rowIndexStream = null;
}
if (createBloomFilter)
{
bloomFilterEntry = OrcProto.BloomFilter.CreateBuilder();
bloomFilterIndex = OrcProto.BloomFilterIndex.CreateBuilder();
bloomFilterStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.BLOOM_FILTER);
bloomFilter = new BloomFilter(streamFactory.getRowIndexStride(), streamFactory.getBloomFilterFPP());
}
else
{
bloomFilterEntry = null;
bloomFilterIndex = null;
bloomFilterStream = null;
bloomFilter = null;
}
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is BooleanStatisticsImpl)
{
BooleanStatisticsImpl bkt = (BooleanStatisticsImpl)other;
trueCount += bkt.trueCount;
}
else
{
if (isStatsExists() && trueCount != 0)
{
throw new ArgumentException("Incompatible merging of bool column statistics");
}
}
base.merge(other);
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is TimestampStatisticsImpl)
{
TimestampStatisticsImpl timestampStats = (TimestampStatisticsImpl)other;
if (minimum == null)
{
minimum = timestampStats.minimum;
maximum = timestampStats.maximum;
}
else if (timestampStats.minimum != null)
{
if (minimum > timestampStats.minimum)
{
minimum = timestampStats.minimum;
}
if (maximum < timestampStats.maximum)
{
maximum = timestampStats.maximum;
}
}
}
else
{
if (isStatsExists() && minimum != null)
{
throw new ArgumentException("Incompatible merging of timestamp column statistics");
}
}
base.merge(other);
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is DecimalStatisticsImpl)
{
DecimalStatisticsImpl dec = (DecimalStatisticsImpl)other;
if (minimum == null)
{
minimum = dec.minimum;
maximum = dec.maximum;
sum = dec.sum;
}
else if (dec.minimum != null)
{
if (minimum.CompareTo(dec.minimum) > 0)
{
minimum = dec.minimum;
}
if (maximum.CompareTo(dec.maximum) < 0)
{
maximum = dec.maximum;
}
if (sum == null || dec.sum == null)
{
sum = null;
}
else
{
sum += dec.sum;
}
}
}
else
{
if (isStatsExists() && minimum != null)
{
throw new ArgumentException("Incompatible merging of decimal column statistics");
}
}
base.merge(other);
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is DateStatisticsImpl)
{
DateStatisticsImpl dateStats = (DateStatisticsImpl)other;
if (minimum == null)
{
minimum = dateStats.minimum;
maximum = dateStats.maximum;
}
else if (dateStats.minimum != null)
{
if (minimum > dateStats.minimum)
{
minimum = dateStats.minimum;
}
if (maximum < dateStats.maximum)
{
maximum = dateStats.maximum;
}
}
}
else
{
if (isStatsExists() && minimum != null)
{
throw new ArgumentException("Incompatible merging of date column statistics");
}
}
base.merge(other);
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is StringStatisticsImpl)
{
StringStatisticsImpl str = (StringStatisticsImpl)other;
if (minimum == null)
{
if (str.minimum != null)
{
maximum = str.getMaximum();
minimum = str.getMinimum();
}
else
{
/* both are empty */
maximum = minimum = null;
}
}
else if (str.minimum != null)
{
if (string.CompareOrdinal(minimum, str.minimum) > 0)
{
minimum = str.minimum;
}
if (string.CompareOrdinal(maximum, str.maximum) < 0)
{
maximum = str.maximum;
}
}
sum += str.sum;
}
else
{
if (isStatsExists() && minimum != null)
{
throw new ArgumentException("Incompatible merging of string column statistics");
}
}
base.merge(other);
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is BinaryColumnStatistics)
{
BinaryStatisticsImpl bin = (BinaryStatisticsImpl)other;
sum += bin.sum;
}
else
{
if (isStatsExists() && sum != 0)
{
throw new ArgumentException("Incompatible merging of binary column statistics");
}
}
base.merge(other);
}
public virtual void merge(ColumnStatisticsImpl stats)
{
count += stats.count;
_hasNull |= stats._hasNull;
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is IntegerStatisticsImpl)
{
IntegerStatisticsImpl otherInt = (IntegerStatisticsImpl)other;
if (!hasMinimum)
{
hasMinimum = otherInt.hasMinimum;
minimum = otherInt.minimum;
maximum = otherInt.maximum;
}
else if (otherInt.hasMinimum)
{
if (otherInt.minimum < minimum)
{
minimum = otherInt.minimum;
}
if (otherInt.maximum > maximum)
{
maximum = otherInt.maximum;
}
}
overflow |= otherInt.overflow;
if (!overflow)
{
bool wasPositive = sum >= 0;
sum += otherInt.sum;
if ((otherInt.sum >= 0) == wasPositive)
{
overflow = (sum >= 0) != wasPositive;
}
}
}
else
{
if (isStatsExists() && hasMinimum)
{
throw new ArgumentException("Incompatible merging of integer column statistics");
}
}
base.merge(other);
}
public override void merge(ColumnStatisticsImpl other)
{
if (other is DoubleStatisticsImpl)
{
DoubleStatisticsImpl dbl = (DoubleStatisticsImpl)other;
if (!hasMinimum)
{
hasMinimum = dbl.hasMinimum;
minimum = dbl.minimum;
maximum = dbl.maximum;
}
else if (dbl.hasMinimum)
{
if (dbl.minimum < minimum)
{
minimum = dbl.minimum;
}
if (dbl.maximum > maximum)
{
maximum = dbl.maximum;
}
}
sum += dbl.sum;
}
else
{
if (isStatsExists() && hasMinimum)
{
throw new ArgumentException("Incompatible merging of double column statistics");
}
}
base.merge(other);
}
public virtual void merge(ColumnStatisticsImpl stats)
{
count += stats.count;
_hasNull |= stats._hasNull;
}