private void DirectEncode(IEnumerable <long> values, int numValues, bool aligned,
int?precalculatedFixedBitWidth)
{
int fixedBitWidth;
if (precalculatedFixedBitWidth.HasValue)
{
fixedBitWidth = precalculatedFixedBitWidth.Value;
}
else
{
var histogram = values.GenerateHistogramOfBitWidths();
fixedBitWidth = BitManipulation.GetBitsRequiredForPercentile(histogram, 1.0);
}
fixedBitWidth = aligned
? BitManipulation.FindNearestAlignedDirectWidth(fixedBitWidth)
: BitManipulation.FindNearestDirectWidth(fixedBitWidth);
var encodedFixedBitWidth = fixedBitWidth.EncodeDirectWidth();
var byte1 = 0;
byte1 |= 0x1 << 6; //7..6 Encoding Type
byte1 |= (encodedFixedBitWidth & 0x1f) << 1; //5..1 Fixed Width
byte1 |= (numValues - 1) >> 8; //0 MSB of length
var byte2 = (numValues - 1) & 0xff; //7..0 LSBs of length
_outputStream.WriteByte((byte)byte1);
_outputStream.WriteByte((byte)byte2);
_outputStream.WriteBitpackedIntegers(values, fixedBitWidth);
}
private bool TryPatchEncoding(IEnumerable <long> zigZagValues, IList <long> values, long minValue,
ref int?fixedBitWidth, out int length)
{
var zigZagValuesHistogram = zigZagValues.GenerateHistogramOfBitWidths();
var zigZagHundredthBits = BitManipulation.GetBitsRequiredForPercentile(zigZagValuesHistogram, 1.0);
fixedBitWidth = zigZagHundredthBits; //We'll use this later if if end up DIRECT encoding
var zigZagNinetiethBits = BitManipulation.GetBitsRequiredForPercentile(zigZagValuesHistogram, 0.9);
if (zigZagHundredthBits - zigZagNinetiethBits == 0)
{
//Requires as many bits even if we eliminate 10% of the most difficult values
length = 0;
return(false);
}
var baseReducedValues = new long[values.Count];
var i = 0;
foreach (var value in values)
{
baseReducedValues[i++] = value - minValue;
}
var baseReducedValuesHistogram = baseReducedValues.GenerateHistogramOfBitWidths();
var baseReducedHundredthBits =
BitManipulation.GetBitsRequiredForPercentile(baseReducedValuesHistogram, 1.0);
var baseReducedNinetyfifthBits =
BitManipulation.GetBitsRequiredForPercentile(baseReducedValuesHistogram, 0.95);
if (baseReducedHundredthBits - baseReducedNinetyfifthBits == 0)
{
//In the end, no benefit could be realized from patching
length = 0;
return(false);
}
PatchEncode(minValue, baseReducedValues, baseReducedHundredthBits, baseReducedNinetyfifthBits);
length = values.Count;
return(true);
}
