private void DirectEncode(IEnumerable <long> values, int numValues, bool aligned, int?precalculatedFixedBitWidth) { int fixedBitWidth; if (precalculatedFixedBitWidth.HasValue) { fixedBitWidth = precalculatedFixedBitWidth.Value; } else { var histogram = values.GenerateHistogramOfBitWidths(); fixedBitWidth = BitManipulation.GetBitsRequiredForPercentile(histogram, 1.0); } fixedBitWidth = aligned ? BitManipulation.FindNearestAlignedDirectWidth(fixedBitWidth) : BitManipulation.FindNearestDirectWidth(fixedBitWidth); var encodedFixedBitWidth = fixedBitWidth.EncodeDirectWidth(); var byte1 = 0; byte1 |= 0x1 << 6; //7..6 Encoding Type byte1 |= (encodedFixedBitWidth & 0x1f) << 1; //5..1 Fixed Width byte1 |= (numValues - 1) >> 8; //0 MSB of length var byte2 = (numValues - 1) & 0xff; //7..0 LSBs of length _outputStream.WriteByte((byte)byte1); _outputStream.WriteByte((byte)byte2); _outputStream.WriteBitpackedIntegers(values, fixedBitWidth); }
private bool TryPatchEncoding(IEnumerable <long> zigZagValues, IList <long> values, long minValue, ref int?fixedBitWidth, out int length) { var zigZagValuesHistogram = zigZagValues.GenerateHistogramOfBitWidths(); var zigZagHundredthBits = BitManipulation.GetBitsRequiredForPercentile(zigZagValuesHistogram, 1.0); fixedBitWidth = zigZagHundredthBits; //We'll use this later if if end up DIRECT encoding var zigZagNinetiethBits = BitManipulation.GetBitsRequiredForPercentile(zigZagValuesHistogram, 0.9); if (zigZagHundredthBits - zigZagNinetiethBits == 0) { //Requires as many bits even if we eliminate 10% of the most difficult values length = 0; return(false); } var baseReducedValues = new long[values.Count]; var i = 0; foreach (var value in values) { baseReducedValues[i++] = value - minValue; } var baseReducedValuesHistogram = baseReducedValues.GenerateHistogramOfBitWidths(); var baseReducedHundredthBits = BitManipulation.GetBitsRequiredForPercentile(baseReducedValuesHistogram, 1.0); var baseReducedNinetyfifthBits = BitManipulation.GetBitsRequiredForPercentile(baseReducedValuesHistogram, 0.95); if (baseReducedHundredthBits - baseReducedNinetyfifthBits == 0) { //In the end, no benefit could be realized from patching length = 0; return(false); } PatchEncode(minValue, baseReducedValues, baseReducedHundredthBits, baseReducedNinetyfifthBits); length = values.Count; return(true); }