예제 #1
0
        /// <summary>
        /// Writes a value that fit in bitsCount number of bits in array at bitIndex position
        /// </summary>
        public static void SetBits(ulong[] dest, ulong source, int bitIndex, int bitsCount)
        {
            Debug.Assert(bitsCount > 0 && bitsCount <= ULONG_BITS_COUNT);
            Debug.Assert(BitUtils.GetBitBounds(source) <= bitsCount);

            dest[bitIndex / ULONG_BITS_COUNT] |= source << (bitIndex % ULONG_BITS_COUNT);
            //does the number take more than one element from the array
            if (bitIndex % ULONG_BITS_COUNT + bitsCount > ULONG_BITS_COUNT)
            {
                dest[bitIndex / ULONG_BITS_COUNT + 1] |= source >> (ULONG_BITS_COUNT - bitIndex % ULONG_BITS_COUNT);
            }
        }
예제 #2
0
            public void Add(long value)
            {
                if (IsReady)
                {
                    return;
                }

                if (Count > 1)
                {
                    bool  sign;
                    ulong delta;

                    if (value >= oldValue)
                    {
                        sign  = false;
                        delta = (ulong)(value - oldValue);
                    }
                    else
                    {
                        sign  = true;
                        delta = (ulong)(oldValue - value);
                    }

                    int bits = BitUtils.GetBitBounds(delta);
                    map[bits]++;
                    if (bits > maxIndex)
                    {
                        maxIndex = bits;
                    }

                    if (Type > HelperType.Delta)
                    {
                        if (Sign != sign)
                        {
                            Type = HelperType.Delta;
                        }
                        else if (delta != Delta)
                        {
                            Type = HelperType.DeltaMonotone;
                        }
                    }
                }
                else if (Count == 1)
                {
                    SecondAdd(value);
                }

                oldValue = value;
                Count++;
            }
예제 #3
0
            private void SecondAdd(long value)
            {
                if (value >= oldValue)
                {
                    Sign  = false;
                    Delta = (ulong)(value - oldValue);
                }
                else
                {
                    Sign  = true;
                    Delta = (ulong)(oldValue - value);
                }

                maxIndex = BitUtils.GetBitBounds(Delta);
                map[maxIndex]++;
            }
예제 #4
0
        private static void CoreCompress(BinaryWriter writer, IList <long> src, int smallDeltaBits, int bigDeltaBits, int upCount, int downCount, int equalsCount, bool useOneDelta, ulong oneDeltaValue)
        {
            //store the count of elements
            CountCompression.Serialize(writer, checked ((ulong)src.Count));

            if (src.Count < 2)
            {
                for (int i = 0; i < src.Count; i++)
                {
                    writer.Write(src[i]);
                }
                return;
            }

            ulong[] tmpData  = new ulong[3 + 2 * src.Count];//+3x64 system bits
            int     bitIndex = 0;

            //store if the data is monotone and if is - ascending or descending
            ulong monotoneFlag = 0;//00 - not monotone

            if (upCount == src.Count - 1 - equalsCount)
            {
                monotoneFlag = 2;//10 - monotone and ascending
            }
            else if (downCount == src.Count - 1 - equalsCount)
            {
                monotoneFlag = 3;//11 - monotone and descending
            }
            SetBits(tmpData, monotoneFlag, bitIndex, 2);
            bitIndex += 2;

            //do we have only one delta?
            SetBits(tmpData, useOneDelta ? (ulong)1 : (ulong)0, bitIndex, 1);
            bitIndex++;

            ulong useOneDeltaType = 0;

            if (useOneDelta)
            {
                //stores how much bits is the delta
                SetBits(tmpData, (ulong)bigDeltaBits, bitIndex, 7);
                bitIndex += 7;

                //store the delta
                SetBits(tmpData, oneDeltaValue, bitIndex, bigDeltaBits);
                bitIndex += bigDeltaBits;
            }
            else
            {
                //do we use only one type of delta?
                useOneDeltaType = (smallDeltaBits == bigDeltaBits) ? (ulong)1 : (ulong)0;
                SetBits(tmpData, useOneDeltaType, bitIndex, 1);
                bitIndex++;

                //store how many bits is the big delta and how many is the small delta
                SetBits(tmpData, (ulong)bigDeltaBits, bitIndex, 7);
                bitIndex += 7;
                if (useOneDeltaType == 0)
                {
                    SetBits(tmpData, (ulong)smallDeltaBits, bitIndex, 7);
                    bitIndex += 7;
                }
            }

            //store the first element
            writer.Write(src[0]);

            if (!useOneDelta)
            {
                ulong delta, sign;

                //the sequental is not monotone
                for (int i = 0; i < src.Count - 1; i++)
                {
                    long value1 = src[i];
                    long value2 = src[i + 1];

                    if (value2 >= value1)
                    {
                        delta = (ulong)(value2 - value1);
                        sign  = 0;
                    }
                    else
                    {
                        delta = (ulong)(value1 - value2);
                        sign  = 1;
                    }

                    //type of delta - big or small
                    int actualDeltaBits = bigDeltaBits;
                    if (useOneDeltaType == 0)
                    {
                        int   deltaBits = BitUtils.GetBitBounds(delta);
                        ulong deltaType = 1;
                        if (deltaBits <= smallDeltaBits)
                        {
                            deltaType       = 0;
                            actualDeltaBits = smallDeltaBits;
                        }

                        SetBits(tmpData, deltaType, bitIndex, 1);
                        bitIndex++;
                    }

                    if (monotoneFlag == 0)
                    {
                        //store the sign of delta
                        SetBits(tmpData, sign, bitIndex, 1);
                        bitIndex++;
                    }

                    //at the last, store the delta
                    SetBits(tmpData, delta, bitIndex, actualDeltaBits);
                    bitIndex += actualDeltaBits;
                }
            }

            int dataCount  = (int)Math.Ceiling(bitIndex / 64.0);
            int bytesCount = sizeof(ulong) * dataCount;

            CountCompression.Serialize(writer, checked ((ulong)bytesCount));
            for (int i = 0; i < dataCount; i++)
            {
                writer.Write(tmpData[i]);
            }
        }
예제 #5
0
            public void AddValue(long value)
            {
                if (Count > 1)
                {
                    if (value > oldValue)
                    {
                        delta = (ulong)(value - oldValue);
                        UpCount++;
                        if (UseOneDelta)
                        {
                            if (oneDeltaSign != 1 || delta != OneDeltaValue)
                            {
                                UseOneDelta = false;
                            }
                        }
                    }
                    else if (value < oldValue)
                    {
                        delta = (ulong)(oldValue - value);
                        DownCount++;
                        if (UseOneDelta)
                        {
                            if (oneDeltaSign != -1 || delta != OneDeltaValue)
                            {
                                UseOneDelta = false;
                            }
                        }
                    }
                    else
                    {
                        delta = 0;
                        EqualsCount++;
                        if (UseOneDelta)
                        {
                            if (oneDeltaSign != 0 || delta != OneDeltaValue)
                            {
                                UseOneDelta = false;
                            }
                        }
                    }

                    int deltaBit = BitUtils.GetBitBounds(delta);
                    deltasBits[deltaBit]++;

                    if (deltaBit > BigDeltaBits)
                    {
                        BigDeltaBits = deltaBit;
                    }
                }
                else if (Count == 1)
                {
                    if (value > oldValue)
                    {
                        OneDeltaValue = delta = (ulong)(value - oldValue);
                        oneDeltaSign  = 1;
                        UpCount++;
                    }
                    else if (value < oldValue)
                    {
                        OneDeltaValue = delta = (ulong)(oldValue - value);
                        oneDeltaSign  = -1;
                        DownCount++;
                    }
                    else
                    {
                        OneDeltaValue = delta = 0;
                        oneDeltaSign  = 0;
                        EqualsCount++;
                    }

                    BigDeltaBits = BitUtils.GetBitBounds(delta);
                    deltasBits[BigDeltaBits]++;
                }

                oldValue = value;
                Count++;
            }
예제 #6
0
        public static void Compress(BinaryWriter writer, long[] values, int index, int count, Helper helper)
        {
            writer.Write(VERSION);

            if (count == 0)
            {
                return;
            }

            long value = values[index];

            CountCompression.Serialize(writer, (ulong)value);
            if (count == 1)
            {
                return;
            }

            Debug.Assert(count == helper.Count);
            if (!helper.IsReady)
            {
                helper.Prepare();
            }

            index++;
            count--;
            int maxIndex = index + count - 1;

            helper.Serialize(writer);

            if (helper.Type == HelperType.Raw)
            {
                while (index <= maxIndex)
                {
                    writer.Write(values[index++]);
                }

                return;
            }

            if (helper.Type == HelperType.OneStep)
            {
                return;
            }

            ulong maxDelta       = helper.Delta;
            bool  alwaysUseDelta = helper.AlwaysUseDelta;
            int   bitCount       = BitUtils.GetBitBounds(maxDelta);

            bool writeSign = helper.Type == HelperType.Delta;

            CommonArray common   = new CommonArray();
            int         sizeBits = helper.SizeBits > 0 ? helper.SizeBits : (1 + 1 + 64) * (count - 1);

            common.ByteArray = new byte[(int)Math.Ceiling(sizeBits / 8.0)];
            ulong[] data     = common.UInt64Array;
            int     bitIndex = 0;

            ulong delta;
            bool  sign; //false - positive, true - negative

            for (; index <= maxIndex; index++)
            {
                long newValue = values[index];
                if (newValue >= value)
                {
                    sign  = false;
                    delta = (ulong)(newValue - value);
                }
                else
                {
                    sign  = true;
                    delta = (ulong)(value - newValue);
                }

                if (alwaysUseDelta || delta <= maxDelta)
                {
                    if (!alwaysUseDelta)
                    {
                        SetFlag(data, bitIndex /*, true*/); //use delta
                        bitIndex++;
                    }

                    if (writeSign)
                    {
                        if (sign)
                        {
                            SetFlag(data, bitIndex /*, sign*/);
                        }
                        bitIndex++;
                    }

                    SetBits(data, bitIndex, delta, bitCount);
                    bitIndex += bitCount;
                }
                else
                {
                    //SetFlag(data, bitIndex, false); //don't use delta
                    bitIndex++;

                    SetValue(data, bitIndex, (ulong)newValue);
                    bitIndex += 64;
                }

                value = newValue;
            }

            int bytesCount = (int)Math.Ceiling(bitIndex / 8.0);

            CountCompression.Serialize(writer, (ulong)bytesCount);
            writer.Write(common.ByteArray, 0, bytesCount);
        }