/// <summary> /// Writes a value that fit in bitsCount number of bits in array at bitIndex position /// </summary> public static void SetBits(ulong[] dest, ulong source, int bitIndex, int bitsCount) { Debug.Assert(bitsCount > 0 && bitsCount <= ULONG_BITS_COUNT); Debug.Assert(BitUtils.GetBitBounds(source) <= bitsCount); dest[bitIndex / ULONG_BITS_COUNT] |= source << (bitIndex % ULONG_BITS_COUNT); //does the number take more than one element from the array if (bitIndex % ULONG_BITS_COUNT + bitsCount > ULONG_BITS_COUNT) { dest[bitIndex / ULONG_BITS_COUNT + 1] |= source >> (ULONG_BITS_COUNT - bitIndex % ULONG_BITS_COUNT); } }
public void Add(long value) { if (IsReady) { return; } if (Count > 1) { bool sign; ulong delta; if (value >= oldValue) { sign = false; delta = (ulong)(value - oldValue); } else { sign = true; delta = (ulong)(oldValue - value); } int bits = BitUtils.GetBitBounds(delta); map[bits]++; if (bits > maxIndex) { maxIndex = bits; } if (Type > HelperType.Delta) { if (Sign != sign) { Type = HelperType.Delta; } else if (delta != Delta) { Type = HelperType.DeltaMonotone; } } } else if (Count == 1) { SecondAdd(value); } oldValue = value; Count++; }
private void SecondAdd(long value) { if (value >= oldValue) { Sign = false; Delta = (ulong)(value - oldValue); } else { Sign = true; Delta = (ulong)(oldValue - value); } maxIndex = BitUtils.GetBitBounds(Delta); map[maxIndex]++; }
private static void CoreCompress(BinaryWriter writer, IList <long> src, int smallDeltaBits, int bigDeltaBits, int upCount, int downCount, int equalsCount, bool useOneDelta, ulong oneDeltaValue) { //store the count of elements CountCompression.Serialize(writer, checked ((ulong)src.Count)); if (src.Count < 2) { for (int i = 0; i < src.Count; i++) { writer.Write(src[i]); } return; } ulong[] tmpData = new ulong[3 + 2 * src.Count];//+3x64 system bits int bitIndex = 0; //store if the data is monotone and if is - ascending or descending ulong monotoneFlag = 0;//00 - not monotone if (upCount == src.Count - 1 - equalsCount) { monotoneFlag = 2;//10 - monotone and ascending } else if (downCount == src.Count - 1 - equalsCount) { monotoneFlag = 3;//11 - monotone and descending } SetBits(tmpData, monotoneFlag, bitIndex, 2); bitIndex += 2; //do we have only one delta? SetBits(tmpData, useOneDelta ? (ulong)1 : (ulong)0, bitIndex, 1); bitIndex++; ulong useOneDeltaType = 0; if (useOneDelta) { //stores how much bits is the delta SetBits(tmpData, (ulong)bigDeltaBits, bitIndex, 7); bitIndex += 7; //store the delta SetBits(tmpData, oneDeltaValue, bitIndex, bigDeltaBits); bitIndex += bigDeltaBits; } else { //do we use only one type of delta? useOneDeltaType = (smallDeltaBits == bigDeltaBits) ? (ulong)1 : (ulong)0; SetBits(tmpData, useOneDeltaType, bitIndex, 1); bitIndex++; //store how many bits is the big delta and how many is the small delta SetBits(tmpData, (ulong)bigDeltaBits, bitIndex, 7); bitIndex += 7; if (useOneDeltaType == 0) { SetBits(tmpData, (ulong)smallDeltaBits, bitIndex, 7); bitIndex += 7; } } //store the first element writer.Write(src[0]); if (!useOneDelta) { ulong delta, sign; //the sequental is not monotone for (int i = 0; i < src.Count - 1; i++) { long value1 = src[i]; long value2 = src[i + 1]; if (value2 >= value1) { delta = (ulong)(value2 - value1); sign = 0; } else { delta = (ulong)(value1 - value2); sign = 1; } //type of delta - big or small int actualDeltaBits = bigDeltaBits; if (useOneDeltaType == 0) { int deltaBits = BitUtils.GetBitBounds(delta); ulong deltaType = 1; if (deltaBits <= smallDeltaBits) { deltaType = 0; actualDeltaBits = smallDeltaBits; } SetBits(tmpData, deltaType, bitIndex, 1); bitIndex++; } if (monotoneFlag == 0) { //store the sign of delta SetBits(tmpData, sign, bitIndex, 1); bitIndex++; } //at the last, store the delta SetBits(tmpData, delta, bitIndex, actualDeltaBits); bitIndex += actualDeltaBits; } } int dataCount = (int)Math.Ceiling(bitIndex / 64.0); int bytesCount = sizeof(ulong) * dataCount; CountCompression.Serialize(writer, checked ((ulong)bytesCount)); for (int i = 0; i < dataCount; i++) { writer.Write(tmpData[i]); } }
public void AddValue(long value) { if (Count > 1) { if (value > oldValue) { delta = (ulong)(value - oldValue); UpCount++; if (UseOneDelta) { if (oneDeltaSign != 1 || delta != OneDeltaValue) { UseOneDelta = false; } } } else if (value < oldValue) { delta = (ulong)(oldValue - value); DownCount++; if (UseOneDelta) { if (oneDeltaSign != -1 || delta != OneDeltaValue) { UseOneDelta = false; } } } else { delta = 0; EqualsCount++; if (UseOneDelta) { if (oneDeltaSign != 0 || delta != OneDeltaValue) { UseOneDelta = false; } } } int deltaBit = BitUtils.GetBitBounds(delta); deltasBits[deltaBit]++; if (deltaBit > BigDeltaBits) { BigDeltaBits = deltaBit; } } else if (Count == 1) { if (value > oldValue) { OneDeltaValue = delta = (ulong)(value - oldValue); oneDeltaSign = 1; UpCount++; } else if (value < oldValue) { OneDeltaValue = delta = (ulong)(oldValue - value); oneDeltaSign = -1; DownCount++; } else { OneDeltaValue = delta = 0; oneDeltaSign = 0; EqualsCount++; } BigDeltaBits = BitUtils.GetBitBounds(delta); deltasBits[BigDeltaBits]++; } oldValue = value; Count++; }
public static void Compress(BinaryWriter writer, long[] values, int index, int count, Helper helper) { writer.Write(VERSION); if (count == 0) { return; } long value = values[index]; CountCompression.Serialize(writer, (ulong)value); if (count == 1) { return; } Debug.Assert(count == helper.Count); if (!helper.IsReady) { helper.Prepare(); } index++; count--; int maxIndex = index + count - 1; helper.Serialize(writer); if (helper.Type == HelperType.Raw) { while (index <= maxIndex) { writer.Write(values[index++]); } return; } if (helper.Type == HelperType.OneStep) { return; } ulong maxDelta = helper.Delta; bool alwaysUseDelta = helper.AlwaysUseDelta; int bitCount = BitUtils.GetBitBounds(maxDelta); bool writeSign = helper.Type == HelperType.Delta; CommonArray common = new CommonArray(); int sizeBits = helper.SizeBits > 0 ? helper.SizeBits : (1 + 1 + 64) * (count - 1); common.ByteArray = new byte[(int)Math.Ceiling(sizeBits / 8.0)]; ulong[] data = common.UInt64Array; int bitIndex = 0; ulong delta; bool sign; //false - positive, true - negative for (; index <= maxIndex; index++) { long newValue = values[index]; if (newValue >= value) { sign = false; delta = (ulong)(newValue - value); } else { sign = true; delta = (ulong)(value - newValue); } if (alwaysUseDelta || delta <= maxDelta) { if (!alwaysUseDelta) { SetFlag(data, bitIndex /*, true*/); //use delta bitIndex++; } if (writeSign) { if (sign) { SetFlag(data, bitIndex /*, sign*/); } bitIndex++; } SetBits(data, bitIndex, delta, bitCount); bitIndex += bitCount; } else { //SetFlag(data, bitIndex, false); //don't use delta bitIndex++; SetValue(data, bitIndex, (ulong)newValue); bitIndex += 64; } value = newValue; } int bytesCount = (int)Math.Ceiling(bitIndex / 8.0); CountCompression.Serialize(writer, (ulong)bytesCount); writer.Write(common.ByteArray, 0, bytesCount); }