/// <summary>
/// Writes a value that fit in bitsCount number of bits in array at bitIndex position
/// </summary>
public static void SetBits(ulong[] dest, ulong source, int bitIndex, int bitsCount)
{
Debug.Assert(bitsCount > 0 && bitsCount <= ULONG_BITS_COUNT);
Debug.Assert(BitUtils.GetBitBounds(source) <= bitsCount);
dest[bitIndex / ULONG_BITS_COUNT] |= source << (bitIndex % ULONG_BITS_COUNT);
//does the number take more than one element from the array
if (bitIndex % ULONG_BITS_COUNT + bitsCount > ULONG_BITS_COUNT)
{
dest[bitIndex / ULONG_BITS_COUNT + 1] |= source >> (ULONG_BITS_COUNT - bitIndex % ULONG_BITS_COUNT);
}
}
public void Add(long value)
{
if (IsReady)
{
return;
}
if (Count > 1)
{
bool sign;
ulong delta;
if (value >= oldValue)
{
sign = false;
delta = (ulong)(value - oldValue);
}
else
{
sign = true;
delta = (ulong)(oldValue - value);
}
int bits = BitUtils.GetBitBounds(delta);
map[bits]++;
if (bits > maxIndex)
{
maxIndex = bits;
}
if (Type > HelperType.Delta)
{
if (Sign != sign)
{
Type = HelperType.Delta;
}
else if (delta != Delta)
{
Type = HelperType.DeltaMonotone;
}
}
}
else if (Count == 1)
{
SecondAdd(value);
}
oldValue = value;
Count++;
}
private void SecondAdd(long value)
{
if (value >= oldValue)
{
Sign = false;
Delta = (ulong)(value - oldValue);
}
else
{
Sign = true;
Delta = (ulong)(oldValue - value);
}
maxIndex = BitUtils.GetBitBounds(Delta);
map[maxIndex]++;
}
private static void CoreCompress(BinaryWriter writer, IList <long> src, int smallDeltaBits, int bigDeltaBits, int upCount, int downCount, int equalsCount, bool useOneDelta, ulong oneDeltaValue)
{
//store the count of elements
CountCompression.Serialize(writer, checked ((ulong)src.Count));
if (src.Count < 2)
{
for (int i = 0; i < src.Count; i++)
{
writer.Write(src[i]);
}
return;
}
ulong[] tmpData = new ulong[3 + 2 * src.Count];//+3x64 system bits
int bitIndex = 0;
//store if the data is monotone and if is - ascending or descending
ulong monotoneFlag = 0;//00 - not monotone
if (upCount == src.Count - 1 - equalsCount)
{
monotoneFlag = 2;//10 - monotone and ascending
}
else if (downCount == src.Count - 1 - equalsCount)
{
monotoneFlag = 3;//11 - monotone and descending
}
SetBits(tmpData, monotoneFlag, bitIndex, 2);
bitIndex += 2;
//do we have only one delta?
SetBits(tmpData, useOneDelta ? (ulong)1 : (ulong)0, bitIndex, 1);
bitIndex++;
ulong useOneDeltaType = 0;
if (useOneDelta)
{
//stores how much bits is the delta
SetBits(tmpData, (ulong)bigDeltaBits, bitIndex, 7);
bitIndex += 7;
//store the delta
SetBits(tmpData, oneDeltaValue, bitIndex, bigDeltaBits);
bitIndex += bigDeltaBits;
}
else
{
//do we use only one type of delta?
useOneDeltaType = (smallDeltaBits == bigDeltaBits) ? (ulong)1 : (ulong)0;
SetBits(tmpData, useOneDeltaType, bitIndex, 1);
bitIndex++;
//store how many bits is the big delta and how many is the small delta
SetBits(tmpData, (ulong)bigDeltaBits, bitIndex, 7);
bitIndex += 7;
if (useOneDeltaType == 0)
{
SetBits(tmpData, (ulong)smallDeltaBits, bitIndex, 7);
bitIndex += 7;
}
}
//store the first element
writer.Write(src[0]);
if (!useOneDelta)
{
ulong delta, sign;
//the sequental is not monotone
for (int i = 0; i < src.Count - 1; i++)
{
long value1 = src[i];
long value2 = src[i + 1];
if (value2 >= value1)
{
delta = (ulong)(value2 - value1);
sign = 0;
}
else
{
delta = (ulong)(value1 - value2);
sign = 1;
}
//type of delta - big or small
int actualDeltaBits = bigDeltaBits;
if (useOneDeltaType == 0)
{
int deltaBits = BitUtils.GetBitBounds(delta);
ulong deltaType = 1;
if (deltaBits <= smallDeltaBits)
{
deltaType = 0;
actualDeltaBits = smallDeltaBits;
}
SetBits(tmpData, deltaType, bitIndex, 1);
bitIndex++;
}
if (monotoneFlag == 0)
{
//store the sign of delta
SetBits(tmpData, sign, bitIndex, 1);
bitIndex++;
}
//at the last, store the delta
SetBits(tmpData, delta, bitIndex, actualDeltaBits);
bitIndex += actualDeltaBits;
}
}
int dataCount = (int)Math.Ceiling(bitIndex / 64.0);
int bytesCount = sizeof(ulong) * dataCount;
CountCompression.Serialize(writer, checked ((ulong)bytesCount));
for (int i = 0; i < dataCount; i++)
{
writer.Write(tmpData[i]);
}
}
public void AddValue(long value)
{
if (Count > 1)
{
if (value > oldValue)
{
delta = (ulong)(value - oldValue);
UpCount++;
if (UseOneDelta)
{
if (oneDeltaSign != 1 || delta != OneDeltaValue)
{
UseOneDelta = false;
}
}
}
else if (value < oldValue)
{
delta = (ulong)(oldValue - value);
DownCount++;
if (UseOneDelta)
{
if (oneDeltaSign != -1 || delta != OneDeltaValue)
{
UseOneDelta = false;
}
}
}
else
{
delta = 0;
EqualsCount++;
if (UseOneDelta)
{
if (oneDeltaSign != 0 || delta != OneDeltaValue)
{
UseOneDelta = false;
}
}
}
int deltaBit = BitUtils.GetBitBounds(delta);
deltasBits[deltaBit]++;
if (deltaBit > BigDeltaBits)
{
BigDeltaBits = deltaBit;
}
}
else if (Count == 1)
{
if (value > oldValue)
{
OneDeltaValue = delta = (ulong)(value - oldValue);
oneDeltaSign = 1;
UpCount++;
}
else if (value < oldValue)
{
OneDeltaValue = delta = (ulong)(oldValue - value);
oneDeltaSign = -1;
DownCount++;
}
else
{
OneDeltaValue = delta = 0;
oneDeltaSign = 0;
EqualsCount++;
}
BigDeltaBits = BitUtils.GetBitBounds(delta);
deltasBits[BigDeltaBits]++;
}
oldValue = value;
Count++;
}
public static void Compress(BinaryWriter writer, long[] values, int index, int count, Helper helper)
{
writer.Write(VERSION);
if (count == 0)
{
return;
}
long value = values[index];
CountCompression.Serialize(writer, (ulong)value);
if (count == 1)
{
return;
}
Debug.Assert(count == helper.Count);
if (!helper.IsReady)
{
helper.Prepare();
}
index++;
count--;
int maxIndex = index + count - 1;
helper.Serialize(writer);
if (helper.Type == HelperType.Raw)
{
while (index <= maxIndex)
{
writer.Write(values[index++]);
}
return;
}
if (helper.Type == HelperType.OneStep)
{
return;
}
ulong maxDelta = helper.Delta;
bool alwaysUseDelta = helper.AlwaysUseDelta;
int bitCount = BitUtils.GetBitBounds(maxDelta);
bool writeSign = helper.Type == HelperType.Delta;
CommonArray common = new CommonArray();
int sizeBits = helper.SizeBits > 0 ? helper.SizeBits : (1 + 1 + 64) * (count - 1);
common.ByteArray = new byte[(int)Math.Ceiling(sizeBits / 8.0)];
ulong[] data = common.UInt64Array;
int bitIndex = 0;
ulong delta;
bool sign; //false - positive, true - negative
for (; index <= maxIndex; index++)
{
long newValue = values[index];
if (newValue >= value)
{
sign = false;
delta = (ulong)(newValue - value);
}
else
{
sign = true;
delta = (ulong)(value - newValue);
}
if (alwaysUseDelta || delta <= maxDelta)
{
if (!alwaysUseDelta)
{
SetFlag(data, bitIndex /*, true*/); //use delta
bitIndex++;
}
if (writeSign)
{
if (sign)
{
SetFlag(data, bitIndex /*, sign*/);
}
bitIndex++;
}
SetBits(data, bitIndex, delta, bitCount);
bitIndex += bitCount;
}
else
{
//SetFlag(data, bitIndex, false); //don't use delta
bitIndex++;
SetValue(data, bitIndex, (ulong)newValue);
bitIndex += 64;
}
value = newValue;
}
int bytesCount = (int)Math.Ceiling(bitIndex / 8.0);
CountCompression.Serialize(writer, (ulong)bytesCount);
writer.Write(common.ByteArray, 0, bytesCount);
}