public static Helper Deserialize(BinaryReader reader)
{
Helper helper = new Helper();
byte b = reader.ReadByte();
helper.Sign = (b & 0x80) != 0;
helper.IsReady = (b & 0x40) != 0;
helper.AlwaysUseDelta = (b & 0x20) != 0;
helper.Type = (HelperType)(b & 0x1f);
if (helper.Type == HelperType.Raw)
{
return(helper);
}
if (helper.Type == HelperType.OneStep)
{
helper.Delta = CountCompression.Deserialize(reader);
return(helper);
}
helper.DeltaBits = reader.ReadByte();
helper.Delta = (1UL << helper.DeltaBits) - 1;
return(helper);
}
public void Serialize(BinaryWriter writer)
{
byte b = 0;
if (Sign)
{
b |= (byte)0x80;
}
if (IsReady)
{
b |= (byte)0x40;
}
if (AlwaysUseDelta)
{
b |= (byte)0x20;
}
b |= (byte)Type;
writer.Write(b);
if (Type == HelperType.Raw)
{
return;
}
if (Type == HelperType.OneStep)
{
CountCompression.Serialize(writer, Delta);
return;
}
writer.Write(DeltaBits);
}
private static void CoreCompress(BinaryWriter writer, IList <long> src, int smallDeltaBits, int bigDeltaBits, int upCount, int downCount, int equalsCount, bool useOneDelta, ulong oneDeltaValue)
{
//store the count of elements
CountCompression.Serialize(writer, checked ((ulong)src.Count));
if (src.Count < 2)
{
for (int i = 0; i < src.Count; i++)
{
writer.Write(src[i]);
}
return;
}
ulong[] tmpData = new ulong[3 + 2 * src.Count];//+3x64 system bits
int bitIndex = 0;
//store if the data is monotone and if is - ascending or descending
ulong monotoneFlag = 0;//00 - not monotone
if (upCount == src.Count - 1 - equalsCount)
{
monotoneFlag = 2;//10 - monotone and ascending
}
else if (downCount == src.Count - 1 - equalsCount)
{
monotoneFlag = 3;//11 - monotone and descending
}
SetBits(tmpData, monotoneFlag, bitIndex, 2);
bitIndex += 2;
//do we have only one delta?
SetBits(tmpData, useOneDelta ? (ulong)1 : (ulong)0, bitIndex, 1);
bitIndex++;
ulong useOneDeltaType = 0;
if (useOneDelta)
{
//stores how much bits is the delta
SetBits(tmpData, (ulong)bigDeltaBits, bitIndex, 7);
bitIndex += 7;
//store the delta
SetBits(tmpData, oneDeltaValue, bitIndex, bigDeltaBits);
bitIndex += bigDeltaBits;
}
else
{
//do we use only one type of delta?
useOneDeltaType = (smallDeltaBits == bigDeltaBits) ? (ulong)1 : (ulong)0;
SetBits(tmpData, useOneDeltaType, bitIndex, 1);
bitIndex++;
//store how many bits is the big delta and how many is the small delta
SetBits(tmpData, (ulong)bigDeltaBits, bitIndex, 7);
bitIndex += 7;
if (useOneDeltaType == 0)
{
SetBits(tmpData, (ulong)smallDeltaBits, bitIndex, 7);
bitIndex += 7;
}
}
//store the first element
writer.Write(src[0]);
if (!useOneDelta)
{
ulong delta, sign;
//the sequental is not monotone
for (int i = 0; i < src.Count - 1; i++)
{
long value1 = src[i];
long value2 = src[i + 1];
if (value2 >= value1)
{
delta = (ulong)(value2 - value1);
sign = 0;
}
else
{
delta = (ulong)(value1 - value2);
sign = 1;
}
//type of delta - big or small
int actualDeltaBits = bigDeltaBits;
if (useOneDeltaType == 0)
{
int deltaBits = BitUtils.GetBitBounds(delta);
ulong deltaType = 1;
if (deltaBits <= smallDeltaBits)
{
deltaType = 0;
actualDeltaBits = smallDeltaBits;
}
SetBits(tmpData, deltaType, bitIndex, 1);
bitIndex++;
}
if (monotoneFlag == 0)
{
//store the sign of delta
SetBits(tmpData, sign, bitIndex, 1);
bitIndex++;
}
//at the last, store the delta
SetBits(tmpData, delta, bitIndex, actualDeltaBits);
bitIndex += actualDeltaBits;
}
}
int dataCount = (int)Math.Ceiling(bitIndex / 64.0);
int bytesCount = sizeof(ulong) * dataCount;
CountCompression.Serialize(writer, checked ((ulong)bytesCount));
for (int i = 0; i < dataCount; i++)
{
writer.Write(tmpData[i]);
}
}
public static IList <long> CoreDecompress(BinaryReader reader)
{
int resLength = (int)CountCompression.Deserialize(reader);
IList <long> res = new List <long>(resLength);
if (resLength < 2)
{
if (resLength == 1)
{
res.Add(reader.ReadInt64());
}
return(res);
}
//read the first element
res.Add(reader.ReadInt64());
int bytesCount = (int)CountCompression.Deserialize(reader);
ulong[] tmpData = new ulong[(int)Math.Ceiling(bytesCount / 8.0)];
for (int j = 0; j < tmpData.Length; j++)
{
tmpData[j] = reader.ReadUInt64();
}
int bitIndex = 0, i;
//is data monote?
ulong monotoneFlag = GetBits(tmpData, bitIndex, 2);
bitIndex += 2;
//have we only one delta?
bool useOneDelta = GetBits(tmpData, bitIndex, 1) == 1;
bitIndex++;
int smallDeltaBits = 0, bigDeltaBits = 0;
long delta = 0;
ulong useOneDeltaType = 0, sign;
if (useOneDelta)
{
//how many bits is the delta, what is it value?
bigDeltaBits = (int)GetBits(tmpData, bitIndex, 7);
bitIndex += 7;
delta = (long)GetBits(tmpData, bitIndex, bigDeltaBits);
bitIndex += bigDeltaBits;
}
else
{
//how many types of delta we use - one or two. What are they?
useOneDeltaType = GetBits(tmpData, bitIndex, 1);
bitIndex++;
bigDeltaBits = (int)GetBits(tmpData, bitIndex, 7);
bitIndex += 7;
if (useOneDeltaType == 0)
{
smallDeltaBits = (int)GetBits(tmpData, bitIndex, 7);
bitIndex += 7;
}
}
if (useOneDelta)
{
sign = monotoneFlag & 1;
if (sign == 0)
{
for (i = 1; i < resLength; i++)
{
res.Add(res[i - 1] + delta);
}
}
else
{
for (i = 1; i < resLength; i++)
{
res.Add(res[i - 1] - delta);
}
}
}
else
{
for (i = 1; i < resLength; i++)
{
int actualDeltaBits = bigDeltaBits;
if (useOneDeltaType == 0)
{
ulong deltaType = GetBits(tmpData, bitIndex, 1);
bitIndex++;
if (deltaType == 0)
{
actualDeltaBits = smallDeltaBits;
}
}
if (monotoneFlag == 0)
{
sign = GetBits(tmpData, bitIndex, 1);
bitIndex++;
}
else
{
sign = monotoneFlag & 1;
}
delta = (long)GetBits(tmpData, bitIndex, actualDeltaBits);
bitIndex += actualDeltaBits;
res.Add(res[i - 1] + ((sign == 0) ? delta : -delta));
}
}
return(res);
}
public static void Compress(BinaryWriter writer, long[] values, int index, int count, Helper helper)
{
writer.Write(VERSION);
if (count == 0)
{
return;
}
long value = values[index];
CountCompression.Serialize(writer, (ulong)value);
if (count == 1)
{
return;
}
Debug.Assert(count == helper.Count);
if (!helper.IsReady)
{
helper.Prepare();
}
index++;
count--;
int maxIndex = index + count - 1;
helper.Serialize(writer);
if (helper.Type == HelperType.Raw)
{
while (index <= maxIndex)
{
writer.Write(values[index++]);
}
return;
}
if (helper.Type == HelperType.OneStep)
{
return;
}
ulong maxDelta = helper.Delta;
bool alwaysUseDelta = helper.AlwaysUseDelta;
int bitCount = BitUtils.GetBitBounds(maxDelta);
bool writeSign = helper.Type == HelperType.Delta;
CommonArray common = new CommonArray();
int sizeBits = helper.SizeBits > 0 ? helper.SizeBits : (1 + 1 + 64) * (count - 1);
common.ByteArray = new byte[(int)Math.Ceiling(sizeBits / 8.0)];
ulong[] data = common.UInt64Array;
int bitIndex = 0;
ulong delta;
bool sign; //false - positive, true - negative
for (; index <= maxIndex; index++)
{
long newValue = values[index];
if (newValue >= value)
{
sign = false;
delta = (ulong)(newValue - value);
}
else
{
sign = true;
delta = (ulong)(value - newValue);
}
if (alwaysUseDelta || delta <= maxDelta)
{
if (!alwaysUseDelta)
{
SetFlag(data, bitIndex /*, true*/); //use delta
bitIndex++;
}
if (writeSign)
{
if (sign)
{
SetFlag(data, bitIndex /*, sign*/);
}
bitIndex++;
}
SetBits(data, bitIndex, delta, bitCount);
bitIndex += bitCount;
}
else
{
//SetFlag(data, bitIndex, false); //don't use delta
bitIndex++;
SetValue(data, bitIndex, (ulong)newValue);
bitIndex += 64;
}
value = newValue;
}
int bytesCount = (int)Math.Ceiling(bitIndex / 8.0);
CountCompression.Serialize(writer, (ulong)bytesCount);
writer.Write(common.ByteArray, 0, bytesCount);
}
public static void Decompress(BinaryReader reader, Action <int, long> values, int count)
{
if (reader.ReadByte() != VERSION)
{
throw new Exception("Invalid delta compression version.");
}
if (count == 0)
{
return;
}
int index = 0;
long value = (long)CountCompression.Deserialize(reader);
values(index, value);
if (count == 1)
{
return;
}
var helper = Helper.Deserialize(reader);
index++;
count--;
int maxIndex = index + count - 1;
if (helper.Type == HelperType.Raw)
{
while (index <= maxIndex)
{
values(index++, reader.ReadInt64());
}
return;
}
if (helper.Type == HelperType.OneStep)
{
long step = (long)helper.Delta;
if (helper.Sign == false)
{
while (index <= maxIndex)
{
values(index++, (value += step));
}
}
else
{
while (index <= maxIndex)
{
values(index++, (value -= step));
}
}
return;
}
bool readSign = helper.Type == HelperType.Delta;
ulong maxDelta = helper.Delta;
bool alwaysUseDelta = helper.AlwaysUseDelta;
bool sign = helper.Sign;
int bitCount = helper.DeltaBits;
CommonArray common = new CommonArray();
int bytesCount = (int)CountCompression.Deserialize(reader);
common.ByteArray = reader.ReadBytes(bytesCount);
ulong[] data = common.UInt64Array;
int bitIndex = 0;
for (; index <= maxIndex; index++)
{
long newValue;
bool useDelta = alwaysUseDelta || GetFlag(data, bitIndex++);
if (useDelta)
{
if (readSign)
{
sign = GetFlag(data, bitIndex);
bitIndex++;
}
long delta = (long)GetBits(data, bitIndex, bitCount);
bitIndex += bitCount;
newValue = sign ? value - delta : value + delta;
}
else
{
newValue = (long)GetValue(data, bitIndex);
bitIndex += 64;
}
values(index, newValue);
value = newValue;
}
}
