public static Helper Deserialize(BinaryReader reader) { Helper helper = new Helper(); byte b = reader.ReadByte(); helper.Sign = (b & 0x80) != 0; helper.IsReady = (b & 0x40) != 0; helper.AlwaysUseDelta = (b & 0x20) != 0; helper.Type = (HelperType)(b & 0x1f); if (helper.Type == HelperType.Raw) { return(helper); } if (helper.Type == HelperType.OneStep) { helper.Delta = CountCompression.Deserialize(reader); return(helper); } helper.DeltaBits = reader.ReadByte(); helper.Delta = (1UL << helper.DeltaBits) - 1; return(helper); }
public void Serialize(BinaryWriter writer) { byte b = 0; if (Sign) { b |= (byte)0x80; } if (IsReady) { b |= (byte)0x40; } if (AlwaysUseDelta) { b |= (byte)0x20; } b |= (byte)Type; writer.Write(b); if (Type == HelperType.Raw) { return; } if (Type == HelperType.OneStep) { CountCompression.Serialize(writer, Delta); return; } writer.Write(DeltaBits); }
private static void CoreCompress(BinaryWriter writer, IList <long> src, int smallDeltaBits, int bigDeltaBits, int upCount, int downCount, int equalsCount, bool useOneDelta, ulong oneDeltaValue) { //store the count of elements CountCompression.Serialize(writer, checked ((ulong)src.Count)); if (src.Count < 2) { for (int i = 0; i < src.Count; i++) { writer.Write(src[i]); } return; } ulong[] tmpData = new ulong[3 + 2 * src.Count];//+3x64 system bits int bitIndex = 0; //store if the data is monotone and if is - ascending or descending ulong monotoneFlag = 0;//00 - not monotone if (upCount == src.Count - 1 - equalsCount) { monotoneFlag = 2;//10 - monotone and ascending } else if (downCount == src.Count - 1 - equalsCount) { monotoneFlag = 3;//11 - monotone and descending } SetBits(tmpData, monotoneFlag, bitIndex, 2); bitIndex += 2; //do we have only one delta? SetBits(tmpData, useOneDelta ? (ulong)1 : (ulong)0, bitIndex, 1); bitIndex++; ulong useOneDeltaType = 0; if (useOneDelta) { //stores how much bits is the delta SetBits(tmpData, (ulong)bigDeltaBits, bitIndex, 7); bitIndex += 7; //store the delta SetBits(tmpData, oneDeltaValue, bitIndex, bigDeltaBits); bitIndex += bigDeltaBits; } else { //do we use only one type of delta? useOneDeltaType = (smallDeltaBits == bigDeltaBits) ? (ulong)1 : (ulong)0; SetBits(tmpData, useOneDeltaType, bitIndex, 1); bitIndex++; //store how many bits is the big delta and how many is the small delta SetBits(tmpData, (ulong)bigDeltaBits, bitIndex, 7); bitIndex += 7; if (useOneDeltaType == 0) { SetBits(tmpData, (ulong)smallDeltaBits, bitIndex, 7); bitIndex += 7; } } //store the first element writer.Write(src[0]); if (!useOneDelta) { ulong delta, sign; //the sequental is not monotone for (int i = 0; i < src.Count - 1; i++) { long value1 = src[i]; long value2 = src[i + 1]; if (value2 >= value1) { delta = (ulong)(value2 - value1); sign = 0; } else { delta = (ulong)(value1 - value2); sign = 1; } //type of delta - big or small int actualDeltaBits = bigDeltaBits; if (useOneDeltaType == 0) { int deltaBits = BitUtils.GetBitBounds(delta); ulong deltaType = 1; if (deltaBits <= smallDeltaBits) { deltaType = 0; actualDeltaBits = smallDeltaBits; } SetBits(tmpData, deltaType, bitIndex, 1); bitIndex++; } if (monotoneFlag == 0) { //store the sign of delta SetBits(tmpData, sign, bitIndex, 1); bitIndex++; } //at the last, store the delta SetBits(tmpData, delta, bitIndex, actualDeltaBits); bitIndex += actualDeltaBits; } } int dataCount = (int)Math.Ceiling(bitIndex / 64.0); int bytesCount = sizeof(ulong) * dataCount; CountCompression.Serialize(writer, checked ((ulong)bytesCount)); for (int i = 0; i < dataCount; i++) { writer.Write(tmpData[i]); } }
public static IList <long> CoreDecompress(BinaryReader reader) { int resLength = (int)CountCompression.Deserialize(reader); IList <long> res = new List <long>(resLength); if (resLength < 2) { if (resLength == 1) { res.Add(reader.ReadInt64()); } return(res); } //read the first element res.Add(reader.ReadInt64()); int bytesCount = (int)CountCompression.Deserialize(reader); ulong[] tmpData = new ulong[(int)Math.Ceiling(bytesCount / 8.0)]; for (int j = 0; j < tmpData.Length; j++) { tmpData[j] = reader.ReadUInt64(); } int bitIndex = 0, i; //is data monote? ulong monotoneFlag = GetBits(tmpData, bitIndex, 2); bitIndex += 2; //have we only one delta? bool useOneDelta = GetBits(tmpData, bitIndex, 1) == 1; bitIndex++; int smallDeltaBits = 0, bigDeltaBits = 0; long delta = 0; ulong useOneDeltaType = 0, sign; if (useOneDelta) { //how many bits is the delta, what is it value? bigDeltaBits = (int)GetBits(tmpData, bitIndex, 7); bitIndex += 7; delta = (long)GetBits(tmpData, bitIndex, bigDeltaBits); bitIndex += bigDeltaBits; } else { //how many types of delta we use - one or two. What are they? useOneDeltaType = GetBits(tmpData, bitIndex, 1); bitIndex++; bigDeltaBits = (int)GetBits(tmpData, bitIndex, 7); bitIndex += 7; if (useOneDeltaType == 0) { smallDeltaBits = (int)GetBits(tmpData, bitIndex, 7); bitIndex += 7; } } if (useOneDelta) { sign = monotoneFlag & 1; if (sign == 0) { for (i = 1; i < resLength; i++) { res.Add(res[i - 1] + delta); } } else { for (i = 1; i < resLength; i++) { res.Add(res[i - 1] - delta); } } } else { for (i = 1; i < resLength; i++) { int actualDeltaBits = bigDeltaBits; if (useOneDeltaType == 0) { ulong deltaType = GetBits(tmpData, bitIndex, 1); bitIndex++; if (deltaType == 0) { actualDeltaBits = smallDeltaBits; } } if (monotoneFlag == 0) { sign = GetBits(tmpData, bitIndex, 1); bitIndex++; } else { sign = monotoneFlag & 1; } delta = (long)GetBits(tmpData, bitIndex, actualDeltaBits); bitIndex += actualDeltaBits; res.Add(res[i - 1] + ((sign == 0) ? delta : -delta)); } } return(res); }
public static void Compress(BinaryWriter writer, long[] values, int index, int count, Helper helper) { writer.Write(VERSION); if (count == 0) { return; } long value = values[index]; CountCompression.Serialize(writer, (ulong)value); if (count == 1) { return; } Debug.Assert(count == helper.Count); if (!helper.IsReady) { helper.Prepare(); } index++; count--; int maxIndex = index + count - 1; helper.Serialize(writer); if (helper.Type == HelperType.Raw) { while (index <= maxIndex) { writer.Write(values[index++]); } return; } if (helper.Type == HelperType.OneStep) { return; } ulong maxDelta = helper.Delta; bool alwaysUseDelta = helper.AlwaysUseDelta; int bitCount = BitUtils.GetBitBounds(maxDelta); bool writeSign = helper.Type == HelperType.Delta; CommonArray common = new CommonArray(); int sizeBits = helper.SizeBits > 0 ? helper.SizeBits : (1 + 1 + 64) * (count - 1); common.ByteArray = new byte[(int)Math.Ceiling(sizeBits / 8.0)]; ulong[] data = common.UInt64Array; int bitIndex = 0; ulong delta; bool sign; //false - positive, true - negative for (; index <= maxIndex; index++) { long newValue = values[index]; if (newValue >= value) { sign = false; delta = (ulong)(newValue - value); } else { sign = true; delta = (ulong)(value - newValue); } if (alwaysUseDelta || delta <= maxDelta) { if (!alwaysUseDelta) { SetFlag(data, bitIndex /*, true*/); //use delta bitIndex++; } if (writeSign) { if (sign) { SetFlag(data, bitIndex /*, sign*/); } bitIndex++; } SetBits(data, bitIndex, delta, bitCount); bitIndex += bitCount; } else { //SetFlag(data, bitIndex, false); //don't use delta bitIndex++; SetValue(data, bitIndex, (ulong)newValue); bitIndex += 64; } value = newValue; } int bytesCount = (int)Math.Ceiling(bitIndex / 8.0); CountCompression.Serialize(writer, (ulong)bytesCount); writer.Write(common.ByteArray, 0, bytesCount); }
public static void Decompress(BinaryReader reader, Action <int, long> values, int count) { if (reader.ReadByte() != VERSION) { throw new Exception("Invalid delta compression version."); } if (count == 0) { return; } int index = 0; long value = (long)CountCompression.Deserialize(reader); values(index, value); if (count == 1) { return; } var helper = Helper.Deserialize(reader); index++; count--; int maxIndex = index + count - 1; if (helper.Type == HelperType.Raw) { while (index <= maxIndex) { values(index++, reader.ReadInt64()); } return; } if (helper.Type == HelperType.OneStep) { long step = (long)helper.Delta; if (helper.Sign == false) { while (index <= maxIndex) { values(index++, (value += step)); } } else { while (index <= maxIndex) { values(index++, (value -= step)); } } return; } bool readSign = helper.Type == HelperType.Delta; ulong maxDelta = helper.Delta; bool alwaysUseDelta = helper.AlwaysUseDelta; bool sign = helper.Sign; int bitCount = helper.DeltaBits; CommonArray common = new CommonArray(); int bytesCount = (int)CountCompression.Deserialize(reader); common.ByteArray = reader.ReadBytes(bytesCount); ulong[] data = common.UInt64Array; int bitIndex = 0; for (; index <= maxIndex; index++) { long newValue; bool useDelta = alwaysUseDelta || GetFlag(data, bitIndex++); if (useDelta) { if (readSign) { sign = GetFlag(data, bitIndex); bitIndex++; } long delta = (long)GetBits(data, bitIndex, bitCount); bitIndex += bitCount; newValue = sign ? value - delta : value + delta; } else { newValue = (long)GetValue(data, bitIndex); bitIndex += 64; } values(index, newValue); value = newValue; } }