/// <summary> /// Based on the given input, finds the best compression to use on it. /// </summary> /// <param name="input">assumed to be point data (x,x,x,x,x,x,x)</param> /// <returns></returns> internal byte GetBestDefHuff(int[] input) { if (input.Length < 3) { return(NoCompression); } DeltaDelta xfDelDel = new DeltaDelta(); int xfData = 0; int exData = 0; // Perform delta delta 2 times to set up the internal state of // delta delta transform xfDelDel.Transform(input[0], ref xfData, ref exData); xfDelDel.Transform(input[1], ref xfData, ref exData); double sumSq = 0.0; // Compute the variance of the delta delta uint n = 2; for (; n < input.Length; n++) { xfDelDel.Transform(input[n], ref xfData, ref exData); if (0 == exData) { sumSq += ((double)xfData * (double)xfData); } } sumSq *= (0.205625 / (n - 1.0)); int i = DefaultFirstSquareRoot.Length - 2; for (; i > 1; i--) { if (sumSq > DefaultFirstSquareRoot[i]) { break; } } byte retVal = (byte)(IndexedHuffman | (byte)(i + 1)); return(retVal); }
/// <summary> /// Compress - compress the input[] into compressedData /// </summary> /// <param name="bitCount">The count of bits needed for all elements</param> /// <param name="input">input buffer</param> /// <param name="startInputIndex">offset into the input buffer</param> /// <param name="dtxf">data transform. can be null</param> /// <param name="compressedData">The list of bytes to write the compressed input to</param> internal void Compress(int bitCount, int[] input, int startInputIndex, DeltaDelta dtxf, List <byte> compressedData) { if (null == input || null == compressedData) { throw new ArgumentNullException(StrokeCollectionSerializer.ISFDebugMessage("input or compressed data was null in Compress")); } if (bitCount < 0) { throw new ArgumentOutOfRangeException("bitCount"); } if (bitCount == 0) { //adjust if the bitcount is 0 //(this makes bitCount 32) bitCount = (int)(Native.SizeOfInt << 3); } //have the writer adapt to the List<byte> passed in and write to it BitStreamWriter writer = new BitStreamWriter(compressedData); if (null != dtxf) { int xfData = 0; int xfExtra = 0; for (int i = startInputIndex; i < input.Length; i++) { dtxf.Transform(input[i], ref xfData, ref xfExtra); if (xfExtra != 0) { throw new InvalidOperationException(StrokeCollectionSerializer.ISFDebugMessage("Transform returned unexpected results")); } writer.Write((uint)xfData, bitCount); } } else { for (int i = startInputIndex; i < input.Length; i++) { writer.Write((uint)input[i], bitCount); } } }
/// <summary> /// Based on the given input, finds the best compression to use on it. /// </summary> /// <param name="input">assumed to be point data (x,x,x,x,x,x,x)</param> /// <returns></returns> internal byte GetBestDefHuff(int[] input) { if (input.Length < 3) { return NoCompression; } DeltaDelta xfDelDel = new DeltaDelta(); int xfData = 0; int exData = 0; // Perform delta delta 2 times to set up the internal state of // delta delta transform xfDelDel.Transform(input[0], ref xfData, ref exData); xfDelDel.Transform(input[1], ref xfData, ref exData); double sumSq = 0.0; // Compute the variance of the delta delta uint n = 2; for(; n < input.Length; n++) { xfDelDel.Transform(input[n], ref xfData, ref exData); if (0 == exData) { sumSq += ((double)xfData * (double)xfData); } } sumSq *= (0.205625 / (n - 1.0)); int i = DefaultFirstSquareRoot.Length - 2; for(; i > 1; i--) { if(sumSq > DefaultFirstSquareRoot[i]) { break; } } byte retVal = (byte)(IndexedHuffman | (byte)(i + 1)); return retVal; }
/// <summary> /// FindPacketAlgoByte /// </summary> /// <param name="input">input stream to find the best compression for</param> /// <param name="testDelDel"></param> internal byte FindPacketAlgoByte(int[] input, bool testDelDel) { if (input == null) { throw new ArgumentNullException("input"); } // Check for the input item count if (0 == input.Length) { return 0; } // If the input count is less than 3, we cannot do del del testDelDel = testDelDel && (input.Length < 3); int minVal, maxVal; int minDelDel, maxDelDel; uint startIndex = 1; int xfData = 0, xfExtra = 0; DeltaDelta delDel = new DeltaDelta(); // Initialize all the max-min's to initial value minVal = maxVal = minDelDel = maxDelDel = input[0]; // Skip first two elements for del-del if (testDelDel) { delDel.Transform(input[0], ref xfData, ref xfExtra); delDel.Transform(input[1], ref xfData, ref xfExtra); // if we need extra bits, we cannot do del-del if (0 != xfExtra) { testDelDel = false; } } // Initialize DelDelMax/Min if we can do del-del if (testDelDel) { delDel.Transform(input[2], ref xfData, ref xfExtra); // Again, if nExtra is non-zero, we cannot do del-del if (0 != xfExtra) { testDelDel = false; } else { minDelDel = maxDelDel = xfData; // Update raw max/min for two elements UpdateMinMax(input[1], ref maxVal, ref minVal); UpdateMinMax(input[2], ref maxVal, ref minVal); // Following loop starts from 3 startIndex = 3; } } for (uint dataIndex = startIndex; dataIndex < input.Length; dataIndex++) { // Update the raw min-max first UpdateMinMax(input[dataIndex], ref maxVal, ref minVal); if (testDelDel) { // If we can do del-del, first do the transformation delDel.Transform(input[dataIndex], ref xfData, ref xfExtra); // again, cannot do del-del if xfExtra is non-zero // otherwise, update the del-del min/max if (0 != xfExtra) { testDelDel = false; } else { UpdateMinMax(xfData, ref maxDelDel, ref minDelDel); } } } // Find the absolute max for del-del uint ulAbsMaxDelDel = (uint)Math.Max(MathHelper.AbsNoThrow(minDelDel), MathHelper.AbsNoThrow(maxDelDel)); // Find the Math.Abs max for raw uint ulAbsMax = (uint)Math.Max(MathHelper.AbsNoThrow(minVal), MathHelper.AbsNoThrow(maxVal)); // If we could do del-del and Math.Abs max of del-del is at least twice smaller than // original, we do del-del, otherwise, we bitpack raw data if (testDelDel && ((ulAbsMaxDelDel >> 1) < ulAbsMax)) { ulAbsMax = ulAbsMaxDelDel; } else { testDelDel = false; } // Absolute bits int bitCount = 0; while ((0 != (ulAbsMax >> bitCount)) && (31 > bitCount)) { bitCount++; } // Sign bit bitCount++; // Return the algo data return (byte)((byte)(bitCount & 0x1F) | (testDelDel ? (byte)0x20 : (byte)0)); }
/// <summary> /// Uncompress - uncompress a byte[] into an int[] of point data (x,x,x,x,x) /// </summary> /// <param name="bitCount">The number of bits each element uses in input</param> /// <param name="input">compressed data</param> /// <param name="inputIndex">index to begin decoding at</param> /// <param name="dtxf">data xf, can be null</param> /// <param name="outputBuffer">output buffer that is prealloc'd to write to</param> /// <param name="outputBufferIndex">the index of the output buffer to write to</param> internal uint Uncompress(int bitCount, byte[] input, int inputIndex, DeltaDelta dtxf, int[] outputBuffer, int outputBufferIndex) { if (null == input) { throw new ArgumentNullException("input"); } if (inputIndex >= input.Length) { throw new ArgumentOutOfRangeException("inputIndex"); } if (null == outputBuffer) { throw new ArgumentNullException("outputBuffer"); } if (outputBufferIndex >= outputBuffer.Length) { throw new ArgumentOutOfRangeException("outputBufferIndex"); } if (bitCount < 0) { throw new ArgumentOutOfRangeException("bitCount"); } // Adjust bit count if 0 passed in if (bitCount == 0) { //adjust if the bitcount is 0 //(this makes bitCount 32) bitCount = (int)(Native.SizeOfInt << 3); } // Test whether the items are signed. For unsigned number, we don't need mask // If we are trying to compress signed long values with bit count = 5 // The mask will be 1111 1111 1111 0000. The way it is done is, if the 5th // bit is 1, the number is negative numbe, othrwise it's positive. Testing // will be non-zero, ONLY if the 5th bit is 1, in which case we OR with the mask // otherwise we leave the number as it is. uint bitMask = (unchecked((uint)~0) << (bitCount - 1)); uint bitData = 0; BitStreamReader reader = new BitStreamReader(input, inputIndex); if(dtxf != null) { while (!reader.EndOfStream) { bitData = reader.ReadUInt32(bitCount); // Construct the item bitData = ((bitData & bitMask) != 0) ? bitMask | bitData : bitData; int result = dtxf.InverseTransform((int)bitData, 0); Debug.Assert(outputBufferIndex < outputBuffer.Length); outputBuffer[outputBufferIndex++] = result; if (outputBufferIndex == outputBuffer.Length) { //only write as much as the outputbuffer can hold //this is assumed by calling code break; } } } else { while (!reader.EndOfStream) { bitData = reader.ReadUInt32(bitCount); // Construct the item bitData = ((bitData & bitMask) != 0) ? bitMask | bitData : bitData; Debug.Assert(outputBufferIndex < outputBuffer.Length); outputBuffer[outputBufferIndex++] = (int)bitData; if (outputBufferIndex == outputBuffer.Length) { //only write as much as the outputbuffer can hold //this is assumed by calling code break; } } } // Calculate how many bytes were read from input buffer return (uint)((outputBuffer.Length * bitCount + 7) >> 3); }
/// <summary> /// Compress - compress the input[] into compressedData /// </summary> /// <param name="bitCount">The count of bits needed for all elements</param> /// <param name="input">input buffer</param> /// <param name="startInputIndex">offset into the input buffer</param> /// <param name="dtxf">data transform. can be null</param> /// <param name="compressedData">The list of bytes to write the compressed input to</param> internal void Compress(int bitCount, int[] input, int startInputIndex, DeltaDelta dtxf, List<byte> compressedData) { if (null == input || null == compressedData) { throw new ArgumentNullException(StrokeCollectionSerializer.ISFDebugMessage("input or compressed data was null in Compress")); } if (bitCount < 0) { throw new ArgumentOutOfRangeException("bitCount"); } if (bitCount == 0) { //adjust if the bitcount is 0 //(this makes bitCount 32) bitCount = (int)(Native.SizeOfInt << 3); } //have the writer adapt to the List<byte> passed in and write to it BitStreamWriter writer = new BitStreamWriter(compressedData); if (null != dtxf) { int xfData = 0; int xfExtra = 0; for (int i = startInputIndex; i < input.Length; i++) { dtxf.Transform(input[i], ref xfData, ref xfExtra); if (xfExtra != 0) { throw new InvalidOperationException(StrokeCollectionSerializer.ISFDebugMessage("Transform returned unexpected results")); } writer.Write((uint)xfData, bitCount); } } else { for (int i = startInputIndex; i < input.Length; i++) { writer.Write((uint)input[i], bitCount); } } }
/// <summary> /// Compresses int[] packet data, returns it as a byte[] /// </summary> /// <param name="input">assumed to be point data (x,x,x,x,x,x,x)</param> /// <param name="compression">magic byte specifying the compression to use</param> /// <returns></returns> internal byte[] CompressPacketData(int[] input, byte compression) { if (input == null) { throw new ArgumentNullException("input"); } List <byte> compressedData = new List <byte>(); //leave room at the beginning of //compressedData for the compression header byte //which we will add at the end compressedData.Add((byte)0); if (DefaultCompression == (DefaultCompression & compression)) { compression = GetBestDefHuff(input); } if (IndexedHuffman == (DefaultCompression & compression)) { DataXform dtxf = this.HuffModule.FindDtXf(compression); HuffCodec huffCodec = this.HuffModule.FindCodec(compression); huffCodec.Compress(dtxf, input, compressedData); if (((compressedData.Count - 1 /*for the algo byte we just made room for*/) >> 2) > input.Length) { //recompress with no compression (gorilla) compression = NoCompression; //reset compressedData.Clear(); compressedData.Add((byte)0); } } if (NoCompression == (DefaultCompression & compression)) { bool testDelDel = ((compression & 0x20) != 0); compression = this.GorillaCodec.FindPacketAlgoByte(input, testDelDel); DeltaDelta dtxf = null; if ((compression & 0x20) != 0) { dtxf = this.DeltaDelta; } int inputIndex = 0; if (null != dtxf) { //multibyteencode the first two values int xfData = 0; int xfExtra = 0; dtxf.ResetState(); dtxf.Transform(input[0], ref xfData, ref xfExtra); this.MultiByteCodec.SignEncode(xfData, compressedData); dtxf.Transform(input[1], ref xfData, ref xfExtra); this.MultiByteCodec.SignEncode(xfData, compressedData); //advance to the third member, we've already read the first two inputIndex = 2; } //Gorllia time int bitCount = (compression & 0x1F); this.GorillaCodec.Compress(bitCount, //the max count of bits required for each int input, //the input array to compress inputIndex, //the index to start compressing at dtxf, //data transform to use when compressing, can be null compressedData); //a ref to the compressed data that will be written to } // compression / algo data always goes in index 0 compressedData[0] = compression; return(compressedData.ToArray()); }
/// <summary> /// DecompressPacketData - given a compressed byte[], uncompress it to the outputBuffer /// </summary> /// <param name="input">compressed byte from the ISF stream</param> /// <param name="outputBuffer">prealloc'd buffer to write to</param> /// <returns></returns> internal uint DecompressPacketData(byte[] input, int[] outputBuffer) { if (input == null) { throw new ArgumentNullException("input"); } if (input.Length < 2) { throw new ArgumentException(StrokeCollectionSerializer.ISFDebugMessage("Input buffer passed was shorter than expected")); } if (outputBuffer == null) { throw new ArgumentNullException("outputBuffer"); } if (outputBuffer.Length == 0) { throw new ArgumentException(StrokeCollectionSerializer.ISFDebugMessage("output buffer length was zero")); } byte compression = input[0]; uint totalBytesRead = 1; //we just read one int inputIndex = 1; switch (compression & 0xC0) { case 0x80: //IndexedHuffman { DataXform dtxf = this.HuffModule.FindDtXf(compression); HuffCodec huffCodec = this.HuffModule.FindCodec(compression); totalBytesRead += huffCodec.Uncompress(dtxf, input, inputIndex, outputBuffer); return(totalBytesRead); } case 0x00: //NoCompression { int outputBufferIndex = 0; DeltaDelta dtxf = null; if ((compression & 0x20) != 0) { dtxf = this.DeltaDelta; } int bitCount = 0; if ((compression & 0x1F) == 0) { bitCount = Native.BitsPerInt; //32 } else { bitCount = (compression & 0x1F); } if (null != dtxf) { //must have at least two more bytes besides the //initial algo byte if (input.Length < 3) { throw new ArgumentException(StrokeCollectionSerializer.ISFDebugMessage("Input buffer was too short (must be at least 3 bytes)")); } //multibyteencode the first two values int xfData = 0; int xfExtra = 0; dtxf.ResetState(); uint bytesRead = this.MultiByteCodec.SignDecode(input, inputIndex, ref xfData); //advance our index inputIndex += (int)bytesRead; totalBytesRead += bytesRead; int result = dtxf.InverseTransform(xfData, xfExtra); Debug.Assert(outputBufferIndex < outputBuffer.Length); outputBuffer[outputBufferIndex++] = result; bytesRead = this.MultiByteCodec.SignDecode(input, inputIndex, ref xfData); //advance our index inputIndex += (int)bytesRead; totalBytesRead += bytesRead; result = dtxf.InverseTransform(xfData, xfExtra); Debug.Assert(outputBufferIndex < outputBuffer.Length); outputBuffer[outputBufferIndex++] = result; } totalBytesRead += this.GorillaCodec.Uncompress(bitCount, //the max count of bits required for each int input, //the input array to uncompress inputIndex, //the index to start uncompressing at dtxf, //data transform to use when compressing, can be null outputBuffer, //a ref to the output buffer to write to outputBufferIndex); //the index of the output buffer to write to return(totalBytesRead); } default: { throw new ArgumentException(StrokeCollectionSerializer.ISFDebugMessage("Invalid decompression algo byte")); } } }
/// <summary> /// FindPacketAlgoByte /// </summary> /// <param name="input">input stream to find the best compression for</param> /// <param name="testDelDel"></param> internal byte FindPacketAlgoByte(int[] input, bool testDelDel) { if (input == null) { throw new ArgumentNullException("input"); } // Check for the input item count if (0 == input.Length) { return(0); } // If the input count is less than 3, we cannot do del del testDelDel = testDelDel && (input.Length < 3); int minVal, maxVal; int minDelDel, maxDelDel; uint startIndex = 1; int xfData = 0, xfExtra = 0; DeltaDelta delDel = new DeltaDelta(); // Initialize all the max-min's to initial value minVal = maxVal = minDelDel = maxDelDel = input[0]; // Skip first two elements for del-del if (testDelDel) { delDel.Transform(input[0], ref xfData, ref xfExtra); delDel.Transform(input[1], ref xfData, ref xfExtra); // if we need extra bits, we cannot do del-del if (0 != xfExtra) { testDelDel = false; } } // Initialize DelDelMax/Min if we can do del-del if (testDelDel) { delDel.Transform(input[2], ref xfData, ref xfExtra); // Again, if nExtra is non-zero, we cannot do del-del if (0 != xfExtra) { testDelDel = false; } else { minDelDel = maxDelDel = xfData; // Update raw max/min for two elements UpdateMinMax(input[1], ref maxVal, ref minVal); UpdateMinMax(input[2], ref maxVal, ref minVal); // Following loop starts from 3 startIndex = 3; } } for (uint dataIndex = startIndex; dataIndex < input.Length; dataIndex++) { // Update the raw min-max first UpdateMinMax(input[dataIndex], ref maxVal, ref minVal); if (testDelDel) { // If we can do del-del, first do the transformation delDel.Transform(input[dataIndex], ref xfData, ref xfExtra); // again, cannot do del-del if xfExtra is non-zero // otherwise, update the del-del min/max if (0 != xfExtra) { testDelDel = false; } else { UpdateMinMax(xfData, ref maxDelDel, ref minDelDel); } } } // Find the absolute max for del-del uint ulAbsMaxDelDel = (uint)Math.Max(MathHelper.AbsNoThrow(minDelDel), MathHelper.AbsNoThrow(maxDelDel)); // Find the Math.Abs max for raw uint ulAbsMax = (uint)Math.Max(MathHelper.AbsNoThrow(minVal), MathHelper.AbsNoThrow(maxVal)); // If we could do del-del and Math.Abs max of del-del is at least twice smaller than // original, we do del-del, otherwise, we bitpack raw data if (testDelDel && ((ulAbsMaxDelDel >> 1) < ulAbsMax)) { ulAbsMax = ulAbsMaxDelDel; } else { testDelDel = false; } // Absolute bits int bitCount = 0; while ((0 != (ulAbsMax >> bitCount)) && (31 > bitCount)) { bitCount++; } // Sign bit bitCount++; // Return the algo data return((byte)((byte)(bitCount & 0x1F) | (testDelDel ? (byte)0x20 : (byte)0))); }
/// <summary> /// Uncompress - uncompress a byte[] into an int[] of point data (x,x,x,x,x) /// </summary> /// <param name="bitCount">The number of bits each element uses in input</param> /// <param name="input">compressed data</param> /// <param name="inputIndex">index to begin decoding at</param> /// <param name="dtxf">data xf, can be null</param> /// <param name="outputBuffer">output buffer that is prealloc'd to write to</param> /// <param name="outputBufferIndex">the index of the output buffer to write to</param> internal uint Uncompress(int bitCount, byte[] input, int inputIndex, DeltaDelta dtxf, int[] outputBuffer, int outputBufferIndex) { if (null == input) { throw new ArgumentNullException("input"); } if (inputIndex >= input.Length) { throw new ArgumentOutOfRangeException("inputIndex"); } if (null == outputBuffer) { throw new ArgumentNullException("outputBuffer"); } if (outputBufferIndex >= outputBuffer.Length) { throw new ArgumentOutOfRangeException("outputBufferIndex"); } if (bitCount < 0) { throw new ArgumentOutOfRangeException("bitCount"); } // Adjust bit count if 0 passed in if (bitCount == 0) { //adjust if the bitcount is 0 //(this makes bitCount 32) bitCount = (int)(Native.SizeOfInt << 3); } // Test whether the items are signed. For unsigned number, we don't need mask // If we are trying to compress signed long values with bit count = 5 // The mask will be 1111 1111 1111 0000. The way it is done is, if the 5th // bit is 1, the number is negative numbe, othrwise it's positive. Testing // will be non-zero, ONLY if the 5th bit is 1, in which case we OR with the mask // otherwise we leave the number as it is. uint bitMask = (unchecked ((uint)~0) << (bitCount - 1)); uint bitData = 0; BitStreamReader reader = new BitStreamReader(input, inputIndex); if (dtxf != null) { while (!reader.EndOfStream) { bitData = reader.ReadUInt32(bitCount); // Construct the item bitData = ((bitData & bitMask) != 0) ? bitMask | bitData : bitData; int result = dtxf.InverseTransform((int)bitData, 0); Debug.Assert(outputBufferIndex < outputBuffer.Length); outputBuffer[outputBufferIndex++] = result; if (outputBufferIndex == outputBuffer.Length) { //only write as much as the outputbuffer can hold //this is assumed by calling code break; } } } else { while (!reader.EndOfStream) { bitData = reader.ReadUInt32(bitCount); // Construct the item bitData = ((bitData & bitMask) != 0) ? bitMask | bitData : bitData; Debug.Assert(outputBufferIndex < outputBuffer.Length); outputBuffer[outputBufferIndex++] = (int)bitData; if (outputBufferIndex == outputBuffer.Length) { //only write as much as the outputbuffer can hold //this is assumed by calling code break; } } } // Calculate how many bytes were read from input buffer return((uint)((outputBuffer.Length * bitCount + 7) >> 3)); }