/// <summary>Compress a slice in memory</summary> public static int CompressTo(Slice input, CompressedBitmapWriter output) { if (input.IsNullOrEmpty) { return(0); } unsafe { fixed(byte *ptr = input.Array) { return(CompressToUnsafe(ptr + input.Offset, input.Count, output)); } } }
/// <summary>Performs a logical NOT on a compressed bitmaps</summary> /// <param name="bitmap">Compressed bitmap</param> /// <param name="size">Minimum logical size of the result (bits in the uncompressed bitmap)</param> /// <returns>Compressed slice with the result of flipping all the bits in <paramref name="bitmap"/>, containing up to at least <paramref name="size"/> bits.</returns> /// <remarks>If <paramref name="bitmap"/> is larger than <paramref name="size"/>, then the resulting bitmap will be larger.</remarks> public static CompressedBitmap Not(this CompressedBitmap bitmap, int size) { if (bitmap == null) { throw new ArgumentNullException(nameof(bitmap)); } // there is a high change that the final bitmap will have the same size, with an optional extra filler word at the end var writer = new CompressedBitmapWriter(bitmap.Count + 1); int n = 0; if (bitmap.Count > 0) { foreach (var word in bitmap) { if (word.IsLiteral) { writer.Write(CompressedWord.MakeLiteral((uint)(~word.Literal))); n += 31; } else { int fc = word.FillCount; writer.Write(word.FillBit == 1 ? CompressedWord.ALL_ZEROES : CompressedWord.ALL_ONES, fc); n += 31 * fc; } } } if (n < size) { writer.Write(CompressedWord.ALL_ONES, size / 31); int r = size % 31; if (r > 0) { writer.Write((1u << r) - 1); } } return(writer.GetBitmap()); }
/// <summary>Compress a buffer in native memory</summary> /// <param name="buffer">Pointer to the native memory buffer to compress</param> /// <param name="count">Number of bytes to compress</param> /// <param name="output">Where to write the compressed words</param> /// <returns>Number of extra bits that where output in the last literal word (or none if 0)</returns> internal static unsafe int CompressToUnsafe(byte *buffer, int count, CompressedBitmapWriter output) { // Simplified algorithm: // 1) read 31 bits from input (BE) // 2) if not all 0 (or all 1), then output a literal word (MSB set to 0), and jump back to step 1) // 3) set LENGTH = 1, FILL_BIT = 0 (or 1) // 4) Peek at next 31 bits, and if they are still all 0 (or 1), increment N, and jump back to step 4) // 5) output a repeat word, with MSB set to 1, followed by FILL_BIT, and then LENGTH-1 (30 bit), and jump back to step 1) // Optimizations: // - for very small inputs (3 bytes or less) we return a single literal word // - we read 64 bits at a time in the buffer, because it fits nicely in an UInt64 register var bucket = new UncompressedWordReader(buffer, count); uint word; while ((word = bucket.Read()) != UncompressedWordReader.NotEnough) { output.Write(word); } // if there are remaining bits, they are padded with 0 and written as a literal int bits = bucket.Bits; if (bits > 0) { //note: MSB will already be 0 word = bucket.ReadLast(); output.Write(word); } // write the header output.Pack(); return(bits); }
internal static CompressedBitmap CompressedBinaryExpression([NotNull] CompressedBitmap left, [NotNull] CompressedBitmap right, LogicalOperation op) { Contract.Requires(left != null && right != null && op != LogicalOperation.And && Enum.IsDefined(typeof(LogicalOperation), op)); var writer = new CompressedBitmapWriter(); using (var liter = left.GetEnumerator()) using (var riter = right.GetEnumerator()) { int ln = 0; // remaining count of current word in left int rn = 0; // remaining count of current word in right int lw = 0; // value of current word in left (if ln > 0) int rw = 0; // value of current word in right (if rn > 0) const int DONE = -1; while (true) { if (ln == 0) { if (!liter.MoveNext()) { // left is done if (op == LogicalOperation.And || rn == DONE) { // no need to continue break; } // continue with right until it's done ln = DONE; lw = 0; continue; } ln = liter.Current.WordCount; lw = liter.Current.WordValue; } if (rn == 0) { if (!riter.MoveNext()) { // right is done if (op == LogicalOperation.And || ln == DONE) { // no need to continue break; } // continue with left until it's done rn = DONE; rw = 0; } rn = riter.Current.WordCount; rw = riter.Current.WordValue; } if (ln == DONE) { // copy right writer.Write((uint)rw, rn); rn = 0; } else if (rn == DONE) { // copy left writer.Write((uint)lw, ln); ln = 0; } else { // merge left & right int n = Math.Min(ln, rn); switch (op) { case LogicalOperation.And: writer.Write((uint)(lw & rw), n); break; case LogicalOperation.AndNot: writer.Write((uint)(lw & ~rw), n); break; case LogicalOperation.Or: writer.Write((uint)(lw | rw), n); break; case LogicalOperation.OrNot: writer.Write((uint)(lw | ~rw), n); break; case LogicalOperation.Xor: writer.Write((uint)(lw ^ rw), n); break; case LogicalOperation.XorNot: writer.Write((uint)(lw ^ ~rw), n); break; default: throw new InvalidOperationException(); } ln -= n; rn -= n; } } } return writer.GetBitmap(); }
public static CompressedBitmap Not([NotNull] this CompressedBitmap bitmap, int size) { if (bitmap == null) throw new ArgumentNullException("bitmap"); // there is a high change that the final bitmap will have the same size, with an optional extra filler word at the end var writer = new CompressedBitmapWriter(bitmap.Count + 1); int n = 0; if (bitmap.Count > 0) { foreach (var word in bitmap) { if (word.IsLiteral) { writer.Write(CompressedWord.MakeLiteral((uint)(~word.Literal))); n += 31; } else { int fc = word.FillCount; writer.Write(word.FillBit == 1 ? CompressedWord.ALL_ZEROES : CompressedWord.ALL_ONES, fc); n += 31 * fc; } } } if (n < size) { writer.Write(CompressedWord.ALL_ONES, size / 31); int r = size % 31; if (r > 0) writer.Write((1u << r) - 1); } return writer.GetBitmap(); }
/// <summary>Compress a buffer in native memory</summary> /// <param name="buffer">Pointer to the native memory buffer to compress</param> /// <param name="count">Number of bytes to compress</param> /// <param name="output">Where to write the compressed words</param> /// <returns>Number of extra bits that where output in the last literal word (or none if 0)</returns> internal static unsafe int CompressToUnsafe(byte* buffer, int count, CompressedBitmapWriter output) { // Simplified algorithm: // 1) read 31 bits from input (BE) // 2) if not all 0 (or all 1), then output a literal word (MSB set to 0), and jump back to step 1) // 3) set LENGTH = 1, FILL_BIT = 0 (or 1) // 4) Peek at next 31 bits, and if they are still all 0 (or 1), increment N, and jump back to step 4) // 5) output a repeat word, with MSB set to 1, followed by FILL_BIT, and then LENGTH-1 (30 bit), and jump back to step 1) // Optimisations: // - for very small inputs (3 bytes or less) we return a single literal word // - we read 64 bits at a time in the buffer, because it fits nicely in an UInt64 register var bucket = new UncompressedWordReader(buffer, count); uint word; while ((word = bucket.Read()) != UncompressedWordReader.NotEnough) { output.Write(word); } // if there are reamining bits, they are padded with 0 and written as a literal int bits = bucket.Bits; if (bits > 0) { //note: MSB will already be 0 word = bucket.ReadLast(); output.Write(word); } // write the header output.Pack(); return bits; }
/// <summary>Compress a slice in memory</summary> public static int CompressTo(Slice input, CompressedBitmapWriter output) { if (input.IsNullOrEmpty) return 0; unsafe { fixed (byte* ptr = input.Array) { return CompressToUnsafe(ptr + input.Offset, input.Count, output); } } }
/// <summary>Performs a binary operation between two compressed bitmaps</summary> /// <param name="left">First compressed bitmap</param> /// <param name="right">Second compressed bitmap</param> /// <param name="op">Type of operation to perform (And, Or, Xor, ...)</param> /// <returns>Compressed slice with the result of boolean expression <paramref name="left"/> AND <paramref name="right"/></returns> internal static CompressedBitmap CompressedBinaryExpression(CompressedBitmap left, CompressedBitmap right, LogicalOperation op) { Contract.Requires(left != null && right != null && /*op != LogicalOperation.And &&*/ Enum.IsDefined(typeof(LogicalOperation), op)); var writer = new CompressedBitmapWriter(); using (var liter = left.GetEnumerator()) using (var riter = right.GetEnumerator()) { int ln = 0; // remaining count of current word in left int rn = 0; // remaining count of current word in right uint lw = 0; // value of current word in left (if ln > 0) uint rw = 0; // value of current word in right (if rn > 0) const int DONE = -1; while (true) { if (ln == 0) { if (!liter.MoveNext()) { // left is done if (op == LogicalOperation.And || rn == DONE) { // no need to continue break; } // continue with right until it's done ln = DONE; lw = 0; continue; } ln = liter.Current.WordCount; lw = liter.Current.WordValue; } if (rn == 0) { if (!riter.MoveNext()) { // right is done if (op == LogicalOperation.And || ln == DONE) { // no need to continue break; } // continue with left until it's done rn = DONE; rw = 0; continue; } rn = riter.Current.WordCount; rw = riter.Current.WordValue; } if (ln == DONE) { // copy right writer.Write((uint)rw, rn); rn = 0; } else if (rn == DONE) { // copy left writer.Write((uint)lw, ln); ln = 0; } else { // merge left & right int n = Math.Min(ln, rn); switch (op) { case LogicalOperation.And: writer.Write((uint)(lw & rw), n); break; case LogicalOperation.AndNot: writer.Write((uint)(lw & (~rw & LITERAL_MASK)), n); break; case LogicalOperation.Or: writer.Write((uint)(lw | rw), n); break; case LogicalOperation.OrNot: writer.Write((uint)(lw | (~rw & LITERAL_MASK)), n); break; case LogicalOperation.Xor: writer.Write((uint)(lw ^ rw), n); break; case LogicalOperation.XorNot: writer.Write((uint)(lw ^ (~rw & LITERAL_MASK)), n); break; default: throw new InvalidOperationException(); } ln -= n; rn -= n; } } } return(writer.GetBitmap()); }