Ejemplo n.º 1
0
        public void headlessUncompress(int[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos, int num, IntWrapper initvalue)
        {
            if (inlength == 0)
            {
                return;
            }
            int init = inpos.get();

            F1.headlessUncompress(@in, inpos, inlength, @out, outpos, num, initvalue);
            inlength -= inpos.get() - init;
            num      -= outpos.get();
            F2.headlessUncompress(@in, inpos, inlength, @out, outpos, num, initvalue);
        }
Ejemplo n.º 2
0
    /*@Override*/
    public void uncompress(int[] @in, IntWrapper inpos, int inlength, int[] @out,
                           IntWrapper outpos)
    {
        if (inlength == 0)
        {
            return;
        }
        /* final */
        int outlength = @in[inpos.get()];

        inpos.increment();
        headlessUncompress(@in, inpos, inlength, @out, outpos, outlength);
    }
Ejemplo n.º 3
0
        public void uncompress(sbyte[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos)
        {
            int p          = inpos.get();
            int initoffset = 0;
            int finalp     = inpos.get() + inlength;
            int tmpoutpos  = outpos.get();

            for (int v = 0; p < finalp; @out[tmpoutpos++] = (initoffset = initoffset + v))
            {
                v = @in[p] & 0x7F;
                if (@in[p] < 0)
                {
                    p += 1;
                    continue;
                }
                v = ((@in[p + 1] & 0x7F) << 7) | v;
                if (@in[p + 1] < 0)
                {
                    p += 2;
                    continue;
                }
                v = ((@in[p + 2] & 0x7F) << 14) | v;
                if (@in[p + 2] < 0)
                {
                    p += 3;
                    continue;
                }
                v = ((@in[p + 3] & 0x7F) << 21) | v;
                if (@in[p + 3] < 0)
                {
                    p += 4;
                    continue;
                }
                v  = ((@in[p + 4] & 0x7F) << 28) | v;
                p += 5;
            }
            outpos.set(tmpoutpos);
            inpos.add(p);
        }
Ejemplo n.º 4
0
        public void headlessUncompress(int[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos, int num)
        {
            int outlength = Util.greatestMultiple(num, BLOCK_SIZE);
            int tmpinpos  = inpos.get();
            int s         = outpos.get();

            for (; s + BLOCK_SIZE * 4 - 1 < outpos.get() + outlength; s += BLOCK_SIZE * 4)
            {
                int mbits1 = (int)((uint)@in[tmpinpos] >> 24);

                int mbits2 = (int)((uint)@in[tmpinpos] >> 16) & 0xFF;

                int mbits3 = (int)((uint)@in[tmpinpos] >> 8) & 0xFF;

                int mbits4 = (int)((uint)@in[tmpinpos]) & 0xFF;
                ++tmpinpos;
                BitPacking.fastunpack(@in, tmpinpos, @out, s, mbits1);
                tmpinpos += mbits1;
                BitPacking
                .fastunpack(@in, tmpinpos, @out, s + BLOCK_SIZE, mbits2);
                tmpinpos += mbits2;
                BitPacking.fastunpack(@in, tmpinpos, @out, s + 2 * BLOCK_SIZE,
                                      mbits3);
                tmpinpos += mbits3;
                BitPacking.fastunpack(@in, tmpinpos, @out, s + 3 * BLOCK_SIZE,
                                      mbits4);
                tmpinpos += mbits4;
            }
            for (; s < outpos.get() + outlength; s += BLOCK_SIZE)
            {
                int mbits = @in[tmpinpos];
                ++tmpinpos;
                BitPacking.fastunpack(@in, tmpinpos, @out, s, mbits);
                tmpinpos += mbits;
            }
            outpos.add(outlength);
            inpos.set(tmpinpos);
        }
        private static int compress(PerformanceLogger logger, IntegerCODEC codec, int[] src, int[] dst)
        {
            IntWrapper inpos  = new IntWrapper();
            IntWrapper outpos = new IntWrapper();

            logger.compressionTimer.start();
            codec.compress(src, inpos, src.Length, dst, outpos);
            logger.compressionTimer.end();
            int outSize = outpos.get();

            logger.addOriginalSize(src.Length);
            logger.addCompressedSize(outSize);
            return(outSize);
        }
Ejemplo n.º 6
0
        private static void testCodec(IntegerCODEC c, IntegerCODEC co, int[][] data, int max)
        {
            int N         = data.Length;
            int maxlength = 0;

            for (int k = 0; k < N; ++k)
            {
                if (data[k].Length > maxlength)
                {
                    maxlength = data[k].Length;
                }
            }
            int[] buffer  = new int[maxlength + 1024];
            int[] dataout = new int[4 * maxlength + 1024];
            // 4x + 1024 to account for the possibility of some negative
            // compression.
            IntWrapper inpos  = new IntWrapper();
            IntWrapper outpos = new IntWrapper();

            for (int k = 0; k < N; ++k)
            {
                int[] backupdata = Arrays.copyOf(data[k], data[k].Length);

                inpos.set(1);
                outpos.set(0);

                if (!(c is IntegratedIntegerCODEC))
                {
                    Delta.delta(backupdata);
                }

                c.compress(backupdata, inpos, backupdata.Length - inpos.get(), dataout, outpos);

                int thiscompsize = outpos.get() + 1;
                inpos.set(0);
                outpos.set(1);
                buffer[0] = backupdata[0];
                co.uncompress(dataout, inpos, thiscompsize - 1, buffer, outpos);

                if (!(c is IntegratedIntegerCODEC))
                {
                    Delta.fastinverseDelta(buffer);
                }

                // Check assertions.
                Assert2.assertEquals(outpos.get(), data[k].Length); //"length is not match"
                int[] bufferCutout = Arrays.copyOf(buffer, outpos.get());
                Assert2.assertArrayEquals(data[k], bufferCutout);   //"failed to reconstruct original data"
            }
        }
Ejemplo n.º 7
0
        public void compress(int[] inBuf, IntWrapper inPos, int inLen, int[] outBuf, IntWrapper outPos)
        {
            inLen = inLen - inLen % BLOCK_LENGTH;
            if (inLen == 0)
            {
                return;
            }

            outBuf[outPos.get()] = inLen;
            outPos.increment();

            DeltaZigzagEncoding.Encoder ctx = new DeltaZigzagEncoding.Encoder(0);
            int[] work = new int[BLOCK_LENGTH];

            int op        = outPos.get();
            int ip        = inPos.get();
            int inPosLast = ip + inLen;

            for (; ip < inPosLast; ip += BLOCK_LENGTH)
            {
                ctx.encodeArray(inBuf, ip, BLOCK_LENGTH, work);
                int bits1 = Util.maxbits32(work, 0);
                int bits2 = Util.maxbits32(work, 32);
                int bits3 = Util.maxbits32(work, 64);
                int bits4 = Util.maxbits32(work, 96);
                outBuf[op++] = (bits1 << 24) | (bits2 << 16)
                               | (bits3 << 8) | (bits4 << 0);
                op += pack(work, 0, outBuf, op, bits1);
                op += pack(work, 32, outBuf, op, bits2);
                op += pack(work, 64, outBuf, op, bits3);
                op += pack(work, 96, outBuf, op, bits4);
            }

            inPos.add(inLen);
            outPos.set(op);
        }
Ejemplo n.º 8
0
        private static int compressWithSkipTable(object c, int[] data, int[] output, IntWrapper outpos, int[] metadata, int blocksize)
        {
            int metapos = 0;

            metadata[metapos++] = data.Length;
            IntWrapper inpos     = new IntWrapper();
            int        initvalue = 0;
            IntWrapper ival      = new IntWrapper(initvalue);

            while (inpos.get() < data.Length)
            {
                metadata[metapos++] = outpos.get();
                metadata[metapos++] = initvalue;
                if (c is SkippableIntegerCODEC)
                {
                    int size = blocksize > data.Length - inpos.get() ? data.Length
                               - inpos.get() : blocksize;
                    initvalue = Delta.delta(data, inpos.get(), size, initvalue);

                    ((SkippableIntegerCODEC)c).headlessCompress(data, inpos,
                                                                blocksize, output, outpos);
                }
                else if (c is SkippableIntegratedIntegerCODEC)
                {
                    ival.set(initvalue);
                    ((SkippableIntegratedIntegerCODEC)c).headlessCompress(data,
                                                                          inpos, blocksize, output, outpos, ival);
                    initvalue = ival.get();
                }
                else
                {
                    throw new Exception("Unrecognized codec " + c);
                }
            }
            return(metapos);
        }
Ejemplo n.º 9
0
        public void headlessDemo()
        {
            _testOutputHelper.WriteLine("Compressing arrays with minimal header...");
            int[] uncompressed1 = { 1, 2, 1, 3, 1 };
            int[] uncompressed2 = { 3, 2, 4, 6, 1 };

            int[] compressed = new int[uncompressed1.Length + uncompressed2.Length + 1024];

            SkippableIntegerCODEC codec = new SkippableComposition(new BinaryPacking(), new VariableByte());

            // compressing
            IntWrapper outPos = new IntWrapper();

            IntWrapper previous = new IntWrapper();

            codec.headlessCompress(uncompressed1, new IntWrapper(), uncompressed1.Length, compressed, outPos);
            int length1 = outPos.get() - previous.get();

            previous = new IntWrapper(outPos.get());
            codec.headlessCompress(uncompressed2, new IntWrapper(), uncompressed2.Length, compressed, outPos);
            int length2 = outPos.get() - previous.get();

            compressed = Arrays.copyOf(compressed, length1 + length2);
            _testOutputHelper.WriteLine("compressed unsorted integers from " + uncompressed1.Length * 4 + "B to " + length1 * 4 + "B");
            _testOutputHelper.WriteLine("compressed unsorted integers from " + uncompressed2.Length * 4 + "B to " + length2 * 4 + "B");
            _testOutputHelper.WriteLine("Total compressed output " + compressed.Length);

            int[] recovered1 = new int[uncompressed1.Length];
            int[] recovered2 = new int[uncompressed1.Length];

            IntWrapper inPos = new IntWrapper();

            _testOutputHelper.WriteLine("Decoding first array starting at pos = " + inPos);
            codec.headlessUncompress(compressed, inPos, compressed.Length, recovered1, new IntWrapper(0), uncompressed1.Length);

            _testOutputHelper.WriteLine("Decoding second array starting at pos = " + inPos);
            codec.headlessUncompress(compressed, inPos, compressed.Length, recovered2, new IntWrapper(0), uncompressed2.Length);
            if (!Arrays.equals(uncompressed1, recovered1))
            {
                throw new Exception("First array does not match.");
            }
            if (!Arrays.equals(uncompressed2, recovered2))
            {
                throw new Exception("Second array does not match.");
            }

            _testOutputHelper.WriteLine("The arrays match, your code is probably ok.");
        }
Ejemplo n.º 10
0
        public void compress(int[] @in, IntWrapper inpos, int inlength, sbyte[] @out, IntWrapper outpos)
        {
            if (inlength == 0)
            {
                return;
            }
            int initoffset = 0;
            int outpostmp  = outpos.get();

            for (int k = inpos.get(); k < inpos.get() + inlength; ++k)
            {
                long val = (@in[k] - initoffset) & 0xFFFFFFFFL;  // To be consistent with unsigned integers in C/C++
                initoffset = @in[k];
                if (val < (1 << 7))
                {
                    @out[outpostmp++] = (sbyte)(val | (1 << 7));
                }
                else if (val < (1 << 14))
                {
                    @out[outpostmp++] = (sbyte)extract7bits(0, val);
                    @out[outpostmp++] = (sbyte)(extract7bitsmaskless(1, (val)) | (1 << 7));
                }
                else if (val < (1 << 21))
                {
                    @out[outpostmp++] = (sbyte)extract7bits(0, val);
                    @out[outpostmp++] = (sbyte)extract7bits(1, val);
                    @out[outpostmp++] = (sbyte)(extract7bitsmaskless(2, (val)) | (1 << 7));
                }
                else if (val < (1 << 28))
                {
                    @out[outpostmp++] = (sbyte)extract7bits(0, val);
                    @out[outpostmp++] = (sbyte)extract7bits(1, val);
                    @out[outpostmp++] = (sbyte)extract7bits(2, val);
                    @out[outpostmp++] = (sbyte)(extract7bitsmaskless(3, (val)) | (1 << 7));
                }
                else
                {
                    @out[outpostmp++] = (sbyte)extract7bits(0, val);
                    @out[outpostmp++] = (sbyte)extract7bits(1, val);
                    @out[outpostmp++] = (sbyte)extract7bits(2, val);
                    @out[outpostmp++] = (sbyte)extract7bits(3, val);
                    @out[outpostmp++] = (sbyte)(extract7bitsmaskless(4, (val)) | (1 << 7));
                }
            }
            outpos.set(outpostmp);
            inpos.add(inlength);
        }
Ejemplo n.º 11
0
        public void headlessCompress(int[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos, IntWrapper initvalue)
        {
            if (inlength == 0)
            {
                return;
            }
            int init = inpos.get();

            F1.headlessCompress(@in, inpos, inlength, @out, outpos, initvalue);
            if (outpos.get() == 0)
            {
                @out[0] = 0;
                outpos.increment();
            }
            inlength -= inpos.get() - init;
            F2.headlessCompress(@in, inpos, inlength, @out, outpos, initvalue);
        }
Ejemplo n.º 12
0
        private void encodePage(int[] @in, IntWrapper inpos, int thissize,
                                int[] @out, IntWrapper outpos)
        {
            int        tmpoutpos  = outpos.get();
            int        tmpinpos   = inpos.get();
            IntWrapper bestb      = new IntWrapper();
            IntWrapper bestexcept = new IntWrapper();

            for (int finalinpos = tmpinpos + thissize; tmpinpos + BLOCK_SIZE <= finalinpos; tmpinpos += BLOCK_SIZE)
            {
                getBestBFromData(@in, tmpinpos, bestb, bestexcept);
                int tmpbestb   = bestb.get();
                int nbrexcept  = bestexcept.get();
                int exceptsize = 0;
                int remember   = tmpoutpos;
                tmpoutpos++;
                if (nbrexcept > 0)
                {
                    int c = 0;
                    for (int i = 0; i < BLOCK_SIZE; ++i)
                    {
                        if ((int)((uint)@in[tmpinpos + i] >> bits[tmpbestb]) != 0)
                        {
                            exceptbuffer[c + nbrexcept] = i;
                            exceptbuffer[c]             = (int)((uint)@in[tmpinpos + i] >> bits[tmpbestb]);
                            ++c;
                        }
                    }
                    exceptsize = S9.compress(exceptbuffer, 0,
                                             2 * nbrexcept, @out, tmpoutpos);
                    tmpoutpos += exceptsize;
                }
                @out[remember] = tmpbestb | (nbrexcept << 8)
                                 | (exceptsize << 16);
                for (int k = 0; k < BLOCK_SIZE; k += 32)
                {
                    BitPacking.fastpack(@in, tmpinpos + k, @out,
                                        tmpoutpos, bits[tmpbestb]);
                    tmpoutpos += bits[tmpbestb];
                }
            }
            inpos.set(tmpinpos);
            outpos.set(tmpoutpos);
        }
Ejemplo n.º 13
0
        public void compress(int[] inBuf, IntWrapper inPos, int inLen, int[] outBuf, IntWrapper outPos)
        {
            inLen = inLen - inLen % BLOCK_LENGTH;
            if (inLen == 0)
            {
                return;
            }

            outBuf[outPos.get()] = inLen;
            outPos.increment();

            int context = 0;

            int[] work = new int[32];

            int op        = outPos.get();
            int ip        = inPos.get();
            int inPosLast = ip + inLen;

            for (; ip < inPosLast; ip += BLOCK_LENGTH)
            {
                int bits1 = xorMaxBits(inBuf, ip + 0, 32, context);
                int bits2 = xorMaxBits(inBuf, ip + 32, 32,
                                       inBuf[ip + 31]);
                int bits3 = xorMaxBits(inBuf, ip + 64, 32,
                                       inBuf[ip + 63]);
                int bits4 = xorMaxBits(inBuf, ip + 96, 32,
                                       inBuf[ip + 95]);
                outBuf[op++] = (bits1 << 24) | (bits2 << 16)
                               | (bits3 << 8) | (bits4 << 0);
                op += xorPack(inBuf, ip + 0, outBuf, op, bits1,
                              context, work);
                op += xorPack(inBuf, ip + 32, outBuf, op, bits2,
                              inBuf[ip + 31], work);
                op += xorPack(inBuf, ip + 64, outBuf, op, bits3,
                              inBuf[ip + 63], work);
                op += xorPack(inBuf, ip + 96, outBuf, op, bits4,
                              inBuf[ip + 95], work);
                context = inBuf[ip + 127];
            }

            inPos.add(inLen);
            outPos.set(op);
        }
Ejemplo n.º 14
0
        public void consistentTest()
        {
            const int N = 4096;

            int[] data = new int[N];
            int[] rev  = new int[N];
            for (int k = 0; k < N; ++k)
            {
                data[k] = k % 128;
            }
            foreach (SkippableIntegerCODEC c in codecs)
            {
                _testOutputHelper.WriteLine("[SkippeableBasicTest.consistentTest] codec = " + c);
                int[] outBuf = new int[N + 1024];
                for (int n = 0; n <= N; ++n)
                {
                    IntWrapper inPos  = new IntWrapper();
                    IntWrapper outPos = new IntWrapper();
                    c.headlessCompress(data, inPos, n, outBuf, outPos);

                    IntWrapper inPoso  = new IntWrapper();
                    IntWrapper outPoso = new IntWrapper();
                    c.headlessUncompress(outBuf, inPoso, outPos.get(), rev,
                                         outPoso, n);
                    if (outPoso.get() != n)
                    {
                        throw new Exception("bug " + n);
                    }
                    if (inPoso.get() != outPos.get())
                    {
                        throw new Exception("bug " + n + " " + inPoso.get() + " " + outPos.get());
                    }
                    for (int j = 0; j < n; ++j)
                    {
                        if (data[j] != rev[j])
                        {
                            throw new Exception("bug");
                        }
                    }
                }
            }
        }
Ejemplo n.º 15
0
        public void headlessCompress(int[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos)
        {
            inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
            int tmpoutpos = outpos.get();
            int s         = inpos.get();

            for (; s + BLOCK_SIZE * 4 - 1 < inpos.get() + inlength; s += BLOCK_SIZE * 4)
            {
                int mbits1 = Util.maxbits(@in, s, BLOCK_SIZE);

                int mbits2 = Util.maxbits(@in, s + BLOCK_SIZE, BLOCK_SIZE);

                int mbits3 = Util.maxbits(@in, s + 2 * BLOCK_SIZE, BLOCK_SIZE);

                int mbits4 = Util.maxbits(@in, s + 3 * BLOCK_SIZE, BLOCK_SIZE);
                @out[tmpoutpos++] = (mbits1 << 24) | (mbits2 << 16)
                                    | (mbits3 << 8) | (mbits4);
                BitPacking.fastpackwithoutmask(@in, s, @out, tmpoutpos,
                                               mbits1);
                tmpoutpos += mbits1;
                BitPacking.fastpackwithoutmask(@in, s + BLOCK_SIZE, @out,
                                               tmpoutpos, mbits2);
                tmpoutpos += mbits2;
                BitPacking.fastpackwithoutmask(@in, s + 2 * BLOCK_SIZE, @out,
                                               tmpoutpos, mbits3);
                tmpoutpos += mbits3;
                BitPacking.fastpackwithoutmask(@in, s + 3 * BLOCK_SIZE, @out,
                                               tmpoutpos, mbits4);
                tmpoutpos += mbits4;
            }
            for (; s < inpos.get() + inlength; s += BLOCK_SIZE)
            {
                int mbits = Util.maxbits(@in, s, BLOCK_SIZE);
                @out[tmpoutpos++] = mbits;
                BitPacking.fastpackwithoutmask(@in, s, @out, tmpoutpos,
                                               mbits);
                tmpoutpos += mbits;
            }
            inpos.add(inlength);
            outpos.set(tmpoutpos);
        }
Ejemplo n.º 16
0
        public void uncompress(int[] inBuf, IntWrapper inPos, int inLen, int[] outBuf, IntWrapper outPos)
        {
            if (inLen == 0)
            {
                return;
            }

            int outLen = inBuf[inPos.get()];

            inPos.increment();

            int context = 0;

            int[] work = new int[32];

            int ip         = inPos.get();
            int op         = outPos.get();
            int outPosLast = op + outLen;

            for (; op < outPosLast; op += BLOCK_LENGTH)
            {
                int bits1 = (int)((uint)inBuf[ip] >> 24);
                int bits2 = (int)((uint)inBuf[ip] >> 16) & 0xFF;
                int bits3 = (int)((uint)inBuf[ip] >> 8) & 0xFF;
                int bits4 = (int)((uint)inBuf[ip] >> 0) & 0xFF;
                ++ip;
                ip += xorUnpack(inBuf, ip, outBuf, op + 0, bits1,
                                context, work);
                ip += xorUnpack(inBuf, ip, outBuf, op + 32, bits2,
                                outBuf[op + 31], work);
                ip += xorUnpack(inBuf, ip, outBuf, op + 64, bits3,
                                outBuf[op + 63], work);
                ip += xorUnpack(inBuf, ip, outBuf, op + 96, bits4,
                                outBuf[op + 95], work);
                context = outBuf[op + 127];
            }

            outPos.add(outLen);
            inPos.set(ip);
        }
Ejemplo n.º 17
0
        /**
         * Standard benchmark
         *
         * @param csvLog
         *            Writer for CSV log.
         * @param c
         *            the codec
         * @param data
         *            arrays of input data
         * @param repeat
         *            How many times to repeat the test
         * @param verbose
         *            whether to output result on screen
         */
        private static void testCodec(StreamWriter csvLog, int sparsity, object c, int[][] data, int repeat, bool verbose)
        {
            if (verbose)
            {
                Console.WriteLine("# " + c);
                Console.WriteLine("# bits per int, compress speed (mis), decompression speed (mis) ");
            }

            int N = data.Length;

            int totalSize = 0;
            int maxLength = 0;

            for (int k = 0; k < N; ++k)
            {
                totalSize += data[k].Length;
                if (data[k].Length > maxLength)
                {
                    maxLength = data[k].Length;
                }
            }

            // 4x + 1024 to account for the possibility of some negative
            // compression.
            int[] compressBuffer   = new int[4 * maxLength + 1024];
            int[] decompressBuffer = new int[maxLength + 1024];
            int[] metadataBuffer   = new int[maxLength];

            const int blocksize = 1024;

            // These variables hold time in microseconds (10^-6).
            double compressTime   = 0;
            double decompressTime = 0;

            const int times = 5;

            int size = 0;

            for (int r = 0; r < repeat; ++r)
            {
                size = 0;
                for (int k = 0; k < N; ++k)
                {
                    int[] backupdata = Arrays.copyOf(data[k], data[k].Length);

                    // compress data.
                    long       beforeCompress = Port.System.nanoTime() / 1000;
                    IntWrapper outpos         = new IntWrapper();
                    compressWithSkipTable(c, backupdata, compressBuffer, outpos,
                                          metadataBuffer, blocksize);
                    long afterCompress = Port.System.nanoTime() / 1000;

                    // measure time of compression.
                    compressTime += afterCompress - beforeCompress;

                    int thiscompsize = outpos.get();
                    size += thiscompsize;
                    // dry run
                    int volume = 0;
                    {
                        IntWrapper compressedpos = new IntWrapper(0);
                        volume = decompressFromSkipTable(c, compressBuffer,
                                                         compressedpos, metadataBuffer, blocksize,
                                                         decompressBuffer);

                        // let us check the answer
                        if (volume != backupdata.Length)
                        {
                            throw new Exception(
                                      "Bad output size with codec " + c);
                        }
                        for (int j = 0; j < volume; ++j)
                        {
                            if (data[k][j] != decompressBuffer[j])
                            {
                                throw new Exception("bug in codec " + c);
                            }
                        }
                    }
                    // extract (uncompress) data
                    long beforeDecompress = Port.System.nanoTime() / 1000;
                    for (int t = 0; t < times; ++t)
                    {
                        IntWrapper compressedpos = new IntWrapper(0);
                        volume = decompressFromSkipTable(c, compressBuffer,
                                                         compressedpos, metadataBuffer, blocksize,
                                                         decompressBuffer);
                    }
                    long afterDecompress = Port.System.nanoTime() / 1000;

                    // measure time of extraction (uncompression).
                    decompressTime += (afterDecompress - beforeDecompress) / (double)times;
                    if (volume != data[k].Length)
                    {
                        throw new Exception("we have a bug (diff length) "
                                            + c + " expected " + data[k].Length + " got "
                                            + volume);
                    }

                    // verify: compare original array with
                    // compressed and
                    // uncompressed.
                    for (int m = 0; m < outpos.get(); ++m)
                    {
                        if (decompressBuffer[m] != data[k][m])
                        {
                            throw new Exception(
                                      "we have a bug (actual difference), expected "
                                      + data[k][m] + " found "
                                      + decompressBuffer[m] + " at " + m);
                        }
                    }
                }
            }

            if (verbose)
            {
                double bitsPerInt      = size * 32.0 / totalSize;
                double compressSpeed   = Math.Round(totalSize * repeat / (compressTime));
                double decompressSpeed = Math.Round(totalSize * repeat / (decompressTime));
                Console.WriteLine("\t{0:0.00}\t{1}\t{2}", bitsPerInt, compressSpeed, decompressSpeed);
                csvLog.WriteLine("\"{0}\",{1},{2:0.00},{3},{4}", c, sparsity, bitsPerInt, compressSpeed, decompressSpeed);
            }
        }
Ejemplo n.º 18
0
        public void advancedExample()
        {
            const int TotalSize = 2342351; // some arbitrary number
            const int ChunkSize = 16384;   // size of each chunk, choose a multiple of 128

            Console.WriteLine("Compressing " + TotalSize + " integers using chunks of " + ChunkSize + " integers ("
                              + ChunkSize * 4 / 1024 + "KB)");
            Console.WriteLine("(It is often better for applications to work in chunks fitting in CPU cache.)");
            int[] data = new int[TotalSize];
            // data should be sorted for best
            // results
            for (int k = 0; k < data.Length; ++k)
            {
                data[k] = k;
            }
            // next we compose a CODEC. Most of the processing
            // will be done with binary packing, and leftovers will
            // be processed using variable byte, using variable byte
            // only for the last chunk!
            IntegratedIntegerCODEC regularcodec = new IntegratedBinaryPacking();
            IntegratedVariableByte ivb          = new IntegratedVariableByte();
            IntegratedIntegerCODEC lastcodec    = new IntegratedComposition(regularcodec, ivb);

            // output vector should be large enough...
            int[] compressed = new int[TotalSize + 1024];

            /**
             *
             * compressing
             *
             */
            IntWrapper inputoffset  = new IntWrapper(0);
            IntWrapper outputoffset = new IntWrapper(0);

            for (int k = 0; k < TotalSize / ChunkSize; ++k)
            {
                regularcodec.compress(data, inputoffset, ChunkSize, compressed, outputoffset);
            }
            lastcodec.compress(data, inputoffset, TotalSize % ChunkSize, compressed, outputoffset);
            // got it!
            // inputoffset should be at data.Length but outputoffset tells
            // us where we are...
            Console.WriteLine(
                "compressed from " + data.Length * 4 / 1024 + "KB to " + outputoffset.intValue() * 4 / 1024 + "KB");
            // we can repack the data:
            compressed = Arrays.copyOf(compressed, outputoffset.intValue());

            /**
             *
             * now uncompressing
             *
             * We are *not* assuming that the original array length is known,
             * however we assume that the chunk size (ChunkSize) is known.
             *
             */
            int[]      recovered = new int[ChunkSize];
            IntWrapper compoff   = new IntWrapper(0);
            IntWrapper recoffset;
            int        currentpos = 0;

            while (compoff.get() < compressed.Length)
            {
                recoffset = new IntWrapper(0);
                regularcodec.uncompress(compressed, compoff, compressed.Length - compoff.get(), recovered, recoffset);

                if (recoffset.get() < ChunkSize)
                {// last chunk detected
                    ivb.uncompress(compressed, compoff, compressed.Length - compoff.get(), recovered, recoffset);
                }
                for (int i = 0; i < recoffset.get(); ++i)
                {
                    if (data[currentpos + i] != recovered[i])
                    {
                        throw new Exception("bug"); // could use assert
                    }
                }
                currentpos += recoffset.get();
            }
            Console.WriteLine("data is recovered without loss");
            Console.WriteLine();
        }
Ejemplo n.º 19
0
        /**
         * Standard benchmark
         *
         * @param csvLog
         *                Writer for CSV log.
         * @param c
         *                the codec
         * @param data
         *                arrays of input data
         * @param repeat
         *                How many times to repeat the test
         * @param verbose
         *                whether to output result on screen
         */
        private static void testCodec(StreamWriter csvLog, int sparsity, IntegerCODEC c, int[][] data, int repeat, bool verbose)
        {
            if (verbose)
            {
                Console.WriteLine("# " + c);
                Console.WriteLine("# bits per int, compress speed (mis), decompression speed (mis) ");
            }

            int N = data.Length;

            int totalSize = 0;
            int maxLength = 0;

            for (int k = 0; k < N; ++k)
            {
                totalSize += data[k].Length;
                if (data[k].Length > maxLength)
                {
                    maxLength = data[k].Length;
                }
            }

            // 4x + 1024 to account for the possibility of some negative
            // compression.
            int[] compressBuffer   = new int[4 * maxLength + 1024];
            int[] decompressBuffer = new int[maxLength + 1024];

            // These variables hold time in microseconds (10^-6).
            long compressTime   = 0;
            long decompressTime = 0;

            int        size   = 0;
            IntWrapper inpos  = new IntWrapper();
            IntWrapper outpos = new IntWrapper();

            for (int r = 0; r < repeat; ++r)
            {
                size = 0;
                for (int k = 0; k < N; ++k)
                {
                    int[] backupdata = Arrays.copyOf(data[k],
                                                     data[k].Length);

                    // compress data.
                    long beforeCompress = Port.System.nanoTime() / 1000;
                    inpos.set(1);
                    outpos.set(0);
                    if (!(c is IntegratedIntegerCODEC))
                    {
                        Delta.delta(backupdata);
                    }
                    c.compress(backupdata, inpos, backupdata.Length
                               - inpos.get(), compressBuffer, outpos);
                    long afterCompress = Port.System.nanoTime() / 1000;

                    // measure time of compression.
                    compressTime += afterCompress - beforeCompress;

                    int thiscompsize = outpos.get() + 1;
                    size += thiscompsize;

                    // extract (uncompress) data
                    long beforeDecompress = Port.System.nanoTime() / 1000;
                    inpos.set(0);
                    outpos.set(1);
                    decompressBuffer[0] = backupdata[0];
                    c.uncompress(compressBuffer, inpos,
                                 thiscompsize - 1, decompressBuffer,
                                 outpos);
                    if (!(c is IntegratedIntegerCODEC))
                    {
                        Delta.fastinverseDelta(decompressBuffer);
                    }
                    long afterDecompress = Port.System.nanoTime() / 1000;

                    // measure time of extraction (uncompression).
                    decompressTime += afterDecompress
                                      - beforeDecompress;
                    if (outpos.get() != data[k].Length)
                    {
                        throw new Exception(
                                  "we have a bug (diff length) "
                                  + c + " expected "
                                  + data[k].Length
                                  + " got "
                                  + outpos.get());
                    }

                    // verify: compare original array with
                    // compressed and
                    // uncompressed.

                    for (int m = 0; m < outpos.get(); ++m)
                    {
                        if (decompressBuffer[m] != data[k][m])
                        {
                            throw new Exception(
                                      "we have a bug (actual difference), expected "
                                      + data[k][m]
                                      + " found "
                                      + decompressBuffer[m]
                                      + " at " + m);
                        }
                    }
                }
            }

            if (verbose)
            {
                double bitsPerInt      = size * 32.0 / totalSize;
                long   compressSpeed   = totalSize * repeat / (compressTime);
                long   decompressSpeed = totalSize * repeat / (decompressTime);

                Console.WriteLine("\t{0:0.00}\t{1}\t{2}", bitsPerInt, compressSpeed, decompressSpeed);
                csvLog.WriteLine("\"{0}\",{1},{2:0.00},{3},{4}", c, sparsity, bitsPerInt, compressSpeed, decompressSpeed);
            }
        }
Ejemplo n.º 20
0
        public void compress(int[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos)
        {
            if (inlength == 0)
            {
                return;
            }
            int initoffset = 0;

            ByteBuffer buf = ByteBuffer.allocateDirect(inlength * 8);

            buf.order(ByteOrder.LITTLE_ENDIAN);

            for (int k = inpos.get(); k < inpos.get() + inlength; ++k)
            {
                long val = (@in[k] - initoffset) & 0xFFFFFFFFL; // To be consistent with unsigned integers in C/C++
                initoffset = @in[k];
                if (val < (1 << 7))
                {
                    buf.put((sbyte)(val | (1 << 7)));
                }
                else if (val < (1 << 14))
                {
                    buf.put((sbyte)extract7bits(0, val));
                    buf.put((sbyte)(extract7bitsmaskless(1, (val)) | (1 << 7)));
                }
                else if (val < (1 << 21))
                {
                    buf.put((sbyte)extract7bits(0, val));
                    buf.put((sbyte)extract7bits(1, val));
                    buf.put((sbyte)(extract7bitsmaskless(2, (val)) | (1 << 7)));
                }
                else if (val < (1 << 28))
                {
                    buf.put((sbyte)extract7bits(0, val));
                    buf.put((sbyte)extract7bits(1, val));
                    buf.put((sbyte)extract7bits(2, val));
                    buf.put((sbyte)(extract7bitsmaskless(3, (val)) | (1 << 7)));
                }
                else
                {
                    buf.put((sbyte)extract7bits(0, val));
                    buf.put((sbyte)extract7bits(1, val));
                    buf.put((sbyte)extract7bits(2, val));
                    buf.put((sbyte)extract7bits(3, val));
                    buf.put((sbyte)(extract7bitsmaskless(4, (val)) | (1 << 7)));
                }
            }
            while (buf.position() % 4 != 0)
            {
                buf.put((sbyte)0);
            }
            int length = buf.position();

            buf.flip();
            IntBuffer ibuf = buf.asIntBuffer();

            ibuf.get(@out, outpos.get(), length / 4);

            outpos.add(length / 4);
            inpos.add(inlength);
        }
Ejemplo n.º 21
0
        public static void bytebench(List <int[]> postings, CompressionMode cm, bool verbose)
        {
            int maxlength = 0;

            foreach (int[] x in postings)
            {
                if (maxlength < x.Length)
                {
                    maxlength = x.Length;
                }
            }
            if (verbose)
            {
                Console.WriteLine("Max array length: " + maxlength);
            }
            sbyte[] compbuffer   = new sbyte[6 * (maxlength + 1024)];
            int[]   decompbuffer = new int[maxlength];
            if (verbose)
            {
                Console.WriteLine("Scheme -- bits/int -- speed (mis)");
            }

            foreach (ByteIntegerCODEC c in (cm == CompressionMode.DELTA ? bcodecs : regbcodecs))
            {
                long      bef        = 0;
                long      aft        = 0;
                long      decomptime = 0;
                long      volumein   = 0;
                long      volumeout  = 0;
                sbyte[][] compdata   = new sbyte[postings.Count][];
                for (int k = 0; k < postings.Count; ++k)
                {
                    int[]      @in    = postings[k];
                    IntWrapper inpos  = new IntWrapper(0);
                    IntWrapper outpos = new IntWrapper(0);
                    c.compress(@in, inpos, @in.Length, compbuffer,
                               outpos);
                    int clength = outpos.get();
                    inpos  = new IntWrapper(0);
                    outpos = new IntWrapper(0);
                    c.uncompress(compbuffer, inpos, clength,
                                 decompbuffer, outpos);
                    volumein  += @in.Length;
                    volumeout += clength;

                    if (outpos.get() != @in.Length)
                    {
                        throw new Exception("bug");
                    }
                    for (int z = 0; z < @in.Length; ++z)
                    {
                        if (@in[z] != decompbuffer[z])
                        {
                            throw new Exception(
                                      "bug");
                        }
                    }
                    compdata[k] = Arrays
                                  .copyOf(compbuffer, clength);
                }
                bef = Port.System.nanoTime();
                foreach (sbyte[] cin in compdata)
                {
                    IntWrapper inpos  = new IntWrapper(0);
                    IntWrapper outpos = new IntWrapper(0);
                    c.uncompress(cin, inpos, cin.Length,
                                 decompbuffer, outpos);
                    if (inpos.get() != cin.Length)
                    {
                        throw new Exception("bug");
                    }
                }
                aft         = Port.System.nanoTime();
                decomptime += (aft - bef);
                double bitsPerInt      = volumeout * 8.0 / volumein;
                double decompressSpeed = volumein * 1000.0 / (decomptime);
                if (verbose)
                {
                    Console.WriteLine(c + "\t" + string.Format("\t{0:0.00}\t{1:0.00}", bitsPerInt, decompressSpeed));
                }
            }
        }