private static void testCodec(IntegerCODEC c, IntegerCODEC co, int[][] data, int max)
{
int N = data.Length;
int maxlength = 0;
for (int k = 0; k < N; ++k)
{
if (data[k].Length > maxlength)
{
maxlength = data[k].Length;
}
}
int[] buffer = new int[maxlength + 1024];
int[] dataout = new int[4 * maxlength + 1024];
// 4x + 1024 to account for the possibility of some negative
// compression.
IntWrapper inpos = new IntWrapper();
IntWrapper outpos = new IntWrapper();
for (int k = 0; k < N; ++k)
{
int[] backupdata = Arrays.copyOf(data[k], data[k].Length);
inpos.set(1);
outpos.set(0);
if (!(c is IntegratedIntegerCODEC))
{
Delta.delta(backupdata);
}
c.compress(backupdata, inpos, backupdata.Length - inpos.get(), dataout, outpos);
int thiscompsize = outpos.get() + 1;
inpos.set(0);
outpos.set(1);
buffer[0] = backupdata[0];
co.uncompress(dataout, inpos, thiscompsize - 1, buffer, outpos);
if (!(c is IntegratedIntegerCODEC))
{
Delta.fastinverseDelta(buffer);
}
// Check assertions.
Assert2.assertEquals(outpos.get(), data[k].Length); //"length is not match"
int[] bufferCutout = Arrays.copyOf(buffer, outpos.get());
Assert2.assertArrayEquals(data[k], bufferCutout); //"failed to reconstruct original data"
}
}
private void decodePage(int[] @in, IntWrapper inpos, int[] @out, IntWrapper outpos, int thissize)
{
int tmpoutpos = outpos.get();
int tmpinpos = inpos.get();
for (int run = 0; run < thissize / BLOCK_SIZE; ++run, tmpoutpos += BLOCK_SIZE)
{
int b = @in[tmpinpos] & 0xFF;
int cexcept = (int)((uint)@in[tmpinpos] >> 8) & 0xFF;
int exceptsize = (int)((uint)@in[tmpinpos] >> 16);
++tmpinpos;
S9.uncompress(@in, tmpinpos, exceptsize, exceptbuffer,
0, 2 * cexcept);
tmpinpos += exceptsize;
for (int k = 0; k < BLOCK_SIZE; k += 32)
{
BitPacking.fastunpack(@in, tmpinpos, @out,
tmpoutpos + k, bits[b]);
tmpinpos += bits[b];
}
for (int k = 0; k < cexcept; ++k)
{
@out[tmpoutpos + exceptbuffer[k + cexcept]] |= (exceptbuffer[k] << bits[b]);
}
}
outpos.set(tmpoutpos);
inpos.set(tmpinpos);
}
protected static void getBestBFromData(int[] @in, int pos, IntWrapper bestb, IntWrapper bestexcept)
{
int mb = Util.maxbits(@in, pos, BLOCK_SIZE);
int mini = 0;
if (mini + 28 < bits[invbits[mb]])
{
mini = bits[invbits[mb]] - 28; // 28 is the max for
}
// exceptions
int besti = bits.Length - 1;
int exceptcounter = 0;
for (int i = mini; i < bits.Length - 1; ++i)
{
int tmpcounter = 0;
for (int k = pos; k < BLOCK_SIZE + pos; ++k)
{
if ((int)((uint)@in[k] >> bits[i]) != 0)
{
++tmpcounter;
}
}
if (tmpcounter * 10 <= BLOCK_SIZE)
{
besti = i;
exceptcounter = tmpcounter;
break;
}
}
bestb.set(besti);
bestexcept.set(exceptcounter);
}
public void uncompress(int[] inBuf, IntWrapper inPos, int inLen, int[] outBuf, IntWrapper outPos)
{
if (inLen == 0)
{
return;
}
int outLen = inBuf[inPos.get()];
inPos.increment();
DeltaZigzagEncoding.Decoder ctx = new DeltaZigzagEncoding.Decoder(0);
int[] work = new int[BLOCK_LENGTH];
int ip = inPos.get();
int op = outPos.get();
int outPosLast = op + outLen;
for (; op < outPosLast; op += BLOCK_LENGTH)
{
int n = inBuf[ip++];
ip += unpack(inBuf, ip, work, 0, (n >> 24) & 0x3F);
ip += unpack(inBuf, ip, work, 32, (n >> 16) & 0x3F);
ip += unpack(inBuf, ip, work, 64, (n >> 8) & 0x3F);
ip += unpack(inBuf, ip, work, 96, (n >> 0) & 0x3F);
ctx.decodeArray(work, 0, BLOCK_LENGTH, outBuf, op);
}
outPos.add(outLen);
inPos.set(ip);
}
public void headlessUncompress(int[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos, int num, IntWrapper initvalue)
{
int s = 0;
int val = 0;
int p = inpos.get();
int initoffset = initvalue.get();
int tmpoutpos = outpos.get();
int finaloutpos = num + tmpoutpos;
for (int v = 0, shift = 0; tmpoutpos < finaloutpos;)
{
val = @in[p];
int c = (int)((uint)val >> s);
s += 8;
p += s >> 5;
s = s & 31;
v += ((c & 127) << shift);
if ((c & 128) == 128)
{
@out[tmpoutpos++] = (initoffset = initoffset + v);
v = 0;
shift = 0;
}
else
{
shift += 7;
}
}
initvalue.set(@out[tmpoutpos - 1]);
outpos.set(tmpoutpos);
inpos.set(p + (s != 0 ? 1 : 0));
}
public void uncompress(int[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos)
{
int s = 0;
int val = 0;
int p = inpos.get();
int finalp = inpos.get() + inlength;
int tmpoutpos = outpos.get();
int initoffset = 0;
for (int v = 0, shift = 0; p < finalp;)
{
val = @in[p];
int c = (sbyte)(int)((uint)val >> s);
s += 8;
p += s >> 5;
s = s & 31;
v += ((c & 127) << shift);
if ((c & 128) == 128)
{
@out[tmpoutpos] = v + initoffset;
initoffset = @out[tmpoutpos];
tmpoutpos++;
v = 0;
shift = 0;
}
else
{
shift += 7;
}
}
outpos.set(tmpoutpos);
inpos.add(inlength);
}
public void saulTest()
{
foreach (IntegerCODEC C in codecs)
{
for (int x = 0; x < 50; ++x)
{
int[] a = { 2, 3, 4, 5 };
int[] b = new int[90];
int[] c = new int[a.Length];
IntWrapper aOffset = new IntWrapper(0);
IntWrapper bOffset = new IntWrapper(x);
C.compress(a, aOffset, a.Length, b, bOffset);
int len = bOffset.get() - x;
bOffset.set(x);
IntWrapper cOffset = new IntWrapper(0);
C.uncompress(b, bOffset, len, c, cOffset);
if (!Arrays.equals(a, c))
{
Console.WriteLine("Problem with " + C);
}
Assert2.assertArrayEquals(a, c);
}
}
}
public void headlessCompress(int[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos, IntWrapper initvalue)
{
inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
if (inlength == 0)
{
return;
}
int tmpoutpos = outpos.get();
int initoffset = initvalue.get();
initvalue.set(@in[inpos.get() + inlength - 1]);
int s = inpos.get();
for (; s + BLOCK_SIZE * 4 - 1 < inpos.get() + inlength; s += BLOCK_SIZE * 4)
{
int mbits1 = Util.maxdiffbits(initoffset, @in, s, BLOCK_SIZE);
int initoffset2 = @in[s + 31];
int mbits2 = Util.maxdiffbits(initoffset2, @in, s + BLOCK_SIZE, BLOCK_SIZE);
int initoffset3 = @in[s + BLOCK_SIZE + 31];
int mbits3 = Util
.maxdiffbits(initoffset3, @in, s + 2 * BLOCK_SIZE, BLOCK_SIZE);
int initoffset4 = @in[s + 2 * BLOCK_SIZE + 31];
int mbits4 = Util
.maxdiffbits(initoffset4, @in, s + 3 * BLOCK_SIZE, BLOCK_SIZE);
@out[tmpoutpos++] = (mbits1 << 24) | (mbits2 << 16) | (mbits3 << 8)
| (mbits4);
IntegratedBitPacking.integratedpack(initoffset, @in, s, @out,
tmpoutpos, mbits1);
tmpoutpos += mbits1;
IntegratedBitPacking.integratedpack(initoffset2, @in, s + BLOCK_SIZE, @out,
tmpoutpos, mbits2);
tmpoutpos += mbits2;
IntegratedBitPacking.integratedpack(initoffset3, @in, s + 2 * BLOCK_SIZE,
@out, tmpoutpos, mbits3);
tmpoutpos += mbits3;
IntegratedBitPacking.integratedpack(initoffset4, @in, s + 3 * BLOCK_SIZE,
@out, tmpoutpos, mbits4);
tmpoutpos += mbits4;
initoffset = @in[s + 3 * BLOCK_SIZE + 31];
}
for (; s < inpos.get() + inlength; s += BLOCK_SIZE)
{
int mbits = Util.maxdiffbits(initoffset, @in, s, BLOCK_SIZE);
@out[tmpoutpos++] = mbits;
IntegratedBitPacking.integratedpack(initoffset, @in, s, @out,
tmpoutpos, mbits);
tmpoutpos += mbits;
initoffset = @in[s + 31];
}
inpos.add(inlength);
outpos.set(tmpoutpos);
}
private void getBestBFromData(int[] @in, int pos, IntWrapper bestb, IntWrapper bestexcept)
{
int mb = Util.maxbits(@in, pos, BLOCK_SIZE);
int mini = 0;
if (mini + 28 < bits[invbits[mb]])
{
mini = bits[invbits[mb]] - 28; // 28 is the max for
}
// exceptions
int besti = bits.Length - 1;
int bestcost = bits[besti] * 4;
int exceptcounter = 0;
for (int i = mini; i < bits.Length - 1; ++i)
{
int tmpcounter = 0;
for (int k = pos; k < BLOCK_SIZE + pos; ++k)
{
if ((int)((uint)@in[k] >> bits[i]) != 0)
{
++tmpcounter;
}
}
if (tmpcounter == BLOCK_SIZE)
{
continue; // no need
}
for (int k = pos, c = 0; k < pos + BLOCK_SIZE; ++k)
{
if ((int)((uint)@in[k] >> bits[i]) != 0)
{
exceptbuffer[tmpcounter + c] = k - pos;
exceptbuffer[c] = (int)((uint)@in[k] >> bits[i]);
++c;
}
}
int thiscost = bits[i] * 4 + S16.estimatecompress(exceptbuffer, 0, 2 * tmpcounter);
if (thiscost <= bestcost)
{
bestcost = thiscost;
besti = i;
exceptcounter = tmpcounter;
}
}
bestb.set(besti);
bestexcept.set(exceptcounter);
}
public void compress(int[] @in, IntWrapper inpos, int inlength, sbyte[] @out, IntWrapper outpos)
{
if (inlength == 0)
{
return;
}
int initoffset = 0;
int outpostmp = outpos.get();
for (int k = inpos.get(); k < inpos.get() + inlength; ++k)
{
long val = (@in[k] - initoffset) & 0xFFFFFFFFL; // To be consistent with unsigned integers in C/C++
initoffset = @in[k];
if (val < (1 << 7))
{
@out[outpostmp++] = (sbyte)(val | (1 << 7));
}
else if (val < (1 << 14))
{
@out[outpostmp++] = (sbyte)extract7bits(0, val);
@out[outpostmp++] = (sbyte)(extract7bitsmaskless(1, (val)) | (1 << 7));
}
else if (val < (1 << 21))
{
@out[outpostmp++] = (sbyte)extract7bits(0, val);
@out[outpostmp++] = (sbyte)extract7bits(1, val);
@out[outpostmp++] = (sbyte)(extract7bitsmaskless(2, (val)) | (1 << 7));
}
else if (val < (1 << 28))
{
@out[outpostmp++] = (sbyte)extract7bits(0, val);
@out[outpostmp++] = (sbyte)extract7bits(1, val);
@out[outpostmp++] = (sbyte)extract7bits(2, val);
@out[outpostmp++] = (sbyte)(extract7bitsmaskless(3, (val)) | (1 << 7));
}
else
{
@out[outpostmp++] = (sbyte)extract7bits(0, val);
@out[outpostmp++] = (sbyte)extract7bits(1, val);
@out[outpostmp++] = (sbyte)extract7bits(2, val);
@out[outpostmp++] = (sbyte)extract7bits(3, val);
@out[outpostmp++] = (sbyte)(extract7bitsmaskless(4, (val)) | (1 << 7));
}
}
outpos.set(outpostmp);
inpos.add(inlength);
}
public void headlessUncompress(int[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos, int num, IntWrapper initvalue)
{
int outlength = Util.greatestMultiple(num, BLOCK_SIZE);
int tmpinpos = inpos.get();
int initoffset = initvalue.get();
int s = outpos.get();
for (; s + BLOCK_SIZE * 4 - 1 < outpos.get() + outlength; s += BLOCK_SIZE * 4)
{
int mbits1 = (int)((uint)@in[tmpinpos] >> 24);
int mbits2 = (int)((uint)@in[tmpinpos] >> 16) & 0xFF;
int mbits3 = (int)((uint)@in[tmpinpos] >> 8) & 0xFF;
int mbits4 = (@in[tmpinpos]) & 0xFF;
++tmpinpos;
IntegratedBitPacking.integratedunpack(initoffset, @in, tmpinpos,
@out, s, mbits1);
tmpinpos += mbits1;
initoffset = @out[s + 31];
IntegratedBitPacking.integratedunpack(initoffset, @in, tmpinpos,
@out, s + BLOCK_SIZE, mbits2);
tmpinpos += mbits2;
initoffset = @out[s + BLOCK_SIZE + 31];
IntegratedBitPacking.integratedunpack(initoffset, @in, tmpinpos,
@out, s + 2 * BLOCK_SIZE, mbits3);
tmpinpos += mbits3;
initoffset = @out[s + 2 * BLOCK_SIZE + 31];
IntegratedBitPacking.integratedunpack(initoffset, @in, tmpinpos,
@out, s + 3 * BLOCK_SIZE, mbits4);
tmpinpos += mbits4;
initoffset = @out[s + 3 * BLOCK_SIZE + 31];
}
for (; s < outpos.get() + outlength; s += BLOCK_SIZE)
{
int mbits = @in[tmpinpos];
++tmpinpos;
IntegratedBitPacking.integratedunpack(initoffset, @in, tmpinpos,
@out, s, mbits);
initoffset = @out[s + 31];
tmpinpos += mbits;
}
outpos.add(outlength);
initvalue.set(initoffset);
inpos.set(tmpinpos);
}
private static int decompressFromSkipTable(object c, int[] compressed, IntWrapper compressedpos, int[] metadata, int blocksize, int[] data)
{
int metapos = 0;
int length = metadata[metapos++];
IntWrapper uncomppos = new IntWrapper();
IntWrapper ival = new IntWrapper();
while (uncomppos.get() < length)
{
int num = blocksize;
if (num > length - uncomppos.get())
{
num = length - uncomppos.get();
}
int location = metadata[metapos++];
// Console.WriteLine("location = "+location);
int initvalue = metadata[metapos++];
int outputlocation = uncomppos.get();
if (location != compressedpos.get())
{
throw new Exception("Bug " + location + " "
+ compressedpos.get() + " codec " + c);
}
if (c is SkippableIntegerCODEC)
{
((SkippableIntegerCODEC)c).headlessUncompress(compressed,
compressedpos, compressed.Length - uncomppos.get(),
data, uncomppos, num);
initvalue = Delta.fastinverseDelta(data, outputlocation, num,
initvalue);
}
else if (c is SkippableIntegratedIntegerCODEC)
{
ival.set(initvalue);
((SkippableIntegratedIntegerCODEC)c).headlessUncompress(
compressed, compressedpos, compressed.Length
- uncomppos.get(), data, uncomppos, num, ival);
}
else
{
throw new Exception("Unrecognized codec " + c);
}
}
return(length);
}
public void compress(int[] inBuf, IntWrapper inPos, int inLen, int[] outBuf, IntWrapper outPos)
{
inLen = inLen - inLen % BLOCK_LENGTH;
if (inLen == 0)
{
return;
}
outBuf[outPos.get()] = inLen;
outPos.increment();
int context = 0;
int[] work = new int[32];
int op = outPos.get();
int ip = inPos.get();
int inPosLast = ip + inLen;
for (; ip < inPosLast; ip += BLOCK_LENGTH)
{
int bits1 = xorMaxBits(inBuf, ip + 0, 32, context);
int bits2 = xorMaxBits(inBuf, ip + 32, 32,
inBuf[ip + 31]);
int bits3 = xorMaxBits(inBuf, ip + 64, 32,
inBuf[ip + 63]);
int bits4 = xorMaxBits(inBuf, ip + 96, 32,
inBuf[ip + 95]);
outBuf[op++] = (bits1 << 24) | (bits2 << 16)
| (bits3 << 8) | (bits4 << 0);
op += xorPack(inBuf, ip + 0, outBuf, op, bits1,
context, work);
op += xorPack(inBuf, ip + 32, outBuf, op, bits2,
inBuf[ip + 31], work);
op += xorPack(inBuf, ip + 64, outBuf, op, bits3,
inBuf[ip + 63], work);
op += xorPack(inBuf, ip + 96, outBuf, op, bits4,
inBuf[ip + 95], work);
context = inBuf[ip + 127];
}
inPos.add(inLen);
outPos.set(op);
}
private void encodePage(int[] @in, IntWrapper inpos, int thissize,
int[] @out, IntWrapper outpos)
{
int tmpoutpos = outpos.get();
int tmpinpos = inpos.get();
IntWrapper bestb = new IntWrapper();
IntWrapper bestexcept = new IntWrapper();
for (int finalinpos = tmpinpos + thissize; tmpinpos + BLOCK_SIZE <= finalinpos; tmpinpos += BLOCK_SIZE)
{
getBestBFromData(@in, tmpinpos, bestb, bestexcept);
int tmpbestb = bestb.get();
int nbrexcept = bestexcept.get();
int exceptsize = 0;
int remember = tmpoutpos;
tmpoutpos++;
if (nbrexcept > 0)
{
int c = 0;
for (int i = 0; i < BLOCK_SIZE; ++i)
{
if ((int)((uint)@in[tmpinpos + i] >> bits[tmpbestb]) != 0)
{
exceptbuffer[c + nbrexcept] = i;
exceptbuffer[c] = (int)((uint)@in[tmpinpos + i] >> bits[tmpbestb]);
++c;
}
}
exceptsize = S9.compress(exceptbuffer, 0,
2 * nbrexcept, @out, tmpoutpos);
tmpoutpos += exceptsize;
}
@out[remember] = tmpbestb | (nbrexcept << 8)
| (exceptsize << 16);
for (int k = 0; k < BLOCK_SIZE; k += 32)
{
BitPacking.fastpack(@in, tmpinpos + k, @out,
tmpoutpos, bits[tmpbestb]);
tmpoutpos += bits[tmpbestb];
}
}
inpos.set(tmpinpos);
outpos.set(tmpoutpos);
}
public void headlessCompress(int[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos)
{
inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
int tmpoutpos = outpos.get();
int s = inpos.get();
for (; s + BLOCK_SIZE * 4 - 1 < inpos.get() + inlength; s += BLOCK_SIZE * 4)
{
int mbits1 = Util.maxbits(@in, s, BLOCK_SIZE);
int mbits2 = Util.maxbits(@in, s + BLOCK_SIZE, BLOCK_SIZE);
int mbits3 = Util.maxbits(@in, s + 2 * BLOCK_SIZE, BLOCK_SIZE);
int mbits4 = Util.maxbits(@in, s + 3 * BLOCK_SIZE, BLOCK_SIZE);
@out[tmpoutpos++] = (mbits1 << 24) | (mbits2 << 16)
| (mbits3 << 8) | (mbits4);
BitPacking.fastpackwithoutmask(@in, s, @out, tmpoutpos,
mbits1);
tmpoutpos += mbits1;
BitPacking.fastpackwithoutmask(@in, s + BLOCK_SIZE, @out,
tmpoutpos, mbits2);
tmpoutpos += mbits2;
BitPacking.fastpackwithoutmask(@in, s + 2 * BLOCK_SIZE, @out,
tmpoutpos, mbits3);
tmpoutpos += mbits3;
BitPacking.fastpackwithoutmask(@in, s + 3 * BLOCK_SIZE, @out,
tmpoutpos, mbits4);
tmpoutpos += mbits4;
}
for (; s < inpos.get() + inlength; s += BLOCK_SIZE)
{
int mbits = Util.maxbits(@in, s, BLOCK_SIZE);
@out[tmpoutpos++] = mbits;
BitPacking.fastpackwithoutmask(@in, s, @out, tmpoutpos,
mbits);
tmpoutpos += mbits;
}
inpos.add(inlength);
outpos.set(tmpoutpos);
}
public void uncompress(int[] inBuf, IntWrapper inPos, int inLen, int[] outBuf, IntWrapper outPos)
{
if (inLen == 0)
{
return;
}
int outLen = inBuf[inPos.get()];
inPos.increment();
int context = 0;
int[] work = new int[32];
int ip = inPos.get();
int op = outPos.get();
int outPosLast = op + outLen;
for (; op < outPosLast; op += BLOCK_LENGTH)
{
int bits1 = (int)((uint)inBuf[ip] >> 24);
int bits2 = (int)((uint)inBuf[ip] >> 16) & 0xFF;
int bits3 = (int)((uint)inBuf[ip] >> 8) & 0xFF;
int bits4 = (int)((uint)inBuf[ip] >> 0) & 0xFF;
++ip;
ip += xorUnpack(inBuf, ip, outBuf, op + 0, bits1,
context, work);
ip += xorUnpack(inBuf, ip, outBuf, op + 32, bits2,
outBuf[op + 31], work);
ip += xorUnpack(inBuf, ip, outBuf, op + 64, bits3,
outBuf[op + 63], work);
ip += xorUnpack(inBuf, ip, outBuf, op + 96, bits4,
outBuf[op + 95], work);
context = outBuf[op + 127];
}
outPos.add(outLen);
inPos.set(ip);
}
public void uncompress(sbyte[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos)
{
int p = inpos.get();
int initoffset = 0;
int finalp = inpos.get() + inlength;
int tmpoutpos = outpos.get();
for (int v = 0; p < finalp; @out[tmpoutpos++] = (initoffset = initoffset + v))
{
v = @in[p] & 0x7F;
if (@in[p] < 0)
{
p += 1;
continue;
}
v = ((@in[p + 1] & 0x7F) << 7) | v;
if (@in[p + 1] < 0)
{
p += 2;
continue;
}
v = ((@in[p + 2] & 0x7F) << 14) | v;
if (@in[p + 2] < 0)
{
p += 3;
continue;
}
v = ((@in[p + 3] & 0x7F) << 21) | v;
if (@in[p + 3] < 0)
{
p += 4;
continue;
}
v = ((@in[p + 4] & 0x7F) << 28) | v;
p += 5;
}
outpos.set(tmpoutpos);
inpos.add(p);
}
public void compress(int[] inBuf, IntWrapper inPos, int inLen, int[] outBuf, IntWrapper outPos)
{
inLen = inLen - inLen % BLOCK_LENGTH;
if (inLen == 0)
{
return;
}
outBuf[outPos.get()] = inLen;
outPos.increment();
DeltaZigzagEncoding.Encoder ctx = new DeltaZigzagEncoding.Encoder(0);
int[] work = new int[BLOCK_LENGTH];
int op = outPos.get();
int ip = inPos.get();
int inPosLast = ip + inLen;
for (; ip < inPosLast; ip += BLOCK_LENGTH)
{
ctx.encodeArray(inBuf, ip, BLOCK_LENGTH, work);
int bits1 = Util.maxbits32(work, 0);
int bits2 = Util.maxbits32(work, 32);
int bits3 = Util.maxbits32(work, 64);
int bits4 = Util.maxbits32(work, 96);
outBuf[op++] = (bits1 << 24) | (bits2 << 16)
| (bits3 << 8) | (bits4 << 0);
op += pack(work, 0, outBuf, op, bits1);
op += pack(work, 32, outBuf, op, bits2);
op += pack(work, 64, outBuf, op, bits3);
op += pack(work, 96, outBuf, op, bits4);
}
inPos.add(inLen);
outPos.set(op);
}
private static int compressWithSkipTable(object c, int[] data, int[] output, IntWrapper outpos, int[] metadata, int blocksize)
{
int metapos = 0;
metadata[metapos++] = data.Length;
IntWrapper inpos = new IntWrapper();
int initvalue = 0;
IntWrapper ival = new IntWrapper(initvalue);
while (inpos.get() < data.Length)
{
metadata[metapos++] = outpos.get();
metadata[metapos++] = initvalue;
if (c is SkippableIntegerCODEC)
{
int size = blocksize > data.Length - inpos.get() ? data.Length
- inpos.get() : blocksize;
initvalue = Delta.delta(data, inpos.get(), size, initvalue);
((SkippableIntegerCODEC)c).headlessCompress(data, inpos,
blocksize, output, outpos);
}
else if (c is SkippableIntegratedIntegerCODEC)
{
ival.set(initvalue);
((SkippableIntegratedIntegerCODEC)c).headlessCompress(data,
inpos, blocksize, output, outpos, ival);
initvalue = ival.get();
}
else
{
throw new Exception("Unrecognized codec " + c);
}
}
return(metapos);
}
/**
* Standard benchmark
*
* @param csvLog
* Writer for CSV log.
* @param c
* the codec
* @param data
* arrays of input data
* @param repeat
* How many times to repeat the test
* @param verbose
* whether to output result on screen
*/
private static void testCodec(StreamWriter csvLog, int sparsity, IntegerCODEC c, int[][] data, int repeat, bool verbose)
{
if (verbose)
{
Console.WriteLine("# " + c);
Console.WriteLine("# bits per int, compress speed (mis), decompression speed (mis) ");
}
int N = data.Length;
int totalSize = 0;
int maxLength = 0;
for (int k = 0; k < N; ++k)
{
totalSize += data[k].Length;
if (data[k].Length > maxLength)
{
maxLength = data[k].Length;
}
}
// 4x + 1024 to account for the possibility of some negative
// compression.
int[] compressBuffer = new int[4 * maxLength + 1024];
int[] decompressBuffer = new int[maxLength + 1024];
// These variables hold time in microseconds (10^-6).
long compressTime = 0;
long decompressTime = 0;
int size = 0;
IntWrapper inpos = new IntWrapper();
IntWrapper outpos = new IntWrapper();
for (int r = 0; r < repeat; ++r)
{
size = 0;
for (int k = 0; k < N; ++k)
{
int[] backupdata = Arrays.copyOf(data[k],
data[k].Length);
// compress data.
long beforeCompress = Port.System.nanoTime() / 1000;
inpos.set(1);
outpos.set(0);
if (!(c is IntegratedIntegerCODEC))
{
Delta.delta(backupdata);
}
c.compress(backupdata, inpos, backupdata.Length
- inpos.get(), compressBuffer, outpos);
long afterCompress = Port.System.nanoTime() / 1000;
// measure time of compression.
compressTime += afterCompress - beforeCompress;
int thiscompsize = outpos.get() + 1;
size += thiscompsize;
// extract (uncompress) data
long beforeDecompress = Port.System.nanoTime() / 1000;
inpos.set(0);
outpos.set(1);
decompressBuffer[0] = backupdata[0];
c.uncompress(compressBuffer, inpos,
thiscompsize - 1, decompressBuffer,
outpos);
if (!(c is IntegratedIntegerCODEC))
{
Delta.fastinverseDelta(decompressBuffer);
}
long afterDecompress = Port.System.nanoTime() / 1000;
// measure time of extraction (uncompression).
decompressTime += afterDecompress
- beforeDecompress;
if (outpos.get() != data[k].Length)
{
throw new Exception(
"we have a bug (diff length) "
+ c + " expected "
+ data[k].Length
+ " got "
+ outpos.get());
}
// verify: compare original array with
// compressed and
// uncompressed.
for (int m = 0; m < outpos.get(); ++m)
{
if (decompressBuffer[m] != data[k][m])
{
throw new Exception(
"we have a bug (actual difference), expected "
+ data[k][m]
+ " found "
+ decompressBuffer[m]
+ " at " + m);
}
}
}
}
if (verbose)
{
double bitsPerInt = size * 32.0 / totalSize;
long compressSpeed = totalSize * repeat / (compressTime);
long decompressSpeed = totalSize * repeat / (decompressTime);
Console.WriteLine("\t{0:0.00}\t{1}\t{2}", bitsPerInt, compressSpeed, decompressSpeed);
csvLog.WriteLine("\"{0}\",{1},{2:0.00},{3},{4}", c, sparsity, bitsPerInt, compressSpeed, decompressSpeed);
}
}
public void headlessCompress(int[] @in, IntWrapper inpos, int inlength, int[] @out, IntWrapper outpos, IntWrapper initvalue)
{
if (inlength == 0)
{
return;
}
int initoffset = initvalue.get();
initvalue.set(@in[inpos.get() + inlength - 1]);
ByteBuffer buf = ByteBuffer.allocateDirect(inlength * 8);
buf.order(ByteOrder.LITTLE_ENDIAN);
for (int k = inpos.get(); k < inpos.get() + inlength; ++k)
{
long val = (@in[k] - initoffset) & 0xFFFFFFFFL; // To be consistent with unsigned integers in C/C++
initoffset = @in[k];
if (val < (1 << 7))
{
buf.put((sbyte)(val | (1 << 7)));
}
else if (val < (1 << 14))
{
buf.put((sbyte)extract7bits(0, val));
buf.put((sbyte)(extract7bitsmaskless(1, (val)) | (1 << 7)));
}
else if (val < (1 << 21))
{
buf.put((sbyte)extract7bits(0, val));
buf.put((sbyte)extract7bits(1, val));
buf.put((sbyte)(extract7bitsmaskless(2, (val)) | (1 << 7)));
}
else if (val < (1 << 28))
{
buf.put((sbyte)extract7bits(0, val));
buf.put((sbyte)extract7bits(1, val));
buf.put((sbyte)extract7bits(2, val));
buf.put((sbyte)(extract7bitsmaskless(3, (val)) | (1 << 7)));
}
else
{
buf.put((sbyte)extract7bits(0, val));
buf.put((sbyte)extract7bits(1, val));
buf.put((sbyte)extract7bits(2, val));
buf.put((sbyte)extract7bits(3, val));
buf.put((sbyte)(extract7bitsmaskless(4, (val)) | (1 << 7)));
}
}
while (buf.position() % 4 != 0)
{
buf.put((sbyte)0);
}
int length = buf.position();
buf.flip();
IntBuffer ibuf = buf.asIntBuffer();
ibuf.get(@out, outpos.get(), length / 4);
outpos.add(length / 4);
inpos.add(inlength);
}