public void checkXorBinaryPacking2()
{
IntegerCODEC c = new IntegratedComposition(new XorBinaryPacking(),
new IntegratedVariableByte());
testUnsorted(c);
}
public void checkXorBinaryPacking1()
{
IntegerCODEC c = new IntegratedComposition(new XorBinaryPacking(),
new IntegratedVariableByte());
testZeroInZeroOut(c);
}
public void checkXorBinaryPacking3()
{
IntegerCODEC c = new IntegratedComposition(new XorBinaryPacking(), new IntegratedVariableByte());
IntegerCODEC co = new IntegratedComposition(new XorBinaryPacking(), new IntegratedVariableByte());
test(c, co, 5, 10);
test(c, co, 5, 14);
test(c, co, 2, 18);
}
public void testIntegratedComposition()
{
IntegratedComposition c = new IntegratedComposition(new IntegratedBinaryPacking(), new IntegratedVariableByte());
testBoundary(c);
}
public void basicExample()
{
int[] data = new int[2342351];
Console.WriteLine("Compressing " + data.Length + " integers in one go");
// data should be sorted for best
// results
for (int k = 0; k < data.Length; ++k)
{
data[k] = k;
}
// Very important: the data is in sorted order!!! If not, you
// will get very poor compression with IntegratedBinaryPacking,
// you should use another CODEC.
// next we compose a CODEC. Most of the processing
// will be done with binary packing, and leftovers will
// be processed using variable byte
IntegratedIntegerCODEC codec = new IntegratedComposition(new IntegratedBinaryPacking(),
new IntegratedVariableByte());
// output vector should be large enough...
int[] compressed = new int[data.Length + 1024];
// compressed might not be large enough in some cases
// if you get java.lang.ArrayIndexOutOfBoundsException, try
// allocating more memory
/**
*
* compressing
*
*/
IntWrapper inputoffset = new IntWrapper(0);
IntWrapper outputoffset = new IntWrapper(0);
codec.compress(data, inputoffset, data.Length, compressed, outputoffset);
// got it!
// inputoffset should be at data.Length but outputoffset tells
// us where we are...
Console.WriteLine(
"compressed from " + data.Length * 4 / 1024 + "KB to " + outputoffset.intValue() * 4 / 1024 + "KB");
// we can repack the data: (optional)
compressed = Arrays.copyOf(compressed, outputoffset.intValue());
/**
*
* now uncompressing
*
* This assumes that we otherwise know how many integers have been
* compressed. See basicExampleHeadless for a more general case.
*/
int[] recovered = new int[data.Length];
IntWrapper recoffset = new IntWrapper(0);
codec.uncompress(compressed, new IntWrapper(0), compressed.Length, recovered, recoffset);
if (Arrays.equals(data, recovered))
{
Console.WriteLine("data is recovered without loss");
}
else
{
throw new Exception("bug"); // could use assert
}
Console.WriteLine();
}
public void advancedExample()
{
const int TotalSize = 2342351; // some arbitrary number
const int ChunkSize = 16384; // size of each chunk, choose a multiple of 128
Console.WriteLine("Compressing " + TotalSize + " integers using chunks of " + ChunkSize + " integers ("
+ ChunkSize * 4 / 1024 + "KB)");
Console.WriteLine("(It is often better for applications to work in chunks fitting in CPU cache.)");
int[] data = new int[TotalSize];
// data should be sorted for best
// results
for (int k = 0; k < data.Length; ++k)
{
data[k] = k;
}
// next we compose a CODEC. Most of the processing
// will be done with binary packing, and leftovers will
// be processed using variable byte, using variable byte
// only for the last chunk!
IntegratedIntegerCODEC regularcodec = new IntegratedBinaryPacking();
IntegratedVariableByte ivb = new IntegratedVariableByte();
IntegratedIntegerCODEC lastcodec = new IntegratedComposition(regularcodec, ivb);
// output vector should be large enough...
int[] compressed = new int[TotalSize + 1024];
/**
*
* compressing
*
*/
IntWrapper inputoffset = new IntWrapper(0);
IntWrapper outputoffset = new IntWrapper(0);
for (int k = 0; k < TotalSize / ChunkSize; ++k)
{
regularcodec.compress(data, inputoffset, ChunkSize, compressed, outputoffset);
}
lastcodec.compress(data, inputoffset, TotalSize % ChunkSize, compressed, outputoffset);
// got it!
// inputoffset should be at data.Length but outputoffset tells
// us where we are...
Console.WriteLine(
"compressed from " + data.Length * 4 / 1024 + "KB to " + outputoffset.intValue() * 4 / 1024 + "KB");
// we can repack the data:
compressed = Arrays.copyOf(compressed, outputoffset.intValue());
/**
*
* now uncompressing
*
* We are *not* assuming that the original array length is known,
* however we assume that the chunk size (ChunkSize) is known.
*
*/
int[] recovered = new int[ChunkSize];
IntWrapper compoff = new IntWrapper(0);
IntWrapper recoffset;
int currentpos = 0;
while (compoff.get() < compressed.Length)
{
recoffset = new IntWrapper(0);
regularcodec.uncompress(compressed, compoff, compressed.Length - compoff.get(), recovered, recoffset);
if (recoffset.get() < ChunkSize)
{// last chunk detected
ivb.uncompress(compressed, compoff, compressed.Length - compoff.get(), recovered, recoffset);
}
for (int i = 0; i < recoffset.get(); ++i)
{
if (data[currentpos + i] != recovered[i])
{
throw new Exception("bug"); // could use assert
}
}
currentpos += recoffset.get();
}
Console.WriteLine("data is recovered without loss");
Console.WriteLine();
}
