public void checkXorBinaryPacking2() { IntegerCODEC c = new IntegratedComposition(new XorBinaryPacking(), new IntegratedVariableByte()); testUnsorted(c); }
public void checkXorBinaryPacking1() { IntegerCODEC c = new IntegratedComposition(new XorBinaryPacking(), new IntegratedVariableByte()); testZeroInZeroOut(c); }
public void checkXorBinaryPacking3() { IntegerCODEC c = new IntegratedComposition(new XorBinaryPacking(), new IntegratedVariableByte()); IntegerCODEC co = new IntegratedComposition(new XorBinaryPacking(), new IntegratedVariableByte()); test(c, co, 5, 10); test(c, co, 5, 14); test(c, co, 2, 18); }
public void testIntegratedComposition() { IntegratedComposition c = new IntegratedComposition(new IntegratedBinaryPacking(), new IntegratedVariableByte()); testBoundary(c); }
public void basicExample() { int[] data = new int[2342351]; Console.WriteLine("Compressing " + data.Length + " integers in one go"); // data should be sorted for best // results for (int k = 0; k < data.Length; ++k) { data[k] = k; } // Very important: the data is in sorted order!!! If not, you // will get very poor compression with IntegratedBinaryPacking, // you should use another CODEC. // next we compose a CODEC. Most of the processing // will be done with binary packing, and leftovers will // be processed using variable byte IntegratedIntegerCODEC codec = new IntegratedComposition(new IntegratedBinaryPacking(), new IntegratedVariableByte()); // output vector should be large enough... int[] compressed = new int[data.Length + 1024]; // compressed might not be large enough in some cases // if you get java.lang.ArrayIndexOutOfBoundsException, try // allocating more memory /** * * compressing * */ IntWrapper inputoffset = new IntWrapper(0); IntWrapper outputoffset = new IntWrapper(0); codec.compress(data, inputoffset, data.Length, compressed, outputoffset); // got it! // inputoffset should be at data.Length but outputoffset tells // us where we are... Console.WriteLine( "compressed from " + data.Length * 4 / 1024 + "KB to " + outputoffset.intValue() * 4 / 1024 + "KB"); // we can repack the data: (optional) compressed = Arrays.copyOf(compressed, outputoffset.intValue()); /** * * now uncompressing * * This assumes that we otherwise know how many integers have been * compressed. See basicExampleHeadless for a more general case. */ int[] recovered = new int[data.Length]; IntWrapper recoffset = new IntWrapper(0); codec.uncompress(compressed, new IntWrapper(0), compressed.Length, recovered, recoffset); if (Arrays.equals(data, recovered)) { Console.WriteLine("data is recovered without loss"); } else { throw new Exception("bug"); // could use assert } Console.WriteLine(); }
public void advancedExample() { const int TotalSize = 2342351; // some arbitrary number const int ChunkSize = 16384; // size of each chunk, choose a multiple of 128 Console.WriteLine("Compressing " + TotalSize + " integers using chunks of " + ChunkSize + " integers (" + ChunkSize * 4 / 1024 + "KB)"); Console.WriteLine("(It is often better for applications to work in chunks fitting in CPU cache.)"); int[] data = new int[TotalSize]; // data should be sorted for best // results for (int k = 0; k < data.Length; ++k) { data[k] = k; } // next we compose a CODEC. Most of the processing // will be done with binary packing, and leftovers will // be processed using variable byte, using variable byte // only for the last chunk! IntegratedIntegerCODEC regularcodec = new IntegratedBinaryPacking(); IntegratedVariableByte ivb = new IntegratedVariableByte(); IntegratedIntegerCODEC lastcodec = new IntegratedComposition(regularcodec, ivb); // output vector should be large enough... int[] compressed = new int[TotalSize + 1024]; /** * * compressing * */ IntWrapper inputoffset = new IntWrapper(0); IntWrapper outputoffset = new IntWrapper(0); for (int k = 0; k < TotalSize / ChunkSize; ++k) { regularcodec.compress(data, inputoffset, ChunkSize, compressed, outputoffset); } lastcodec.compress(data, inputoffset, TotalSize % ChunkSize, compressed, outputoffset); // got it! // inputoffset should be at data.Length but outputoffset tells // us where we are... Console.WriteLine( "compressed from " + data.Length * 4 / 1024 + "KB to " + outputoffset.intValue() * 4 / 1024 + "KB"); // we can repack the data: compressed = Arrays.copyOf(compressed, outputoffset.intValue()); /** * * now uncompressing * * We are *not* assuming that the original array length is known, * however we assume that the chunk size (ChunkSize) is known. * */ int[] recovered = new int[ChunkSize]; IntWrapper compoff = new IntWrapper(0); IntWrapper recoffset; int currentpos = 0; while (compoff.get() < compressed.Length) { recoffset = new IntWrapper(0); regularcodec.uncompress(compressed, compoff, compressed.Length - compoff.get(), recovered, recoffset); if (recoffset.get() < ChunkSize) {// last chunk detected ivb.uncompress(compressed, compoff, compressed.Length - compoff.get(), recovered, recoffset); } for (int i = 0; i < recoffset.get(); ++i) { if (data[currentpos + i] != recovered[i]) { throw new Exception("bug"); // could use assert } } currentpos += recoffset.get(); } Console.WriteLine("data is recovered without loss"); Console.WriteLine(); }