/**
* Check that compress and uncompress keep original array.
*
* @param codec CODEC to test.
* @param orig original integers
*/
public static void assertSymmetry(IntegerCODEC codec, params int[] orig)
{
// There are some cases that compressed array is bigger than original
// array. So output array for compress must be larger.
//
// Example:
// - VariableByte compresses an array like [ -1 ].
// - Composition compresses a short array.
const int EXTEND = 1;
int[] compressed = new int[orig.Length + EXTEND];
IntWrapper c_inpos = new IntWrapper(0);
IntWrapper c_outpos = new IntWrapper(0);
codec.compress(orig, c_inpos, orig.Length, compressed,
c_outpos);
Assert2.assertTrue(c_outpos.get() <= orig.Length + EXTEND);
// Uncompress an array.
int[] uncompressed = new int[orig.Length];
IntWrapper u_inpos = new IntWrapper(0);
IntWrapper u_outpos = new IntWrapper(0);
codec.uncompress(compressed, u_inpos, c_outpos.get(),
uncompressed, u_outpos);
// Compare between uncompressed and orig arrays.
int[] target = Arrays.copyOf(uncompressed, u_outpos.get());
Assert2.assertArrayEquals(orig, target);
}
private static void compressAndUncompress(int length, IntegerCODEC c)
{
// Initialize array.
int[] source = new int[length];
for (int i = 0; i < source.Length; ++i)
{
source[i] = i;
}
// Compress an array.
int[] compressed = new int[length];
IntWrapper c_inpos = new IntWrapper(0);
IntWrapper c_outpos = new IntWrapper(0);
c.compress(source, c_inpos, source.Length, compressed, c_outpos);
Assert2.assertTrue(c_outpos.get() <= length);
// Uncompress an array.
int[] uncompressed = new int[length];
IntWrapper u_inpos = new IntWrapper(0);
IntWrapper u_outpos = new IntWrapper(0);
c.uncompress(compressed, u_inpos, c_outpos.get(), uncompressed,
u_outpos);
// Compare between uncompressed and original arrays.
int[] target = Arrays.copyOf(uncompressed, u_outpos.get());
if (!Arrays.equals(source, target))
{
Console.WriteLine("problem with length = " + length + " and " + c);
Console.WriteLine(Arrays.toString(source));
Console.WriteLine(Arrays.toString(target));
}
Assert2.assertArrayEquals(source, target);
}
public static int[] uncompress(IntegerCODEC codec, int[] data, int len)
{
int[] outBuf = new int[len + 1024];
IntWrapper inPos = new IntWrapper();
IntWrapper outPos = new IntWrapper();
codec.uncompress(data, inPos, data.Length, outBuf, outPos);
return(Arrays.copyOf(outBuf, outPos.get()));
}
private static int decompress(PerformanceLogger logger, IntegerCODEC codec, int[] src, int srcLen, int[] dst)
{
IntWrapper inpos = new IntWrapper();
IntWrapper outpos = new IntWrapper();
logger.decompressionTimer.start();
codec.uncompress(src, inpos, srcLen, dst, outpos);
logger.decompressionTimer.end();
return(outpos.get());
}
private static void testSpurious(IntegerCODEC c)
{
int[] x = new int[1024];
int[] y = new int[0];
IntWrapper i0 = new IntWrapper(0);
IntWrapper i1 = new IntWrapper(0);
for (int inlength = 0; inlength < 32; ++inlength)
{
c.compress(x, i0, inlength, y, i1);
Assert2.assertEquals(0, i1.intValue());
}
}
private static int compress(PerformanceLogger logger, IntegerCODEC codec, int[] src, int[] dst)
{
IntWrapper inpos = new IntWrapper();
IntWrapper outpos = new IntWrapper();
logger.compressionTimer.start();
codec.compress(src, inpos, src.Length, dst, outpos);
logger.compressionTimer.end();
int outSize = outpos.get();
logger.addOriginalSize(src.Length);
logger.addCompressedSize(outSize);
return(outSize);
}
private static void testCodec(IntegerCODEC c, IntegerCODEC co, int[][] data, int max)
{
int N = data.Length;
int maxlength = 0;
for (int k = 0; k < N; ++k)
{
if (data[k].Length > maxlength)
{
maxlength = data[k].Length;
}
}
int[] buffer = new int[maxlength + 1024];
int[] dataout = new int[4 * maxlength + 1024];
// 4x + 1024 to account for the possibility of some negative
// compression.
IntWrapper inpos = new IntWrapper();
IntWrapper outpos = new IntWrapper();
for (int k = 0; k < N; ++k)
{
int[] backupdata = Arrays.copyOf(data[k], data[k].Length);
inpos.set(1);
outpos.set(0);
if (!(c is IntegratedIntegerCODEC))
{
Delta.delta(backupdata);
}
c.compress(backupdata, inpos, backupdata.Length - inpos.get(), dataout, outpos);
int thiscompsize = outpos.get() + 1;
inpos.set(0);
outpos.set(1);
buffer[0] = backupdata[0];
co.uncompress(dataout, inpos, thiscompsize - 1, buffer, outpos);
if (!(c is IntegratedIntegerCODEC))
{
Delta.fastinverseDelta(buffer);
}
// Check assertions.
Assert2.assertEquals(outpos.get(), data[k].Length); //"length is not match"
int[] bufferCutout = Arrays.copyOf(buffer, outpos.get());
Assert2.assertArrayEquals(data[k], bufferCutout); //"failed to reconstruct original data"
}
}
private static void test(IntegerCODEC c, IntegerCODEC co, int N, int nbr)
{
ClusteredDataGenerator cdg = new ClusteredDataGenerator();
for (int sparsity = 1; sparsity < 31 - nbr; sparsity += 4)
{
int[][] data = new int[N][];
int max = (1 << (nbr + 9));
for (int k = 0; k < N; ++k)
{
data[k] = cdg.generateClustered((1 << nbr), max);
}
testCodec(c, co, data, max);
}
}
private static void testZeroInZeroOut(IntegerCODEC c)
{
int[] x = new int[0];
int[] y = new int[0];
IntWrapper i0 = new IntWrapper(0);
IntWrapper i1 = new IntWrapper(0);
c.compress(x, i0, 0, y, i1);
Assert2.assertEquals(0, i1.intValue());
int[] @out = new int[0];
IntWrapper outpos = new IntWrapper(0);
c.uncompress(y, i1, 0, @out, outpos);
Assert2.assertEquals(0, outpos.intValue());
}
private void testUnsorted3(IntegerCODEC codec)
{
int[] data = new int[128];
data[127] = -1;
int[] compressed = new int[1024];
IntWrapper inputoffset = new IntWrapper(0);
IntWrapper outputoffset = new IntWrapper(0);
codec.compress(data, inputoffset, data.Length, compressed, outputoffset);
// we can repack the data: (optional)
compressed = Arrays.copyOf(compressed, outputoffset.intValue());
int[] recovered = new int[128];
IntWrapper recoffset = new IntWrapper(0);
codec.uncompress(compressed, new IntWrapper(0), compressed.Length,
recovered, recoffset);
Assert2.assertArrayEquals(data, recovered);
}
private static void checkArray(int[] expected, int[] actualArray,
int actualLen, IntegerCODEC codec)
{
if (actualLen != expected.Length)
{
throw new Exception("Length mismatch:" + " expected=" + expected.Length + " actual=" + actualLen + " codec=" + codec);
}
for (int i = 0; i < expected.Length; ++i)
{
if (actualArray[i] != expected[i])
{
throw new Exception("Value mismatch: "
+ " where=" + i + " expected="
+ expected[i] + " actual="
+ actualArray[i] + " codec="
+ codec);
}
}
}
public void testUnsorted(IntegerCODEC codec)
{
int[] lengths = { 133, 1026, 1333333 };
foreach (int N in lengths)
{
int[] data = new int[N];
// initialize the data (most will be small)
for (int k = 0; k < N; k += 1)
{
data[k] = 3;
}
// throw some larger values
for (int k = 0; k < N; k += 5)
{
data[k] = 100;
}
for (int k = 0; k < N; k += 533)
{
data[k] = 10000;
}
data[5] = -311;
// could need more compressing
int[] compressed = new int[(int)Math.Ceiling(N * 1.01) + 1024];
IntWrapper inputoffset = new IntWrapper(0);
IntWrapper outputoffset = new IntWrapper(0);
codec.compress(data, inputoffset, data.Length, compressed,
outputoffset);
// we can repack the data: (optional)
compressed = Arrays.copyOf(compressed, outputoffset.intValue());
int[] recovered = new int[N];
IntWrapper recoffset = new IntWrapper(0);
codec.uncompress(compressed, new IntWrapper(0), compressed.Length,
recovered, recoffset);
Assert2.assertArrayEquals(data, recovered);
}
}
private static void benchmark(StreamWriter csvWriter, string dataName, string codecName, IntegerCODEC codec, int[][] data, int repeat)
{
PerformanceLogger logger = new PerformanceLogger();
int maxLen = getMaxLen(data);
int[] compressBuffer = new int[4 * maxLen + 1024];
int[] decompressBuffer = new int[maxLen];
for (int i = 0; i < repeat; ++i)
{
foreach (int[] array in data)
{
int compSize = compress(logger, codec, array, compressBuffer);
int decompSize = decompress(logger, codec, compressBuffer, compSize, decompressBuffer);
checkArray(array, decompressBuffer, decompSize, codec);
}
}
if (csvWriter != null)
{
csvWriter.WriteLine("\"{0}\",\"{1}\",{2:0.00},{3:0},{4:0}", dataName, codecName, logger.getBitPerInt(), logger.getCompressSpeed(), logger.getDecompressSpeed());
}
}
private static void testBoundary(IntegerCODEC c)
{
around32(c);
around128(c);
around256(c);
}
private static void around256(IntegerCODEC c)
{
compressAndUncompress(255, c);
compressAndUncompress(256, c);
compressAndUncompress(257, c);
}
private static void around128(IntegerCODEC c)
{
compressAndUncompress(127, c);
compressAndUncompress(128, c);
compressAndUncompress(129, c);
}
private static void around32(IntegerCODEC c)
{
compressAndUncompress(31, c);
compressAndUncompress(32, c);
compressAndUncompress(33, c);
}
/**
* Standard benchmark
*
* @param csvLog
* Writer for CSV log.
* @param c
* the codec
* @param data
* arrays of input data
* @param repeat
* How many times to repeat the test
* @param verbose
* whether to output result on screen
*/
private static void testCodec(StreamWriter csvLog, int sparsity, IntegerCODEC c, int[][] data, int repeat, bool verbose)
{
if (verbose)
{
Console.WriteLine("# " + c);
Console.WriteLine("# bits per int, compress speed (mis), decompression speed (mis) ");
}
int N = data.Length;
int totalSize = 0;
int maxLength = 0;
for (int k = 0; k < N; ++k)
{
totalSize += data[k].Length;
if (data[k].Length > maxLength)
{
maxLength = data[k].Length;
}
}
// 4x + 1024 to account for the possibility of some negative
// compression.
int[] compressBuffer = new int[4 * maxLength + 1024];
int[] decompressBuffer = new int[maxLength + 1024];
// These variables hold time in microseconds (10^-6).
long compressTime = 0;
long decompressTime = 0;
int size = 0;
IntWrapper inpos = new IntWrapper();
IntWrapper outpos = new IntWrapper();
for (int r = 0; r < repeat; ++r)
{
size = 0;
for (int k = 0; k < N; ++k)
{
int[] backupdata = Arrays.copyOf(data[k],
data[k].Length);
// compress data.
long beforeCompress = Port.System.nanoTime() / 1000;
inpos.set(1);
outpos.set(0);
if (!(c is IntegratedIntegerCODEC))
{
Delta.delta(backupdata);
}
c.compress(backupdata, inpos, backupdata.Length
- inpos.get(), compressBuffer, outpos);
long afterCompress = Port.System.nanoTime() / 1000;
// measure time of compression.
compressTime += afterCompress - beforeCompress;
int thiscompsize = outpos.get() + 1;
size += thiscompsize;
// extract (uncompress) data
long beforeDecompress = Port.System.nanoTime() / 1000;
inpos.set(0);
outpos.set(1);
decompressBuffer[0] = backupdata[0];
c.uncompress(compressBuffer, inpos,
thiscompsize - 1, decompressBuffer,
outpos);
if (!(c is IntegratedIntegerCODEC))
{
Delta.fastinverseDelta(decompressBuffer);
}
long afterDecompress = Port.System.nanoTime() / 1000;
// measure time of extraction (uncompression).
decompressTime += afterDecompress
- beforeDecompress;
if (outpos.get() != data[k].Length)
{
throw new Exception(
"we have a bug (diff length) "
+ c + " expected "
+ data[k].Length
+ " got "
+ outpos.get());
}
// verify: compare original array with
// compressed and
// uncompressed.
for (int m = 0; m < outpos.get(); ++m)
{
if (decompressBuffer[m] != data[k][m])
{
throw new Exception(
"we have a bug (actual difference), expected "
+ data[k][m]
+ " found "
+ decompressBuffer[m]
+ " at " + m);
}
}
}
}
if (verbose)
{
double bitsPerInt = size * 32.0 / totalSize;
long compressSpeed = totalSize * repeat / (compressTime);
long decompressSpeed = totalSize * repeat / (decompressTime);
Console.WriteLine("\t{0:0.00}\t{1}\t{2}", bitsPerInt, compressSpeed, decompressSpeed);
csvLog.WriteLine("\"{0}\",{1},{2:0.00},{3},{4}", c, sparsity, bitsPerInt, compressSpeed, decompressSpeed);
}
}
/**
* Compose a scheme from a first one (f1) and a second one (f2). The
* first one is called first and then the second one tries to compress
* whatever remains from the first run.
*
* By convention, the first scheme should be such that if, during
* decoding, a 32-bit zero is first encountered, then there is no
* output.
*
* @param f1
* first codec
* @param f2
* second codec
*/
public Composition(IntegerCODEC f1, IntegerCODEC f2)
{
F1 = f1;
F2 = f2;
}
