public void runSeekTest(CompressionCodec codec)
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
const int COUNT = 16384;
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 500, codec, collect), 1);
TestInStream.PositionCollector[] positions =
new TestInStream.PositionCollector[COUNT];
for (int i = 0; i < COUNT; ++i)
{
positions[i] = new TestInStream.PositionCollector();
@out.getPosition(positions[i]);
// test runs, non-runs
if (i < COUNT / 2)
{
@out.write(i & 1);
}
else
{
@out.write((i / 3) & 1);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
codec, 500), 1);
#pragma warning restore 612
for (int i = 0; i < COUNT; ++i)
{
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
}
for (int i = COUNT - 1; i >= 0; --i)
{
@in.seek(positions[i]);
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
}
}
public void testCompressedSeek()
{
CompressionCodec codec = new SnappyCodec();
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
RunLengthByteWriter @out = new RunLengthByteWriter(new OutStream("test", 500,
codec, collect));
TestInStream.PositionCollector[] positions =
new TestInStream.PositionCollector[2048];
for (int i = 0; i < 2048; ++i)
{
positions[i] = new TestInStream.PositionCollector();
@out.getPosition(positions[i]);
if (i < 1024)
{
@out.write((byte)(i / 4));
}
else
{
@out.write((byte)i);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
RunLengthByteReader @in = new RunLengthByteReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(), codec, 500));
for (int i = 0; i < 2048; ++i)
{
int x = @in.next() & 0xff;
if (i < 1024)
{
Assert.Equal((i / 4) & 0xff, x);
}
else
{
Assert.Equal(i & 0xff, x);
}
}
for (int i = 2047; i >= 0; --i)
{
@in.seek(positions[i]);
int x = @in.next() & 0xff;
if (i < 1024)
{
Assert.Equal((i / 4) & 0xff, x);
}
else
{
Assert.Equal(i & 0xff, x);
}
}
}
public void runSeekTest(CompressionCodec codec)
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
const int COUNT = 16384;
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 500, codec, collect), 1);
TestInStream.PositionCollector[] positions =
new TestInStream.PositionCollector[COUNT];
for (int i = 0; i < COUNT; ++i)
{
positions[i] = new TestInStream.PositionCollector();
@out.getPosition(positions[i]);
// test runs, non-runs
if (i < COUNT / 2)
{
@out.write(i & 1);
}
else
{
@out.write((i / 3) & 1);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
codec, 500), 1);
#pragma warning restore 612
for (int i = 0; i < COUNT; ++i)
{
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
}
for (int i = COUNT - 1; i >= 0; --i)
{
@in.seek(positions[i]);
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
}
}
private void runTest(int numBits)
{
long[] inp = new long[SIZE];
for (int i = 0; i < SIZE; i++)
{
long val = 0;
if (numBits <= 32)
{
if (numBits == 1)
{
val = -1 * rand.Next(2);
}
else
{
int max = (numBits == 32) ? Int32.MaxValue : (int)Math.Pow(2, numBits - 1);
val = rand.Next(max);
}
}
else
{
val = nextLong(rand, (long)Math.Pow(2, numBits - 2));
}
if (val % 2 == 0)
{
val = -val;
}
inp[i] = val;
}
long[] deltaEncoded = deltaEncode(inp);
long minInput = deltaEncoded.Min();
long maxInput = deltaEncoded.Max();
long rangeInput = maxInput - minInput;
SerializationUtils utils = new SerializationUtils();
int fixedWidth = utils.findClosestNumBits(rangeInput);
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
OutStream output = new OutStream("test", SIZE, null, collect);
utils.writeInts(deltaEncoded, 0, deltaEncoded.Length, fixedWidth, output);
output.Flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
long[] buff = new long[SIZE];
#pragma warning disable 612
utils.readInts(buff, 0, SIZE, fixedWidth, InStream.create(null, "test", new ByteBuffer[] { inBuf },
new long[] { 0 }, inBuf.remaining(), null, SIZE));
#pragma warning restore 612
for (int i = 0; i < SIZE; i++)
{
buff[i] = utils.zigzagDecode(buff[i]);
}
Assert.Equal(numBits, fixedWidth);
Assert.Equal(inp, buff);
}
public void testSkips()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
RunLengthIntegerWriterV2 @out = new RunLengthIntegerWriterV2(
new OutStream("test", 100, null, collect), true);
for (int i = 0; i < 2048; ++i)
{
if (i < 1024)
{
@out.write(i);
}
else
{
@out.write(256 * i);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
RunLengthIntegerReaderV2 @in =
new RunLengthIntegerReaderV2(InStream.create(null, "test",
new ByteBuffer[] { inBuf },
new long[] { 0 },
inBuf.remaining(),
null, 100), true, false);
#pragma warning restore 612
for (int i = 0; i < 2048; i += 10)
{
int x = (int)@in.next();
if (i < 1024)
{
Assert.Equal(i, x);
}
else
{
Assert.Equal(256 * i, x);
}
if (i < 2038)
{
@in.skip(9);
}
@in.skip(0);
}
}
public void testSkips()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 100, null, collect), 1);
const int COUNT = 16384;
for (int i = 0; i < COUNT; ++i)
{
if (i < COUNT / 2)
{
@out.write(i & 1);
}
else
{
@out.write((i / 3) & 1);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create
(null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
null, 100), 1);
#pragma warning restore 612
for (int i = 0; i < COUNT; i += 5)
{
int x = (int)@in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
if (i < COUNT - 5)
{
@in.skip(4);
}
@in.skip(0);
}
}
public void testSkips()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
RunLengthByteWriter @out = new RunLengthByteWriter(new OutStream("test", 100,
null, collect));
for (int i = 0; i < 2048; ++i)
{
if (i < 1024)
{
@out.write((byte)(i / 16));
}
else
{
@out.write((byte)i);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
RunLengthByteReader @in = new RunLengthByteReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(), null, 100));
#pragma warning restore 612
for (int i = 0; i < 2048; i += 10)
{
int x = @in.next() & 0xff;
if (i < 1024)
{
Assert.Equal((i / 16) & 0xff, x);
}
else
{
Assert.Equal(i & 0xff, x);
}
if (i < 2038)
{
@in.skip(9);
}
@in.skip(0);
}
}
public void testBiggerItems()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
const int COUNT = 16384;
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 500, null, collect), 3);
for (int i = 0; i < COUNT; ++i)
{
// test runs, non-runs
if (i < COUNT / 2)
{
@out.write(i & 7);
}
else
{
@out.write((i / 3) & 7);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
null, 500), 3);
#pragma warning restore 612
for (int i = 0; i < COUNT; ++i)
{
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 7, x);
}
else
{
Assert.Equal((i / 3) & 7, x);
}
}
}
public void runSeekTest(CompressionCodec codec)
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
RunLengthIntegerWriter @out = new RunLengthIntegerWriter(
new OutStream("test", 1000, codec, collect), true);
TestInStream.PositionCollector[] positions =
new TestInStream.PositionCollector[4096];
Random random = new Random(99);
int[] junk = new int[2048];
for (int i = 0; i < junk.Length; ++i)
{
junk[i] = random.Next();
}
for (int i = 0; i < 4096; ++i)
{
positions[i] = new TestInStream.PositionCollector();
@out.getPosition(positions[i]);
// test runs, incrementing runs, non-runs
if (i < 1024)
{
@out.write(i / 4);
}
else if (i < 2048)
{
@out.write(2 * i);
}
else
{
@out.write(junk[i - 2048]);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
RunLengthIntegerReader @in = new RunLengthIntegerReader(InStream.create
(null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
codec, 1000), true);
#pragma warning restore 612
for (int i = 0; i < 2048; ++i)
{
int x = (int)@in.next();
if (i < 1024)
{
Assert.Equal(i / 4, x);
}
else if (i < 2048)
{
Assert.Equal(2 * i, x);
}
else
{
Assert.Equal(junk[i - 2048], x);
}
}
for (int i = 2047; i >= 0; --i)
{
@in.seek(positions[i]);
int x = (int)@in.next();
if (i < 1024)
{
Assert.Equal(i / 4, x);
}
else if (i < 2048)
{
Assert.Equal(2 * i, x);
}
else
{
Assert.Equal(junk[i - 2048], x);
}
}
}
public void testSkips()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
RunLengthIntegerWriter @out = new RunLengthIntegerWriter(
new OutStream("test", 100, null, collect), true);
for (int i = 0; i < 2048; ++i)
{
if (i < 1024)
{
@out.write(i);
}
else
{
@out.write(256 * i);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
RunLengthIntegerReader @in = new RunLengthIntegerReader(InStream.create
(null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
null, 100), true);
#pragma warning restore 612
for (int i = 0; i < 2048; i += 10)
{
int x = (int)@in.next();
if (i < 1024)
{
Assert.Equal(i, x);
}
else
{
Assert.Equal(256 * i, x);
}
if (i < 2038)
{
@in.skip(9);
}
@in.skip(0);
}
}
public void runSeekTest(CompressionCodec codec)
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
RunLengthIntegerWriter @out = new RunLengthIntegerWriter(
new OutStream("test", 1000, codec, collect), true);
TestInStream.PositionCollector[] positions =
new TestInStream.PositionCollector[4096];
Random random = new Random(99);
int[] junk = new int[2048];
for (int i = 0; i < junk.Length; ++i)
{
junk[i] = random.Next();
}
for (int i = 0; i < 4096; ++i)
{
positions[i] = new TestInStream.PositionCollector();
@out.getPosition(positions[i]);
// test runs, incrementing runs, non-runs
if (i < 1024)
{
@out.write(i / 4);
}
else if (i < 2048)
{
@out.write(2 * i);
}
else
{
@out.write(junk[i - 2048]);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
RunLengthIntegerReader @in = new RunLengthIntegerReader(InStream.create
(null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
codec, 1000), true);
#pragma warning restore 612
for (int i = 0; i < 2048; ++i)
{
int x = (int)@in.next();
if (i < 1024)
{
Assert.Equal(i / 4, x);
}
else if (i < 2048)
{
Assert.Equal(2 * i, x);
}
else
{
Assert.Equal(junk[i - 2048], x);
}
}
for (int i = 2047; i >= 0; --i)
{
@in.seek(positions[i]);
int x = (int)@in.next();
if (i < 1024)
{
Assert.Equal(i / 4, x);
}
else if (i < 2048)
{
Assert.Equal(2 * i, x);
}
else
{
Assert.Equal(junk[i - 2048], x);
}
}
}
private void runTest(int numBits)
{
long[] inp = new long[SIZE];
for (int i = 0; i < SIZE; i++)
{
long val = 0;
if (numBits <= 32)
{
if (numBits == 1)
{
val = -1 * rand.Next(2);
}
else
{
int max = (numBits == 32) ? Int32.MaxValue : (int)Math.Pow(2, numBits - 1);
val = rand.Next(max);
}
}
else
{
val = nextLong(rand, (long)Math.Pow(2, numBits - 2));
}
if (val % 2 == 0)
{
val = -val;
}
inp[i] = val;
}
long[] deltaEncoded = deltaEncode(inp);
long minInput = deltaEncoded.Min();
long maxInput = deltaEncoded.Max();
long rangeInput = maxInput - minInput;
SerializationUtils utils = new SerializationUtils();
int fixedWidth = utils.findClosestNumBits(rangeInput);
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
OutStream output = new OutStream("test", SIZE, null, collect);
utils.writeInts(deltaEncoded, 0, deltaEncoded.Length, fixedWidth, output);
output.Flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
long[] buff = new long[SIZE];
#pragma warning disable 612
utils.readInts(buff, 0, SIZE, fixedWidth, InStream.create(null, "test", new ByteBuffer[] { inBuf },
new long[] { 0 }, inBuf.remaining(), null, SIZE));
#pragma warning restore 612
for (int i = 0; i < SIZE; i++)
{
buff[i] = utils.zigzagDecode(buff[i]);
}
Assert.Equal(numBits, fixedWidth);
Assert.Equal(inp, buff);
}
public void testBiggerItems()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
const int COUNT = 16384;
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 500, null, collect), 3);
for (int i = 0; i < COUNT; ++i)
{
// test runs, non-runs
if (i < COUNT / 2)
{
@out.write(i & 7);
}
else
{
@out.write((i / 3) & 7);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
null, 500), 3);
#pragma warning restore 612
for (int i = 0; i < COUNT; ++i)
{
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 7, x);
}
else
{
Assert.Equal((i / 3) & 7, x);
}
}
}
public void testSkips()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 100, null, collect), 1);
const int COUNT = 16384;
for (int i = 0; i < COUNT; ++i)
{
if (i < COUNT / 2)
{
@out.write(i & 1);
}
else
{
@out.write((i / 3) & 1);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create
(null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
null, 100), 1);
#pragma warning restore 612
for (int i = 0; i < COUNT; i += 5)
{
int x = (int)@in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
if (i < COUNT - 5)
{
@in.skip(4);
}
@in.skip(0);
}
}
