public void runSeekTest(CompressionCodec codec)
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
const int COUNT = 16384;
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 500, codec, collect), 1);
TestInStream.PositionCollector[] positions =
new TestInStream.PositionCollector[COUNT];
for (int i = 0; i < COUNT; ++i)
{
positions[i] = new TestInStream.PositionCollector();
@out.getPosition(positions[i]);
// test runs, non-runs
if (i < COUNT / 2)
{
@out.write(i & 1);
}
else
{
@out.write((i / 3) & 1);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
codec, 500), 1);
#pragma warning restore 612
for (int i = 0; i < COUNT; ++i)
{
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
}
for (int i = COUNT - 1; i >= 0; --i)
{
@in.seek(positions[i]);
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
}
}
public void testCompressedSeek()
{
CompressionCodec codec = new SnappyCodec();
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
RunLengthByteWriter @out = new RunLengthByteWriter(new OutStream("test", 500,
codec, collect));
TestInStream.PositionCollector[] positions =
new TestInStream.PositionCollector[2048];
for (int i = 0; i < 2048; ++i)
{
positions[i] = new TestInStream.PositionCollector();
@out.getPosition(positions[i]);
if (i < 1024)
{
@out.write((byte)(i / 4));
}
else
{
@out.write((byte)i);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
RunLengthByteReader @in = new RunLengthByteReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(), codec, 500));
for (int i = 0; i < 2048; ++i)
{
int x = @in.next() & 0xff;
if (i < 1024)
{
Assert.Equal((i / 4) & 0xff, x);
}
else
{
Assert.Equal(i & 0xff, x);
}
}
for (int i = 2047; i >= 0; --i)
{
@in.seek(positions[i]);
int x = @in.next() & 0xff;
if (i < 1024)
{
Assert.Equal((i / 4) & 0xff, x);
}
else
{
Assert.Equal(i & 0xff, x);
}
}
}
private void runTest(int numBits)
{
long[] inp = new long[SIZE];
for (int i = 0; i < SIZE; i++)
{
long val = 0;
if (numBits <= 32)
{
if (numBits == 1)
{
val = -1 * rand.Next(2);
}
else
{
int max = (numBits == 32) ? Int32.MaxValue : (int)Math.Pow(2, numBits - 1);
val = rand.Next(max);
}
}
else
{
val = nextLong(rand, (long)Math.Pow(2, numBits - 2));
}
if (val % 2 == 0)
{
val = -val;
}
inp[i] = val;
}
long[] deltaEncoded = deltaEncode(inp);
long minInput = deltaEncoded.Min();
long maxInput = deltaEncoded.Max();
long rangeInput = maxInput - minInput;
SerializationUtils utils = new SerializationUtils();
int fixedWidth = utils.findClosestNumBits(rangeInput);
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
OutStream output = new OutStream("test", SIZE, null, collect);
utils.writeInts(deltaEncoded, 0, deltaEncoded.Length, fixedWidth, output);
output.Flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
long[] buff = new long[SIZE];
#pragma warning disable 612
utils.readInts(buff, 0, SIZE, fixedWidth, InStream.create(null, "test", new ByteBuffer[] { inBuf },
new long[] { 0 }, inBuf.remaining(), null, SIZE));
#pragma warning restore 612
for (int i = 0; i < SIZE; i++)
{
buff[i] = utils.zigzagDecode(buff[i]);
}
Assert.Equal(numBits, fixedWidth);
Assert.Equal(inp, buff);
}
public void testCorruptStream()
{
OutputCollector collect = new OutputCollector();
CompressionCodec codec = new ZlibCodec();
OutStream @out = new OutStream("test", 500, codec, collect);
PositionCollector[] positions = new PositionCollector[1024];
for (int i = 0; i < 1024; ++i)
{
positions[i] = new PositionCollector();
@out.getPosition(positions[i]);
@out.WriteByte((byte)i);
}
@out.Flush();
// now try to read the stream with a buffer that is too small
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
InStream @in = InStream.create(null, "test", new ByteBuffer[] { inBuf },
new long[] { 0 }, inBuf.remaining(), codec, 100);
byte[] contents = new byte[1024];
try
{
@in.Read(contents, 0, contents.Length);
fail();
}
catch (IllegalArgumentException iae)
{
// EXPECTED
}
// make a corrupted header
inBuf.clear();
inBuf.put((byte)32);
inBuf.put((byte)0);
inBuf.flip();
@in = InStream.create(null, "test2", new ByteBuffer[] { inBuf }, new long[] { 0 },
inBuf.remaining(), codec, 300);
try
{
@in.ReadByte();
fail();
}
catch (InvalidOperationException ise)
{
// EXPECTED
}
}
public void testSkips()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
RunLengthIntegerWriterV2 @out = new RunLengthIntegerWriterV2(
new OutStream("test", 100, null, collect), true);
for (int i = 0; i < 2048; ++i)
{
if (i < 1024)
{
@out.write(i);
}
else
{
@out.write(256 * i);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
RunLengthIntegerReaderV2 @in =
new RunLengthIntegerReaderV2(InStream.create(null, "test",
new ByteBuffer[] { inBuf },
new long[] { 0 },
inBuf.remaining(),
null, 100), true, false);
#pragma warning restore 612
for (int i = 0; i < 2048; i += 10)
{
int x = (int)@in.next();
if (i < 1024)
{
Assert.Equal(i, x);
}
else
{
Assert.Equal(256 * i, x);
}
if (i < 2038)
{
@in.skip(9);
}
@in.skip(0);
}
}
public void testSkips()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 100, null, collect), 1);
const int COUNT = 16384;
for (int i = 0; i < COUNT; ++i)
{
if (i < COUNT / 2)
{
@out.write(i & 1);
}
else
{
@out.write((i / 3) & 1);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create
(null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
null, 100), 1);
#pragma warning restore 612
for (int i = 0; i < COUNT; i += 5)
{
int x = (int)@in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
if (i < COUNT - 5)
{
@in.skip(4);
}
@in.skip(0);
}
}
public void testSkips()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
RunLengthByteWriter @out = new RunLengthByteWriter(new OutStream("test", 100,
null, collect));
for (int i = 0; i < 2048; ++i)
{
if (i < 1024)
{
@out.write((byte)(i / 16));
}
else
{
@out.write((byte)i);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
RunLengthByteReader @in = new RunLengthByteReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(), null, 100));
#pragma warning restore 612
for (int i = 0; i < 2048; i += 10)
{
int x = @in.next() & 0xff;
if (i < 1024)
{
Assert.Equal((i / 16) & 0xff, x);
}
else
{
Assert.Equal(i & 0xff, x);
}
if (i < 2038)
{
@in.skip(9);
}
@in.skip(0);
}
}
public void testBiggerItems()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
const int COUNT = 16384;
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 500, null, collect), 3);
for (int i = 0; i < COUNT; ++i)
{
// test runs, non-runs
if (i < COUNT / 2)
{
@out.write(i & 7);
}
else
{
@out.write((i / 3) & 7);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
null, 500), 3);
#pragma warning restore 612
for (int i = 0; i < COUNT; ++i)
{
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 7, x);
}
else
{
Assert.Equal((i / 3) & 7, x);
}
}
}
public void testUncompressed()
{
OutputCollector collect = new OutputCollector();
OutStream @out = new OutStream("test", 100, null, collect);
PositionCollector[] positions = new PositionCollector[1024];
for (int i = 0; i < 1024; ++i)
{
positions[i] = new PositionCollector();
@out.getPosition(positions[i]);
@out.WriteByte((byte)i);
}
@out.Flush();
Assert.Equal(1024, collect.buffer.size());
for (int i = 0; i < 1024; ++i)
{
Assert.Equal((byte)i, collect.buffer.get(i));
}
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
InStream @in = InStream.create(null, "test", new ByteBuffer[] { inBuf },
new long[] { 0 }, inBuf.remaining(), null, 100);
#pragma warning restore 612
Assert.Equal("uncompressed stream test position: 0 length: 1024" +
" range: 0 offset: 0 limit: 0",
@in.ToString());
for (int i = 0; i < 1024; ++i)
{
int x = @in.ReadByte();
Assert.Equal(i & 0xff, x);
}
for (int i = 1023; i >= 0; --i)
{
@in.seek(positions[i]);
Assert.Equal(i & 0xff, @in.ReadByte());
}
}
public void testCompressed()
{
OutputCollector collect = new OutputCollector();
CompressionCodec codec = new ZlibCodec();
OutStream @out = new OutStream("test", 300, codec, collect);
PositionCollector[] positions = new PositionCollector[1024];
for (int i = 0; i < 1024; ++i)
{
positions[i] = new PositionCollector();
@out.getPosition(positions[i]);
@out.WriteByte((byte)i);
}
@out.Flush();
Assert.Equal("test", @out.ToString());
Assert.Equal(961, collect.buffer.size());
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
InStream @in = InStream.create(null, "test", new ByteBuffer[] { inBuf },
new long[] { 0 }, inBuf.remaining(), codec, 300);
Assert.Equal("compressed stream test position: 0 length: 961 range: 0" +
" offset: 0 limit: 0 range 0 = 0 to 961",
@in.ToString());
for (int i = 0; i < 1024; ++i)
{
int x = @in.ReadByte();
Assert.Equal(i & 0xff, x);
}
Assert.Equal(0, @in.available());
for (int i = 1023; i >= 0; --i)
{
@in.seek(positions[i]);
Assert.Equal(i & 0xff, @in.ReadByte());
}
}
public void runSeekTest(CompressionCodec codec)
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
RunLengthIntegerWriter @out = new RunLengthIntegerWriter(
new OutStream("test", 1000, codec, collect), true);
TestInStream.PositionCollector[] positions =
new TestInStream.PositionCollector[4096];
Random random = new Random(99);
int[] junk = new int[2048];
for (int i = 0; i < junk.Length; ++i)
{
junk[i] = random.Next();
}
for (int i = 0; i < 4096; ++i)
{
positions[i] = new TestInStream.PositionCollector();
@out.getPosition(positions[i]);
// test runs, incrementing runs, non-runs
if (i < 1024)
{
@out.write(i / 4);
}
else if (i < 2048)
{
@out.write(2 * i);
}
else
{
@out.write(junk[i - 2048]);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
RunLengthIntegerReader @in = new RunLengthIntegerReader(InStream.create
(null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
codec, 1000), true);
#pragma warning restore 612
for (int i = 0; i < 2048; ++i)
{
int x = (int)@in.next();
if (i < 1024)
{
Assert.Equal(i / 4, x);
}
else if (i < 2048)
{
Assert.Equal(2 * i, x);
}
else
{
Assert.Equal(junk[i - 2048], x);
}
}
for (int i = 2047; i >= 0; --i)
{
@in.seek(positions[i]);
int x = (int)@in.next();
if (i < 1024)
{
Assert.Equal(i / 4, x);
}
else if (i < 2048)
{
Assert.Equal(2 * i, x);
}
else
{
Assert.Equal(junk[i - 2048], x);
}
}
}
public void testUncompressedDisjointBuffers()
{
OutputCollector collect = new OutputCollector();
OutStream @out = new OutStream("test", 400, null, collect);
PositionCollector[] positions = new PositionCollector[1024];
DataOutput stream = new DataOutputStream(@out);
for (int i = 0; i < 1024; ++i)
{
positions[i] = new PositionCollector();
@out.getPosition(positions[i]);
stream.writeInt(i);
}
@out.Flush();
Assert.Equal("test", @out.ToString());
Assert.Equal(4096, collect.buffer.size());
ByteBuffer[] inBuf = new ByteBuffer[3];
inBuf[0] = ByteBuffer.allocate(1100);
inBuf[1] = ByteBuffer.allocate(2200);
inBuf[2] = ByteBuffer.allocate(1100);
collect.buffer.setByteBuffer(inBuf[0], 0, 1024);
collect.buffer.setByteBuffer(inBuf[1], 1024, 2048);
collect.buffer.setByteBuffer(inBuf[2], 3072, 1024);
for (int i = 0; i < inBuf.Length; ++i)
{
inBuf[i].flip();
}
InStream @in = InStream.create(null, "test", inBuf,
new long[] { 0, 1024, 3072 }, 4096, null, 400);
Assert.Equal("uncompressed stream test position: 0 length: 4096" +
" range: 0 offset: 0 limit: 0",
@in.ToString());
DataInputStream inStream = new DataInputStream(@in);
for (int i = 0; i < 1024; ++i)
{
int x = inStream.readInt();
Assert.Equal(i, x);
}
Assert.Equal(0, @in.available());
for (int i = 1023; i >= 0; --i)
{
@in.seek(positions[i]);
Assert.Equal(i, inStream.readInt());
}
@in = InStream.create(null, "test", new ByteBuffer[] { inBuf[1], inBuf[2] },
new long[] { 1024, 3072 }, 4096, null, 400);
inStream = new DataInputStream(@in);
positions[256].reset();
@in.seek(positions[256]);
for (int i = 256; i < 1024; ++i)
{
Assert.Equal(i, inStream.readInt());
}
@in = InStream.create(null, "test", new ByteBuffer[] { inBuf[0], inBuf[2] },
new long[] { 0, 3072 }, 4096, null, 400);
inStream = new DataInputStream(@in);
positions[768].reset();
for (int i = 0; i < 256; ++i)
{
Assert.Equal(i, inStream.readInt());
}
@in.seek(positions[768]);
for (int i = 768; i < 1024; ++i)
{
Assert.Equal(i, inStream.readInt());
}
}
public void testDisjointBuffers()
{
OutputCollector collect = new OutputCollector();
CompressionCodec codec = new ZlibCodec();
OutStream @out = new OutStream("test", 400, codec, collect);
PositionCollector[] positions = new PositionCollector[1024];
DataOutput stream = new DataOutputStream(@out);
for (int i = 0; i < 1024; ++i)
{
positions[i] = new PositionCollector();
@out.getPosition(positions[i]);
stream.writeInt(i);
}
@out.Flush();
Assert.Equal("test", @out.ToString());
Assert.Equal(1674, collect.buffer.size());
ByteBuffer[] inBuf = new ByteBuffer[3];
inBuf[0] = ByteBuffer.allocate(500);
inBuf[1] = ByteBuffer.allocate(1200);
inBuf[2] = ByteBuffer.allocate(500);
collect.buffer.setByteBuffer(inBuf[0], 0, 483);
collect.buffer.setByteBuffer(inBuf[1], 483, 1625 - 483);
collect.buffer.setByteBuffer(inBuf[2], 1625, 1674 - 1625);
for (int i = 0; i < inBuf.Length; ++i)
{
inBuf[i].flip();
}
InStream @in = InStream.create(null, "test", inBuf,
new long[] { 0, 483, 1625 }, 1674, codec, 400);
Assert.Equal("compressed stream test position: 0 length: 1674 range: 0" +
" offset: 0 limit: 0 range 0 = 0 to 483;" +
" range 1 = 483 to 1142; range 2 = 1625 to 49",
@in.ToString());
DataInputStream inStream = new DataInputStream(@in);
for (int i = 0; i < 1024; ++i)
{
int x = inStream.readInt();
Assert.Equal(i, x);
}
Assert.Equal(0, @in.available());
for (int i = 1023; i >= 0; --i)
{
@in.seek(positions[i]);
Assert.Equal(i, inStream.readInt());
}
@in = InStream.create(null, "test", new ByteBuffer[] { inBuf[1], inBuf[2] },
new long[] { 483, 1625 }, 1674, codec, 400);
inStream = new DataInputStream(@in);
positions[303].reset();
@in.seek(positions[303]);
for (int i = 303; i < 1024; ++i)
{
Assert.Equal(i, inStream.readInt());
}
@in = InStream.create(null, "test", new ByteBuffer[] { inBuf[0], inBuf[2] },
new long[] { 0, 1625 }, 1674, codec, 400);
inStream = new DataInputStream(@in);
positions[1001].reset();
for (int i = 0; i < 300; ++i)
{
Assert.Equal(i, inStream.readInt());
}
@in.seek(positions[1001]);
for (int i = 1001; i < 1024; ++i)
{
Assert.Equal(i, inStream.readInt());
}
}
