public void runSeekTest(CompressionCodec codec)
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
const int COUNT = 16384;
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 500, codec, collect), 1);
TestInStream.PositionCollector[] positions =
new TestInStream.PositionCollector[COUNT];
for (int i = 0; i < COUNT; ++i)
{
positions[i] = new TestInStream.PositionCollector();
@out.getPosition(positions[i]);
// test runs, non-runs
if (i < COUNT / 2)
{
@out.write(i & 1);
}
else
{
@out.write((i / 3) & 1);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
codec, 500), 1);
#pragma warning restore 612
for (int i = 0; i < COUNT; ++i)
{
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
}
for (int i = COUNT - 1; i >= 0; --i)
{
@in.seek(positions[i]);
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
}
}
public void runSeekTest(CompressionCodec codec)
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
const int COUNT = 16384;
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 500, codec, collect), 1);
TestInStream.PositionCollector[] positions =
new TestInStream.PositionCollector[COUNT];
for (int i = 0; i < COUNT; ++i)
{
positions[i] = new TestInStream.PositionCollector();
@out.getPosition(positions[i]);
// test runs, non-runs
if (i < COUNT / 2)
{
@out.write(i & 1);
}
else
{
@out.write((i / 3) & 1);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
codec, 500), 1);
#pragma warning restore 612
for (int i = 0; i < COUNT; ++i)
{
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
}
for (int i = COUNT - 1; i >= 0; --i)
{
@in.seek(positions[i]);
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
}
}
public void testSkips()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 100, null, collect), 1);
const int COUNT = 16384;
for (int i = 0; i < COUNT; ++i)
{
if (i < COUNT / 2)
{
@out.write(i & 1);
}
else
{
@out.write((i / 3) & 1);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create
(null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
null, 100), 1);
#pragma warning restore 612
for (int i = 0; i < COUNT; i += 5)
{
int x = (int)@in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
if (i < COUNT - 5)
{
@in.skip(4);
}
@in.skip(0);
}
}
public void testBiggerItems()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
const int COUNT = 16384;
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 500, null, collect), 3);
for (int i = 0; i < COUNT; ++i)
{
// test runs, non-runs
if (i < COUNT / 2)
{
@out.write(i & 7);
}
else
{
@out.write((i / 3) & 7);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
null, 500), 3);
#pragma warning restore 612
for (int i = 0; i < COUNT; ++i)
{
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 7, x);
}
else
{
Assert.Equal((i / 3) & 7, x);
}
}
}
/**
* Create a tree writer.
* @param columnId the column id of the column to write
* @param inspector the object inspector to use
* @param schema the row schema
* @param streamFactory limited access to the Writer's data.
* @param nullable can the value be null?
* @
*/
protected TreeWriter(
int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory streamFactory,
bool nullable)
{
this.streamFactory = streamFactory;
this.isCompressed = streamFactory.isCompressed();
this.id = columnId;
this.inspector = inspector;
if (nullable)
{
isPresentOutStream = streamFactory.createStream(id,
OrcProto.Stream.Types.Kind.PRESENT);
isPresent = new BitFieldWriter(isPresentOutStream, 1);
}
else
{
isPresent = null;
}
this.foundNulls = false;
createBloomFilter = streamFactory.getBloomFilterColumns()[columnId];
indexStatistics = ColumnStatisticsImpl.create(schema);
stripeColStatistics = ColumnStatisticsImpl.create(schema);
fileStatistics = ColumnStatisticsImpl.create(schema);
childrenWriters = new TreeWriter[0];
rowIndex = OrcProto.RowIndex.CreateBuilder();
rowIndexEntry = OrcProto.RowIndexEntry.CreateBuilder();
rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry);
stripeStatsBuilders = new List<OrcProto.StripeStatistics.Builder>();
if (streamFactory.buildIndex())
{
rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.ROW_INDEX);
}
else
{
rowIndexStream = null;
}
if (createBloomFilter)
{
bloomFilterEntry = OrcProto.BloomFilter.CreateBuilder();
bloomFilterIndex = OrcProto.BloomFilterIndex.CreateBuilder();
bloomFilterStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.BLOOM_FILTER);
bloomFilter = new BloomFilter(streamFactory.getRowIndexStride(), streamFactory.getBloomFilterFPP());
}
else
{
bloomFilterEntry = null;
bloomFilterIndex = null;
bloomFilterStream = null;
bloomFilter = null;
}
}
public BooleanTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
PositionedOutputStream @out = writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA);
this.writer = new BitFieldWriter(@out, 1);
recordPosition(rowIndexPosition);
}
public void testBiggerItems()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
const int COUNT = 16384;
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 500, null, collect), 3);
for (int i = 0; i < COUNT; ++i)
{
// test runs, non-runs
if (i < COUNT / 2)
{
@out.write(i & 7);
}
else
{
@out.write((i / 3) & 7);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
null, 500), 3);
#pragma warning restore 612
for (int i = 0; i < COUNT; ++i)
{
int x = @in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 7, x);
}
else
{
Assert.Equal((i / 3) & 7, x);
}
}
}
public void testSkips()
{
TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
BitFieldWriter @out = new BitFieldWriter(
new OutStream("test", 100, null, collect), 1);
const int COUNT = 16384;
for (int i = 0; i < COUNT; ++i)
{
if (i < COUNT / 2)
{
@out.write(i & 1);
}
else
{
@out.write((i / 3) & 1);
}
}
@out.flush();
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
#pragma warning disable 612
BitFieldReader @in = new BitFieldReader(InStream.create
(null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
null, 100), 1);
#pragma warning restore 612
for (int i = 0; i < COUNT; i += 5)
{
int x = (int)@in.next();
if (i < COUNT / 2)
{
Assert.Equal(i & 1, x);
}
else
{
Assert.Equal((i / 3) & 1, x);
}
if (i < COUNT - 5)
{
@in.skip(4);
}
@in.skip(0);
}
}