public void runSeekTest(CompressionCodec codec) { TestInStream.OutputCollector collect = new TestInStream.OutputCollector(); const int COUNT = 16384; BitFieldWriter @out = new BitFieldWriter( new OutStream("test", 500, codec, collect), 1); TestInStream.PositionCollector[] positions = new TestInStream.PositionCollector[COUNT]; for (int i = 0; i < COUNT; ++i) { positions[i] = new TestInStream.PositionCollector(); @out.getPosition(positions[i]); // test runs, non-runs if (i < COUNT / 2) { @out.write(i & 1); } else { @out.write((i / 3) & 1); } } @out.flush(); ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size()); collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size()); inBuf.flip(); #pragma warning disable 612 BitFieldReader @in = new BitFieldReader(InStream.create(null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(), codec, 500), 1); #pragma warning restore 612 for (int i = 0; i < COUNT; ++i) { int x = @in.next(); if (i < COUNT / 2) { Assert.Equal(i & 1, x); } else { Assert.Equal((i / 3) & 1, x); } } for (int i = COUNT - 1; i >= 0; --i) { @in.seek(positions[i]); int x = @in.next(); if (i < COUNT / 2) { Assert.Equal(i & 1, x); } else { Assert.Equal((i / 3) & 1, x); } } }
public void testSkips() { TestInStream.OutputCollector collect = new TestInStream.OutputCollector(); BitFieldWriter @out = new BitFieldWriter( new OutStream("test", 100, null, collect), 1); const int COUNT = 16384; for (int i = 0; i < COUNT; ++i) { if (i < COUNT / 2) { @out.write(i & 1); } else { @out.write((i / 3) & 1); } } @out.flush(); ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size()); collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size()); inBuf.flip(); #pragma warning disable 612 BitFieldReader @in = new BitFieldReader(InStream.create (null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(), null, 100), 1); #pragma warning restore 612 for (int i = 0; i < COUNT; i += 5) { int x = (int)@in.next(); if (i < COUNT / 2) { Assert.Equal(i & 1, x); } else { Assert.Equal((i / 3) & 1, x); } if (i < COUNT - 5) { @in.skip(4); } @in.skip(0); } }
public void testBiggerItems() { TestInStream.OutputCollector collect = new TestInStream.OutputCollector(); const int COUNT = 16384; BitFieldWriter @out = new BitFieldWriter( new OutStream("test", 500, null, collect), 3); for (int i = 0; i < COUNT; ++i) { // test runs, non-runs if (i < COUNT / 2) { @out.write(i & 7); } else { @out.write((i / 3) & 7); } } @out.flush(); ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size()); collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size()); inBuf.flip(); #pragma warning disable 612 BitFieldReader @in = new BitFieldReader(InStream.create(null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(), null, 500), 3); #pragma warning restore 612 for (int i = 0; i < COUNT; ++i) { int x = @in.next(); if (i < COUNT / 2) { Assert.Equal(i & 7, x); } else { Assert.Equal((i / 3) & 7, x); } } }
/** * Create a tree writer. * @param columnId the column id of the column to write * @param inspector the object inspector to use * @param schema the row schema * @param streamFactory limited access to the Writer's data. * @param nullable can the value be null? * @ */ protected TreeWriter( int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory streamFactory, bool nullable) { this.streamFactory = streamFactory; this.isCompressed = streamFactory.isCompressed(); this.id = columnId; this.inspector = inspector; if (nullable) { isPresentOutStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.PRESENT); isPresent = new BitFieldWriter(isPresentOutStream, 1); } else { isPresent = null; } this.foundNulls = false; createBloomFilter = streamFactory.getBloomFilterColumns()[columnId]; indexStatistics = ColumnStatisticsImpl.create(schema); stripeColStatistics = ColumnStatisticsImpl.create(schema); fileStatistics = ColumnStatisticsImpl.create(schema); childrenWriters = new TreeWriter[0]; rowIndex = OrcProto.RowIndex.CreateBuilder(); rowIndexEntry = OrcProto.RowIndexEntry.CreateBuilder(); rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry); stripeStatsBuilders = new List<OrcProto.StripeStatistics.Builder>(); if (streamFactory.buildIndex()) { rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.ROW_INDEX); } else { rowIndexStream = null; } if (createBloomFilter) { bloomFilterEntry = OrcProto.BloomFilter.CreateBuilder(); bloomFilterIndex = OrcProto.BloomFilterIndex.CreateBuilder(); bloomFilterStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.BLOOM_FILTER); bloomFilter = new BloomFilter(streamFactory.getRowIndexStride(), streamFactory.getBloomFilterFPP()); } else { bloomFilterEntry = null; bloomFilterIndex = null; bloomFilterStream = null; bloomFilter = null; } }
public BooleanTreeWriter(int columnId, ObjectInspector inspector, TypeDescription schema, StreamFactory writer, bool nullable) : base(columnId, inspector, schema, writer, nullable) { PositionedOutputStream @out = writer.createStream(id, OrcProto.Stream.Types.Kind.DATA); this.writer = new BitFieldWriter(@out, 1); recordPosition(rowIndexPosition); }