public void runSeekTest(CompressionCodec codec)
        {
            TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
            const int      COUNT = 16384;
            BitFieldWriter @out  = new BitFieldWriter(
                new OutStream("test", 500, codec, collect), 1);

            TestInStream.PositionCollector[] positions =
                new TestInStream.PositionCollector[COUNT];
            for (int i = 0; i < COUNT; ++i)
            {
                positions[i] = new TestInStream.PositionCollector();
                @out.getPosition(positions[i]);
                // test runs, non-runs
                if (i < COUNT / 2)
                {
                    @out.write(i & 1);
                }
                else
                {
                    @out.write((i / 3) & 1);
                }
            }
            @out.flush();
            ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());

            collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
            inBuf.flip();
#pragma warning disable 612
            BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
                                                                    new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
                                                                    codec, 500), 1);
#pragma warning restore 612
            for (int i = 0; i < COUNT; ++i)
            {
                int x = @in.next();
                if (i < COUNT / 2)
                {
                    Assert.Equal(i & 1, x);
                }
                else
                {
                    Assert.Equal((i / 3) & 1, x);
                }
            }
            for (int i = COUNT - 1; i >= 0; --i)
            {
                @in.seek(positions[i]);
                int x = @in.next();
                if (i < COUNT / 2)
                {
                    Assert.Equal(i & 1, x);
                }
                else
                {
                    Assert.Equal((i / 3) & 1, x);
                }
            }
        }
 public void runSeekTest(CompressionCodec codec)
 {
     TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
     const int COUNT = 16384;
     BitFieldWriter @out = new BitFieldWriter(
         new OutStream("test", 500, codec, collect), 1);
     TestInStream.PositionCollector[] positions =
         new TestInStream.PositionCollector[COUNT];
     for (int i = 0; i < COUNT; ++i)
     {
         positions[i] = new TestInStream.PositionCollector();
         @out.getPosition(positions[i]);
         // test runs, non-runs
         if (i < COUNT / 2)
         {
             @out.write(i & 1);
         }
         else
         {
             @out.write((i / 3) & 1);
         }
     }
     @out.flush();
     ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
     collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
     inBuf.flip();
     #pragma warning disable 612
     BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
         new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
         codec, 500), 1);
     #pragma warning restore 612
     for (int i = 0; i < COUNT; ++i)
     {
         int x = @in.next();
         if (i < COUNT / 2)
         {
             Assert.Equal(i & 1, x);
         }
         else
         {
             Assert.Equal((i / 3) & 1, x);
         }
     }
     for (int i = COUNT - 1; i >= 0; --i)
     {
         @in.seek(positions[i]);
         int x = @in.next();
         if (i < COUNT / 2)
         {
             Assert.Equal(i & 1, x);
         }
         else
         {
             Assert.Equal((i / 3) & 1, x);
         }
     }
 }
        public void testSkips()
        {
            TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
            BitFieldWriter @out = new BitFieldWriter(
                new OutStream("test", 100, null, collect), 1);
            const int COUNT = 16384;

            for (int i = 0; i < COUNT; ++i)
            {
                if (i < COUNT / 2)
                {
                    @out.write(i & 1);
                }
                else
                {
                    @out.write((i / 3) & 1);
                }
            }
            @out.flush();
            ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());

            collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
            inBuf.flip();
#pragma warning disable 612
            BitFieldReader @in = new BitFieldReader(InStream.create
                                                        (null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
                                                        null, 100), 1);
#pragma warning restore 612
            for (int i = 0; i < COUNT; i += 5)
            {
                int x = (int)@in.next();
                if (i < COUNT / 2)
                {
                    Assert.Equal(i & 1, x);
                }
                else
                {
                    Assert.Equal((i / 3) & 1, x);
                }
                if (i < COUNT - 5)
                {
                    @in.skip(4);
                }
                @in.skip(0);
            }
        }
        public void testBiggerItems()
        {
            TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
            const int      COUNT = 16384;
            BitFieldWriter @out  = new BitFieldWriter(
                new OutStream("test", 500, null, collect), 3);

            for (int i = 0; i < COUNT; ++i)
            {
                // test runs, non-runs
                if (i < COUNT / 2)
                {
                    @out.write(i & 7);
                }
                else
                {
                    @out.write((i / 3) & 7);
                }
            }
            @out.flush();
            ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());

            collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
            inBuf.flip();
#pragma warning disable 612
            BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
                                                                    new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
                                                                    null, 500), 3);
#pragma warning restore 612
            for (int i = 0; i < COUNT; ++i)
            {
                int x = @in.next();
                if (i < COUNT / 2)
                {
                    Assert.Equal(i & 7, x);
                }
                else
                {
                    Assert.Equal((i / 3) & 7, x);
                }
            }
        }
Beispiel #5
0
 /**
  * Create a tree writer.
  * @param columnId the column id of the column to write
  * @param inspector the object inspector to use
  * @param schema the row schema
  * @param streamFactory limited access to the Writer's data.
  * @param nullable can the value be null?
  * @
  */
 protected TreeWriter(
     int columnId,
     ObjectInspector inspector,
     TypeDescription schema,
     StreamFactory streamFactory,
     bool nullable)
 {
     this.streamFactory = streamFactory;
     this.isCompressed = streamFactory.isCompressed();
     this.id = columnId;
     this.inspector = inspector;
     if (nullable)
     {
         isPresentOutStream = streamFactory.createStream(id,
             OrcProto.Stream.Types.Kind.PRESENT);
         isPresent = new BitFieldWriter(isPresentOutStream, 1);
     }
     else
     {
         isPresent = null;
     }
     this.foundNulls = false;
     createBloomFilter = streamFactory.getBloomFilterColumns()[columnId];
     indexStatistics = ColumnStatisticsImpl.create(schema);
     stripeColStatistics = ColumnStatisticsImpl.create(schema);
     fileStatistics = ColumnStatisticsImpl.create(schema);
     childrenWriters = new TreeWriter[0];
     rowIndex = OrcProto.RowIndex.CreateBuilder();
     rowIndexEntry = OrcProto.RowIndexEntry.CreateBuilder();
     rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry);
     stripeStatsBuilders = new List<OrcProto.StripeStatistics.Builder>();
     if (streamFactory.buildIndex())
     {
         rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.ROW_INDEX);
     }
     else
     {
         rowIndexStream = null;
     }
     if (createBloomFilter)
     {
         bloomFilterEntry = OrcProto.BloomFilter.CreateBuilder();
         bloomFilterIndex = OrcProto.BloomFilterIndex.CreateBuilder();
         bloomFilterStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.BLOOM_FILTER);
         bloomFilter = new BloomFilter(streamFactory.getRowIndexStride(), streamFactory.getBloomFilterFPP());
     }
     else
     {
         bloomFilterEntry = null;
         bloomFilterIndex = null;
         bloomFilterStream = null;
         bloomFilter = null;
     }
 }
Beispiel #6
0
 public BooleanTreeWriter(int columnId,
                   ObjectInspector inspector,
                   TypeDescription schema,
                   StreamFactory writer,
                   bool nullable)
     : base(columnId, inspector, schema, writer, nullable)
 {
     PositionedOutputStream @out = writer.createStream(id,
         OrcProto.Stream.Types.Kind.DATA);
     this.writer = new BitFieldWriter(@out, 1);
     recordPosition(rowIndexPosition);
 }
 public void testBiggerItems()
 {
     TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
     const int COUNT = 16384;
     BitFieldWriter @out = new BitFieldWriter(
         new OutStream("test", 500, null, collect), 3);
     for (int i = 0; i < COUNT; ++i)
     {
         // test runs, non-runs
         if (i < COUNT / 2)
         {
             @out.write(i & 7);
         }
         else
         {
             @out.write((i / 3) & 7);
         }
     }
     @out.flush();
     ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
     collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
     inBuf.flip();
     #pragma warning disable 612
     BitFieldReader @in = new BitFieldReader(InStream.create(null, "test",
         new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
         null, 500), 3);
     #pragma warning restore 612
     for (int i = 0; i < COUNT; ++i)
     {
         int x = @in.next();
         if (i < COUNT / 2)
         {
             Assert.Equal(i & 7, x);
         }
         else
         {
             Assert.Equal((i / 3) & 7, x);
         }
     }
 }
 public void testSkips()
 {
     TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
     BitFieldWriter @out = new BitFieldWriter(
         new OutStream("test", 100, null, collect), 1);
     const int COUNT = 16384;
     for (int i = 0; i < COUNT; ++i)
     {
         if (i < COUNT / 2)
         {
             @out.write(i & 1);
         }
         else
         {
             @out.write((i / 3) & 1);
         }
     }
     @out.flush();
     ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
     collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
     inBuf.flip();
     #pragma warning disable 612
     BitFieldReader @in = new BitFieldReader(InStream.create
         (null, "test", new ByteBuffer[] { inBuf }, new long[] { 0 }, inBuf.remaining(),
             null, 100), 1);
     #pragma warning restore 612
     for (int i = 0; i < COUNT; i += 5)
     {
         int x = (int)@in.next();
         if (i < COUNT / 2)
         {
             Assert.Equal(i & 1, x);
         }
         else
         {
             Assert.Equal((i / 3) & 1, x);
         }
         if (i < COUNT - 5)
         {
             @in.skip(4);
         }
         @in.skip(0);
     }
 }