public void  InitReader(ByteSliceReader reader, RawPostingList p, int stream)
        {
            System.Diagnostics.Debug.Assert(stream < streamCount);
            int[] ints = intPool.buffers[p.intStart >> DocumentsWriter.INT_BLOCK_SHIFT];
            int   upto = p.intStart & DocumentsWriter.INT_BLOCK_MASK;

            reader.Init(bytePool, p.byteStart + stream * ByteBlockPool.FIRST_LEVEL_SIZE, ints[upto + stream]);
        }
Ejemplo n.º 2
0
        public void InitReader(ByteSliceReader reader, int termID, int stream)
        {
            Debug.Assert(stream < streamCount);
            int intStart = postingsArray.intStarts[termID];

            int[] ints = intPool.Buffers[intStart >> Int32BlockPool.INT32_BLOCK_SHIFT];
            int   upto = intStart & Int32BlockPool.INT32_BLOCK_MASK;

            reader.Init(bytePool, postingsArray.byteStarts[termID] + stream * ByteBlockPool.FIRST_LEVEL_SIZE, ints[upto + stream]);
        }
Ejemplo n.º 3
0
        public virtual void TestBasic()
        {
            ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random().Next(100)));

            int NUM_STREAM = AtLeast(100);

            ByteSliceWriter writer = new ByteSliceWriter(pool);

            int[] starts = new int[NUM_STREAM];
            int[] uptos = new int[NUM_STREAM];
            int[] counters = new int[NUM_STREAM];

            ByteSliceReader reader = new ByteSliceReader();

            for (int ti = 0; ti < 100; ti++)
            {
                for (int stream = 0; stream < NUM_STREAM; stream++)
                {
                    starts[stream] = -1;
                    counters[stream] = 0;
                }

                int num = AtLeast(3000);
                for (int iter = 0; iter < num; iter++)
                {
                    int stream;
                    if (Random().NextBoolean())
                    {
                        stream = Random().Next(3);
                    }
                    else
                    {
                        stream = Random().Next(NUM_STREAM);
                    }

                    if (VERBOSE)
                    {
                        Console.WriteLine("write stream=" + stream);
                    }

                    if (starts[stream] == -1)
                    {
                        int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
                        starts[stream] = uptos[stream] = spot + pool.ByteOffset;
                        if (VERBOSE)
                        {
                            Console.WriteLine("  init to " + starts[stream]);
                        }
                    }

                    writer.Init(uptos[stream]);
                    int numValue;
                    if (Random().Next(10) == 3)
                    {
                        numValue = Random().Next(100);
                    }
                    else if (Random().Next(5) == 3)
                    {
                        numValue = Random().Next(3);
                    }
                    else
                    {
                        numValue = Random().Next(20);
                    }

                    for (int j = 0; j < numValue; j++)
                    {
                        if (VERBOSE)
                        {
                            Console.WriteLine("    write " + (counters[stream] + j));
                        }
                        // write some large (incl. negative) ints:
                        writer.WriteVInt(Random().Next());
                        writer.WriteVInt(counters[stream] + j);
                    }
                    counters[stream] += numValue;
                    uptos[stream] = writer.Address;
                    if (VERBOSE)
                    {
                        Console.WriteLine("    addr now " + uptos[stream]);
                    }
                }

                for (int stream = 0; stream < NUM_STREAM; stream++)
                {
                    if (VERBOSE)
                    {
                        Console.WriteLine("  stream=" + stream + " count=" + counters[stream]);
                    }

                    if (starts[stream] != -1 && starts[stream] != uptos[stream])
                    {
                        reader.Init(pool, starts[stream], uptos[stream]);
                        for (int j = 0; j < counters[stream]; j++)
                        {
                            reader.ReadVInt();
                            Assert.AreEqual(j, reader.ReadVInt());
                        }
                    }
                }

                pool.Reset();
            }
        }
Ejemplo n.º 4
0
        public virtual void  TestBasic()
        {
            ByteBlockPool pool = new ByteBlockPool(new ByteBlockAllocator(), false);

            int NUM_STREAM = 25;

            ByteSliceWriter writer = new ByteSliceWriter(pool);

            int[] starts   = new int[NUM_STREAM];
            int[] uptos    = new int[NUM_STREAM];
            int[] counters = new int[NUM_STREAM];

            System.Random r = NewRandom();

            ByteSliceReader reader = new ByteSliceReader();

            for (int ti = 0; ti < 100; ti++)
            {
                for (int stream = 0; stream < NUM_STREAM; stream++)
                {
                    starts[stream]   = -1;
                    counters[stream] = 0;
                }

                bool debug = false;

                for (int iter = 0; iter < 10000; iter++)
                {
                    int stream = r.Next(NUM_STREAM);
                    if (debug)
                    {
                        System.Console.Out.WriteLine("write stream=" + stream);
                    }

                    if (starts[stream] == -1)
                    {
                        int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE_ForNUnit);
                        starts[stream] = uptos[stream] = spot + pool.byteOffset;
                        if (debug)
                        {
                            System.Console.Out.WriteLine("  init to " + starts[stream]);
                        }
                    }

                    writer.Init(uptos[stream]);
                    int numValue = r.Next(20);
                    for (int j = 0; j < numValue; j++)
                    {
                        if (debug)
                        {
                            System.Console.Out.WriteLine("    write " + (counters[stream] + j));
                        }
                        writer.WriteVInt(counters[stream] + j);
                        //writer.writeVInt(ti);
                    }
                    counters[stream] += numValue;
                    uptos[stream]     = writer.GetAddress();
                    if (debug)
                    {
                        System.Console.Out.WriteLine("    addr now " + uptos[stream]);
                    }
                }

                for (int stream = 0; stream < NUM_STREAM; stream++)
                {
                    if (debug)
                    {
                        System.Console.Out.WriteLine("  stream=" + stream + " count=" + counters[stream]);
                    }

                    if (starts[stream] != uptos[stream])
                    {
                        reader.Init(pool, starts[stream], uptos[stream]);
                        for (int j = 0; j < counters[stream]; j++)
                        {
                            Assert.AreEqual(j, reader.ReadVInt());
                        }
                        //Assert.AreEqual(ti, reader.readVInt());
                    }
                }

                pool.Reset();
            }
        }
Ejemplo n.º 5
0
 public void InitReader(ByteSliceReader reader, int termID, int stream)
 {
     Debug.Assert(stream < StreamCount);
     int intStart = PostingsArray.IntStarts[termID];
     int[] ints = IntPool.Buffers[intStart >> IntBlockPool.INT_BLOCK_SHIFT];
     int upto = intStart & IntBlockPool.INT_BLOCK_MASK;
     reader.Init(BytePool, PostingsArray.ByteStarts[termID] + stream * ByteBlockPool.FIRST_LEVEL_SIZE, ints[upto + stream]);
 }
Ejemplo n.º 6
0
        public virtual void TestBasic()
        {
            // LUCENENET specific: NUnit will crash with an OOM if we do the full test
            // with verbosity enabled. So, making this a manual setting that can be
            // turned on if, and only if, needed for debugging. If the setting is turned
            // on, we are decresing the number of iterations by 1/3, which seems to
            // keep it from crashing.
            bool isVerbose = false;

            if (!isVerbose)
            {
                Console.WriteLine("Verbosity disabled to keep NUnit from running out of memory - enable manually");
            }

            ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random.Next(100)));

            int NUM_STREAM = AtLeast(100);

            ByteSliceWriter writer = new ByteSliceWriter(pool);

            int[] starts   = new int[NUM_STREAM];
            int[] uptos    = new int[NUM_STREAM];
            int[] counters = new int[NUM_STREAM];

            ByteSliceReader reader = new ByteSliceReader();

            for (int ti = 0; ti < 100; ti++)
            {
                for (int stream = 0; stream < NUM_STREAM; stream++)
                {
                    starts[stream]   = -1;
                    counters[stream] = 0;
                }

                // LUCENENET NOTE: Since upgrading to NUnit 3, this test
                // will crash if VERBOSE is true because of an OutOfMemoryException.
                // This not only keeps this test from finishing, it crashes NUnit
                // and no other tests will run.
                // So, we need to allocate a smaller size to ensure this
                // doesn't happen with verbosity enabled.
                int num = isVerbose ? AtLeast(2000) : AtLeast(3000);
                for (int iter = 0; iter < num; iter++)
                {
                    int stream;
                    if (Random.NextBoolean())
                    {
                        stream = Random.Next(3);
                    }
                    else
                    {
                        stream = Random.Next(NUM_STREAM);
                    }

                    if (isVerbose)
                    {
                        Console.WriteLine("write stream=" + stream);
                    }

                    if (starts[stream] == -1)
                    {
                        int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
                        starts[stream] = uptos[stream] = spot + pool.ByteOffset;
                        if (isVerbose)
                        {
                            Console.WriteLine("  init to " + starts[stream]);
                        }
                    }

                    writer.Init(uptos[stream]);
                    int numValue;
                    if (Random.Next(10) == 3)
                    {
                        numValue = Random.Next(100);
                    }
                    else if (Random.Next(5) == 3)
                    {
                        numValue = Random.Next(3);
                    }
                    else
                    {
                        numValue = Random.Next(20);
                    }

                    for (int j = 0; j < numValue; j++)
                    {
                        if (isVerbose)
                        {
                            Console.WriteLine("    write " + (counters[stream] + j));
                        }
                        // write some large (incl. negative) ints:
                        writer.WriteVInt32(Random.Next());
                        writer.WriteVInt32(counters[stream] + j);
                    }
                    counters[stream] += numValue;
                    uptos[stream]     = writer.Address;
                    if (isVerbose)
                    {
                        Console.WriteLine("    addr now " + uptos[stream]);
                    }
                }

                for (int stream = 0; stream < NUM_STREAM; stream++)
                {
                    if (isVerbose)
                    {
                        Console.WriteLine("  stream=" + stream + " count=" + counters[stream]);
                    }

                    if (starts[stream] != -1 && starts[stream] != uptos[stream])
                    {
                        reader.Init(pool, starts[stream], uptos[stream]);
                        for (int j = 0; j < counters[stream]; j++)
                        {
                            reader.ReadVInt32();
                            Assert.AreEqual(j, reader.ReadVInt32());
                        }
                    }
                }

                pool.Reset();
            }
        }
Ejemplo n.º 7
0
		public virtual void  TestBasic()
		{
			ByteBlockPool pool = new ByteBlockPool(new ByteBlockAllocator(), false);
			
			int NUM_STREAM = 25;
			
			ByteSliceWriter writer = new ByteSliceWriter(pool);
			
			int[] starts = new int[NUM_STREAM];
			int[] uptos = new int[NUM_STREAM];
			int[] counters = new int[NUM_STREAM];
			
			System.Random r = NewRandom();
			
			ByteSliceReader reader = new ByteSliceReader();
			
			for (int ti = 0; ti < 100; ti++)
			{
				
				for (int stream = 0; stream < NUM_STREAM; stream++)
				{
					starts[stream] = - 1;
					counters[stream] = 0;
				}
				
				bool debug = false;
				
				for (int iter = 0; iter < 10000; iter++)
				{
					int stream = r.Next(NUM_STREAM);
					if (debug)
						System.Console.Out.WriteLine("write stream=" + stream);
					
					if (starts[stream] == - 1)
					{
						int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE_ForNUnit);
						starts[stream] = uptos[stream] = spot + pool.byteOffset;
						if (debug)
							System.Console.Out.WriteLine("  init to " + starts[stream]);
					}
					
					writer.Init(uptos[stream]);
					int numValue = r.Next(20);
					for (int j = 0; j < numValue; j++)
					{
						if (debug)
							System.Console.Out.WriteLine("    write " + (counters[stream] + j));
						writer.WriteVInt(counters[stream] + j);
						//writer.writeVInt(ti);
					}
					counters[stream] += numValue;
					uptos[stream] = writer.GetAddress();
					if (debug)
						System.Console.Out.WriteLine("    addr now " + uptos[stream]);
				}
				
				for (int stream = 0; stream < NUM_STREAM; stream++)
				{
					if (debug)
						System.Console.Out.WriteLine("  stream=" + stream + " count=" + counters[stream]);
					
					if (starts[stream] != uptos[stream])
					{
						reader.Init(pool, starts[stream], uptos[stream]);
						for (int j = 0; j < counters[stream]; j++)
							Assert.AreEqual(j, reader.ReadVInt());
						//Assert.AreEqual(ti, reader.readVInt());
					}
				}
				
				pool.Reset();
			}
		}
		public void  InitReader(ByteSliceReader reader, RawPostingList p, int stream)
		{
			System.Diagnostics.Debug.Assert(stream < streamCount);
			int[] ints = intPool.buffers[p.intStart >> DocumentsWriter.INT_BLOCK_SHIFT];
			int upto = p.intStart & DocumentsWriter.INT_BLOCK_MASK;
			reader.Init(bytePool, p.byteStart + stream * ByteBlockPool.FIRST_LEVEL_SIZE, ints[upto + stream]);
		}
Ejemplo n.º 9
0
        public virtual void TestBasic()
        {
            ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random().Next(100)));

            int NUM_STREAM = AtLeast(100);

            ByteSliceWriter writer = new ByteSliceWriter(pool);

            int[] starts   = new int[NUM_STREAM];
            int[] uptos    = new int[NUM_STREAM];
            int[] counters = new int[NUM_STREAM];

            ByteSliceReader reader = new ByteSliceReader();

            for (int ti = 0; ti < 100; ti++)
            {
                for (int stream = 0; stream < NUM_STREAM; stream++)
                {
                    starts[stream]   = -1;
                    counters[stream] = 0;
                }

                int num = AtLeast(3000);
                for (int iter = 0; iter < num; iter++)
                {
                    int stream;
                    if (Random().NextBoolean())
                    {
                        stream = Random().Next(3);
                    }
                    else
                    {
                        stream = Random().Next(NUM_STREAM);
                    }

                    if (VERBOSE)
                    {
                        Console.WriteLine("write stream=" + stream);
                    }

                    if (starts[stream] == -1)
                    {
                        int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
                        starts[stream] = uptos[stream] = spot + pool.ByteOffset;
                        if (VERBOSE)
                        {
                            Console.WriteLine("  init to " + starts[stream]);
                        }
                    }

                    writer.Init(uptos[stream]);
                    int numValue;
                    if (Random().Next(10) == 3)
                    {
                        numValue = Random().Next(100);
                    }
                    else if (Random().Next(5) == 3)
                    {
                        numValue = Random().Next(3);
                    }
                    else
                    {
                        numValue = Random().Next(20);
                    }

                    for (int j = 0; j < numValue; j++)
                    {
                        if (VERBOSE)
                        {
                            Console.WriteLine("    write " + (counters[stream] + j));
                        }
                        // write some large (incl. negative) ints:
                        writer.WriteVInt(Random().Next());
                        writer.WriteVInt(counters[stream] + j);
                    }
                    counters[stream] += numValue;
                    uptos[stream]     = writer.Address;
                    if (VERBOSE)
                    {
                        Console.WriteLine("    addr now " + uptos[stream]);
                    }
                }

                for (int stream = 0; stream < NUM_STREAM; stream++)
                {
                    if (VERBOSE)
                    {
                        Console.WriteLine("  stream=" + stream + " count=" + counters[stream]);
                    }

                    if (starts[stream] != -1 && starts[stream] != uptos[stream])
                    {
                        reader.Init(pool, starts[stream], uptos[stream]);
                        for (int j = 0; j < counters[stream]; j++)
                        {
                            reader.ReadVInt();
                            Assert.AreEqual(j, reader.ReadVInt());
                        }
                    }
                }

                pool.Reset();
            }
        }