// Secondary entry point (for 2nd & subsequent TermsHash), // because token text has already been "interned" into // textStart, so we hash by textStart public void add(Token token, int textStart) { int code = textStart; int hashPos = code & postingsHashMask; System.Diagnostics.Debug.Assert(!postingsCompacted); // Locate RawPostingList in hash p = postingsHash[hashPos]; if (p != null && p.textStart != textStart) { // Conflict: keep searching different locations in // the hash table. int inc = ((code >> 8) + code) | 1; do { code += inc; hashPos = code & postingsHashMask; p = postingsHash[hashPos]; } while (p != null && p.textStart != textStart); } if (p == null) { // First time we are seeing this token since we last // flushed the hash. // Refill? if (0 == perThread.freePostingsCount) { perThread.morePostings(); } // Pull next free RawPostingList from free list p = perThread.freePostings[--perThread.freePostingsCount]; System.Diagnostics.Debug.Assert(p != null); p.textStart = textStart; System.Diagnostics.Debug.Assert(postingsHash[hashPos] == null); postingsHash[hashPos] = p; numPostings++; if (numPostings == postingsHashHalfSize) { rehashPostings(2 * postingsHashSize); } // Init stream slices if (numPostingInt + intPool.intUpto > DocumentsWriter.INT_BLOCK_SIZE) { intPool.nextBuffer(); } if (DocumentsWriter.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt * ByteBlockPool.FIRST_LEVEL_SIZE) { bytePool.NextBuffer(); } intUptos = intPool.buffer; intUptoStart = intPool.intUpto; intPool.intUpto += streamCount; p.intStart = intUptoStart + intPool.intOffset; for (int i = 0; i < streamCount; i++) { int upto = bytePool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE); intUptos[intUptoStart + i] = upto + bytePool.byteOffset; } p.byteStart = intUptos[intUptoStart]; consumer.newTerm(token, p); } else { intUptos = intPool.buffers[p.intStart >> DocumentsWriter.INT_BLOCK_SHIFT]; intUptoStart = p.intStart & DocumentsWriter.INT_BLOCK_MASK; consumer.addTerm(token, p); } }
public virtual void TestBasic() { ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random().Next(100))); int NUM_STREAM = AtLeast(100); ByteSliceWriter writer = new ByteSliceWriter(pool); int[] starts = new int[NUM_STREAM]; int[] uptos = new int[NUM_STREAM]; int[] counters = new int[NUM_STREAM]; ByteSliceReader reader = new ByteSliceReader(); for (int ti = 0; ti < 100; ti++) { for (int stream = 0; stream < NUM_STREAM; stream++) { starts[stream] = -1; counters[stream] = 0; } int num = AtLeast(3000); for (int iter = 0; iter < num; iter++) { int stream; if (Random().NextBoolean()) { stream = Random().Next(3); } else { stream = Random().Next(NUM_STREAM); } if (VERBOSE) { Console.WriteLine("write stream=" + stream); } if (starts[stream] == -1) { int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE); starts[stream] = uptos[stream] = spot + pool.ByteOffset; if (VERBOSE) { Console.WriteLine(" init to " + starts[stream]); } } writer.Init(uptos[stream]); int numValue; if (Random().Next(10) == 3) { numValue = Random().Next(100); } else if (Random().Next(5) == 3) { numValue = Random().Next(3); } else { numValue = Random().Next(20); } for (int j = 0; j < numValue; j++) { if (VERBOSE) { Console.WriteLine(" write " + (counters[stream] + j)); } // write some large (incl. negative) ints: writer.WriteVInt(Random().Next()); writer.WriteVInt(counters[stream] + j); } counters[stream] += numValue; uptos[stream] = writer.Address; if (VERBOSE) { Console.WriteLine(" addr now " + uptos[stream]); } } for (int stream = 0; stream < NUM_STREAM; stream++) { if (VERBOSE) { Console.WriteLine(" stream=" + stream + " count=" + counters[stream]); } if (starts[stream] != -1 && starts[stream] != uptos[stream]) { reader.Init(pool, starts[stream], uptos[stream]); for (int j = 0; j < counters[stream]; j++) { reader.ReadVInt(); Assert.AreEqual(j, reader.ReadVInt()); } } } pool.Reset(); } }
public virtual void TestBasic() { // LUCENENET specific: NUnit will crash with an OOM if we do the full test // with verbosity enabled. So, making this a manual setting that can be // turned on if, and only if, needed for debugging. If the setting is turned // on, we are decresing the number of iterations by 1/3, which seems to // keep it from crashing. bool isVerbose = false; if (!isVerbose) { Console.WriteLine("Verbosity disabled to keep NUnit from running out of memory - enable manually"); } ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random.Next(100))); int NUM_STREAM = AtLeast(100); ByteSliceWriter writer = new ByteSliceWriter(pool); int[] starts = new int[NUM_STREAM]; int[] uptos = new int[NUM_STREAM]; int[] counters = new int[NUM_STREAM]; ByteSliceReader reader = new ByteSliceReader(); for (int ti = 0; ti < 100; ti++) { for (int stream = 0; stream < NUM_STREAM; stream++) { starts[stream] = -1; counters[stream] = 0; } // LUCENENET NOTE: Since upgrading to NUnit 3, this test // will crash if VERBOSE is true because of an OutOfMemoryException. // This not only keeps this test from finishing, it crashes NUnit // and no other tests will run. // So, we need to allocate a smaller size to ensure this // doesn't happen with verbosity enabled. int num = isVerbose ? AtLeast(2000) : AtLeast(3000); for (int iter = 0; iter < num; iter++) { int stream; if (Random.NextBoolean()) { stream = Random.Next(3); } else { stream = Random.Next(NUM_STREAM); } if (isVerbose) { Console.WriteLine("write stream=" + stream); } if (starts[stream] == -1) { int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE); starts[stream] = uptos[stream] = spot + pool.ByteOffset; if (isVerbose) { Console.WriteLine(" init to " + starts[stream]); } } writer.Init(uptos[stream]); int numValue; if (Random.Next(10) == 3) { numValue = Random.Next(100); } else if (Random.Next(5) == 3) { numValue = Random.Next(3); } else { numValue = Random.Next(20); } for (int j = 0; j < numValue; j++) { if (isVerbose) { Console.WriteLine(" write " + (counters[stream] + j)); } // write some large (incl. negative) ints: writer.WriteVInt32(Random.Next()); writer.WriteVInt32(counters[stream] + j); } counters[stream] += numValue; uptos[stream] = writer.Address; if (isVerbose) { Console.WriteLine(" addr now " + uptos[stream]); } } for (int stream = 0; stream < NUM_STREAM; stream++) { if (isVerbose) { Console.WriteLine(" stream=" + stream + " count=" + counters[stream]); } if (starts[stream] != -1 && starts[stream] != uptos[stream]) { reader.Init(pool, starts[stream], uptos[stream]); for (int j = 0; j < counters[stream]; j++) { reader.ReadVInt32(); Assert.AreEqual(j, reader.ReadVInt32()); } } } pool.Reset(); } }