// Secondary entry point (for 2nd & subsequent TermsHash),
// because token text has already been "interned" into
// textStart, so we hash by textStart
public void add(Token token, int textStart)
{
int code = textStart;
int hashPos = code & postingsHashMask;
System.Diagnostics.Debug.Assert(!postingsCompacted);
// Locate RawPostingList in hash
p = postingsHash[hashPos];
if (p != null && p.textStart != textStart)
{
// Conflict: keep searching different locations in
// the hash table.
int inc = ((code >> 8) + code) | 1;
do
{
code += inc;
hashPos = code & postingsHashMask;
p = postingsHash[hashPos];
} while (p != null && p.textStart != textStart);
}
if (p == null)
{
// First time we are seeing this token since we last
// flushed the hash.
// Refill?
if (0 == perThread.freePostingsCount)
{
perThread.morePostings();
}
// Pull next free RawPostingList from free list
p = perThread.freePostings[--perThread.freePostingsCount];
System.Diagnostics.Debug.Assert(p != null);
p.textStart = textStart;
System.Diagnostics.Debug.Assert(postingsHash[hashPos] == null);
postingsHash[hashPos] = p;
numPostings++;
if (numPostings == postingsHashHalfSize)
{
rehashPostings(2 * postingsHashSize);
}
// Init stream slices
if (numPostingInt + intPool.intUpto > DocumentsWriter.INT_BLOCK_SIZE)
{
intPool.nextBuffer();
}
if (DocumentsWriter.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt * ByteBlockPool.FIRST_LEVEL_SIZE)
{
bytePool.NextBuffer();
}
intUptos = intPool.buffer;
intUptoStart = intPool.intUpto;
intPool.intUpto += streamCount;
p.intStart = intUptoStart + intPool.intOffset;
for (int i = 0; i < streamCount; i++)
{
int upto = bytePool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
intUptos[intUptoStart + i] = upto + bytePool.byteOffset;
}
p.byteStart = intUptos[intUptoStart];
consumer.newTerm(token, p);
}
else
{
intUptos = intPool.buffers[p.intStart >> DocumentsWriter.INT_BLOCK_SHIFT];
intUptoStart = p.intStart & DocumentsWriter.INT_BLOCK_MASK;
consumer.addTerm(token, p);
}
}
public virtual void TestBasic()
{
ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random().Next(100)));
int NUM_STREAM = AtLeast(100);
ByteSliceWriter writer = new ByteSliceWriter(pool);
int[] starts = new int[NUM_STREAM];
int[] uptos = new int[NUM_STREAM];
int[] counters = new int[NUM_STREAM];
ByteSliceReader reader = new ByteSliceReader();
for (int ti = 0; ti < 100; ti++)
{
for (int stream = 0; stream < NUM_STREAM; stream++)
{
starts[stream] = -1;
counters[stream] = 0;
}
int num = AtLeast(3000);
for (int iter = 0; iter < num; iter++)
{
int stream;
if (Random().NextBoolean())
{
stream = Random().Next(3);
}
else
{
stream = Random().Next(NUM_STREAM);
}
if (VERBOSE)
{
Console.WriteLine("write stream=" + stream);
}
if (starts[stream] == -1)
{
int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
starts[stream] = uptos[stream] = spot + pool.ByteOffset;
if (VERBOSE)
{
Console.WriteLine(" init to " + starts[stream]);
}
}
writer.Init(uptos[stream]);
int numValue;
if (Random().Next(10) == 3)
{
numValue = Random().Next(100);
}
else if (Random().Next(5) == 3)
{
numValue = Random().Next(3);
}
else
{
numValue = Random().Next(20);
}
for (int j = 0; j < numValue; j++)
{
if (VERBOSE)
{
Console.WriteLine(" write " + (counters[stream] + j));
}
// write some large (incl. negative) ints:
writer.WriteVInt(Random().Next());
writer.WriteVInt(counters[stream] + j);
}
counters[stream] += numValue;
uptos[stream] = writer.Address;
if (VERBOSE)
{
Console.WriteLine(" addr now " + uptos[stream]);
}
}
for (int stream = 0; stream < NUM_STREAM; stream++)
{
if (VERBOSE)
{
Console.WriteLine(" stream=" + stream + " count=" + counters[stream]);
}
if (starts[stream] != -1 && starts[stream] != uptos[stream])
{
reader.Init(pool, starts[stream], uptos[stream]);
for (int j = 0; j < counters[stream]; j++)
{
reader.ReadVInt();
Assert.AreEqual(j, reader.ReadVInt());
}
}
}
pool.Reset();
}
}
public virtual void TestBasic()
{
// LUCENENET specific: NUnit will crash with an OOM if we do the full test
// with verbosity enabled. So, making this a manual setting that can be
// turned on if, and only if, needed for debugging. If the setting is turned
// on, we are decresing the number of iterations by 1/3, which seems to
// keep it from crashing.
bool isVerbose = false;
if (!isVerbose)
{
Console.WriteLine("Verbosity disabled to keep NUnit from running out of memory - enable manually");
}
ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random.Next(100)));
int NUM_STREAM = AtLeast(100);
ByteSliceWriter writer = new ByteSliceWriter(pool);
int[] starts = new int[NUM_STREAM];
int[] uptos = new int[NUM_STREAM];
int[] counters = new int[NUM_STREAM];
ByteSliceReader reader = new ByteSliceReader();
for (int ti = 0; ti < 100; ti++)
{
for (int stream = 0; stream < NUM_STREAM; stream++)
{
starts[stream] = -1;
counters[stream] = 0;
}
// LUCENENET NOTE: Since upgrading to NUnit 3, this test
// will crash if VERBOSE is true because of an OutOfMemoryException.
// This not only keeps this test from finishing, it crashes NUnit
// and no other tests will run.
// So, we need to allocate a smaller size to ensure this
// doesn't happen with verbosity enabled.
int num = isVerbose ? AtLeast(2000) : AtLeast(3000);
for (int iter = 0; iter < num; iter++)
{
int stream;
if (Random.NextBoolean())
{
stream = Random.Next(3);
}
else
{
stream = Random.Next(NUM_STREAM);
}
if (isVerbose)
{
Console.WriteLine("write stream=" + stream);
}
if (starts[stream] == -1)
{
int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
starts[stream] = uptos[stream] = spot + pool.ByteOffset;
if (isVerbose)
{
Console.WriteLine(" init to " + starts[stream]);
}
}
writer.Init(uptos[stream]);
int numValue;
if (Random.Next(10) == 3)
{
numValue = Random.Next(100);
}
else if (Random.Next(5) == 3)
{
numValue = Random.Next(3);
}
else
{
numValue = Random.Next(20);
}
for (int j = 0; j < numValue; j++)
{
if (isVerbose)
{
Console.WriteLine(" write " + (counters[stream] + j));
}
// write some large (incl. negative) ints:
writer.WriteVInt32(Random.Next());
writer.WriteVInt32(counters[stream] + j);
}
counters[stream] += numValue;
uptos[stream] = writer.Address;
if (isVerbose)
{
Console.WriteLine(" addr now " + uptos[stream]);
}
}
for (int stream = 0; stream < NUM_STREAM; stream++)
{
if (isVerbose)
{
Console.WriteLine(" stream=" + stream + " count=" + counters[stream]);
}
if (starts[stream] != -1 && starts[stream] != uptos[stream])
{
reader.Init(pool, starts[stream], uptos[stream]);
for (int j = 0; j < counters[stream]; j++)
{
reader.ReadVInt32();
Assert.AreEqual(j, reader.ReadVInt32());
}
}
}
pool.Reset();
}
}
public virtual void TestBasic()
{
ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random().Next(100)));
int NUM_STREAM = AtLeast(100);
ByteSliceWriter writer = new ByteSliceWriter(pool);
int[] starts = new int[NUM_STREAM];
int[] uptos = new int[NUM_STREAM];
int[] counters = new int[NUM_STREAM];
ByteSliceReader reader = new ByteSliceReader();
for (int ti = 0; ti < 100; ti++)
{
for (int stream = 0; stream < NUM_STREAM; stream++)
{
starts[stream] = -1;
counters[stream] = 0;
}
int num = AtLeast(3000);
for (int iter = 0; iter < num; iter++)
{
int stream;
if (Random().NextBoolean())
{
stream = Random().Next(3);
}
else
{
stream = Random().Next(NUM_STREAM);
}
if (VERBOSE)
{
Console.WriteLine("write stream=" + stream);
}
if (starts[stream] == -1)
{
int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
starts[stream] = uptos[stream] = spot + pool.ByteOffset;
if (VERBOSE)
{
Console.WriteLine(" init to " + starts[stream]);
}
}
writer.Init(uptos[stream]);
int numValue;
if (Random().Next(10) == 3)
{
numValue = Random().Next(100);
}
else if (Random().Next(5) == 3)
{
numValue = Random().Next(3);
}
else
{
numValue = Random().Next(20);
}
for (int j = 0; j < numValue; j++)
{
if (VERBOSE)
{
Console.WriteLine(" write " + (counters[stream] + j));
}
// write some large (incl. negative) ints:
writer.WriteVInt(Random().Next());
writer.WriteVInt(counters[stream] + j);
}
counters[stream] += numValue;
uptos[stream] = writer.Address;
if (VERBOSE)
{
Console.WriteLine(" addr now " + uptos[stream]);
}
}
for (int stream = 0; stream < NUM_STREAM; stream++)
{
if (VERBOSE)
{
Console.WriteLine(" stream=" + stream + " count=" + counters[stream]);
}
if (starts[stream] != -1 && starts[stream] != uptos[stream])
{
reader.Init(pool, starts[stream], uptos[stream]);
for (int j = 0; j < counters[stream]; j++)
{
reader.ReadVInt();
Assert.AreEqual(j, reader.ReadVInt());
}
}
}
pool.Reset();
}
}