public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHashPerThread perThread, TermsHashPerThread nextPerThread, FieldInfo fieldInfo)
{
InitBlock();
this.perThread = perThread;
intPool = perThread.intPool;
charPool = perThread.charPool;
bytePool = perThread.bytePool;
docState = perThread.docState;
fieldState = docInverterPerField.fieldState;
// Sorter requires the char pool.
_sorter = new Sorter <RawPostingList, PostingComparer>(new PostingComparer(this));
this.consumer = perThread.consumer.AddField(this, fieldInfo);
streamCount = consumer.GetStreamCount();
numPostingInt = 2 * streamCount;
this.fieldInfo = fieldInfo;
if (nextPerThread != null)
{
nextPerField = (TermsHashPerField)nextPerThread.AddField(docInverterPerField, fieldInfo);
}
else
{
nextPerField = null;
}
}
public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHash termsHash, TermsHash nextTermsHash, FieldInfo fieldInfo)
{
intPool = termsHash.intPool;
bytePool = termsHash.bytePool;
termBytePool = termsHash.termBytePool;
docState = termsHash.docState;
this.termsHash = termsHash;
bytesUsed = termsHash.bytesUsed;
fieldState = docInverterPerField.fieldState;
this.consumer = termsHash.consumer.AddField(this, fieldInfo);
PostingsBytesStartArray byteStarts = new PostingsBytesStartArray(this, bytesUsed);
bytesHash = new BytesRefHash(termBytePool, HASH_INIT_SIZE, byteStarts);
streamCount = consumer.StreamCount;
numPostingInt = 2 * streamCount;
this.fieldInfo = fieldInfo;
if (nextTermsHash != null)
{
nextPerField = (TermsHashPerField)nextTermsHash.AddField(docInverterPerField, fieldInfo);
}
else
{
nextPerField = null;
}
}
public TermsHashPerThread(DocInverterPerThread docInverterPerThread, TermsHash termsHash, TermsHash nextTermsHash, TermsHashPerThread primaryPerThread)
{
docState = docInverterPerThread.docState;
this.termsHash = termsHash;
this.consumer = termsHash.consumer.AddThread(this);
if (nextTermsHash != null)
{
// We are primary
charPool = new CharBlockPool(termsHash.docWriter);
primary = true;
}
else
{
charPool = primaryPerThread.charPool;
primary = false;
}
intPool = new IntBlockPool(termsHash.docWriter, termsHash.trackAllocations);
bytePool = new ByteBlockPool(termsHash.docWriter.byteBlockAllocator, termsHash.trackAllocations);
if (nextTermsHash != null)
{
nextPerThread = nextTermsHash.AddThread(docInverterPerThread, this);
}
else
{
nextPerThread = null;
}
}
public void Init(ByteBlockPool pool, int startIndex, int endIndex)
{
System.Diagnostics.Debug.Assert(endIndex - startIndex >= 0);
System.Diagnostics.Debug.Assert(startIndex >= 0);
System.Diagnostics.Debug.Assert(endIndex >= 0);
this.pool = pool;
this.endIndex = endIndex;
level = 0;
bufferUpto = startIndex / DocumentsWriter.BYTE_BLOCK_SIZE;
bufferOffset = bufferUpto * DocumentsWriter.BYTE_BLOCK_SIZE;
buffer = pool.buffers[bufferUpto];
upto = startIndex & DocumentsWriter.BYTE_BLOCK_MASK;
int firstSize = ByteBlockPool.levelSizeArray[0];
if (startIndex + firstSize >= endIndex)
{
// There is only this one slice to read
limit = endIndex & DocumentsWriter.BYTE_BLOCK_MASK;
}
else
limit = upto + firstSize - 4;
}
public void Init(ByteBlockPool pool, int startIndex, int endIndex)
{
Debug.Assert(endIndex - startIndex >= 0);
Debug.Assert(startIndex >= 0);
Debug.Assert(endIndex >= 0);
this.Pool = pool;
this.EndIndex = endIndex;
Level = 0;
BufferUpto = startIndex / ByteBlockPool.BYTE_BLOCK_SIZE;
BufferOffset = BufferUpto * ByteBlockPool.BYTE_BLOCK_SIZE;
Buffer = pool.Buffers[BufferUpto];
Upto = startIndex & ByteBlockPool.BYTE_BLOCK_MASK;
int firstSize = ByteBlockPool.LEVEL_SIZE_ARRAY[0];
if (startIndex + firstSize >= endIndex)
{
// There is only this one slice to read
Limit = endIndex & ByteBlockPool.BYTE_BLOCK_MASK;
}
else
{
Limit = Upto + firstSize - 4;
}
}
public TermsHashPerThread(DocInverterPerThread docInverterPerThread, TermsHash termsHash, TermsHash nextTermsHash, TermsHashPerThread primaryPerThread)
{
docState = docInverterPerThread.docState;
this.termsHash = termsHash;
this.consumer = termsHash.consumer.AddThread(this);
if (nextTermsHash != null)
{
// We are primary
charPool = new CharBlockPool(termsHash.docWriter);
primary = true;
}
else
{
charPool = primaryPerThread.charPool;
primary = false;
}
intPool = new IntBlockPool(termsHash.docWriter, termsHash.trackAllocations);
bytePool = new ByteBlockPool(termsHash.docWriter.byteBlockAllocator, termsHash.trackAllocations);
if (nextTermsHash != null)
nextPerThread = nextTermsHash.AddThread(docInverterPerThread, this);
else
nextPerThread = null;
}
} // LUCENENET specific - made constructor internal since this class was meant to be internal
public void Init(ByteBlockPool pool, int startIndex, int endIndex)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(endIndex - startIndex >= 0);
Debugging.Assert(startIndex >= 0);
Debugging.Assert(endIndex >= 0);
}
this.pool = pool;
this.EndIndex = endIndex;
level = 0;
bufferUpto = startIndex / ByteBlockPool.BYTE_BLOCK_SIZE;
BufferOffset = bufferUpto * ByteBlockPool.BYTE_BLOCK_SIZE;
buffer = pool.Buffers[bufferUpto];
upto = startIndex & ByteBlockPool.BYTE_BLOCK_MASK;
int firstSize = ByteBlockPool.LEVEL_SIZE_ARRAY[0];
if (startIndex + firstSize >= endIndex)
{
// There is only this one slice to read
limit = endIndex & ByteBlockPool.BYTE_BLOCK_MASK;
}
else
{
limit = upto + firstSize - 4;
}
}
public void Init(ByteBlockPool pool, int startIndex, int endIndex)
{
System.Diagnostics.Debug.Assert(endIndex - startIndex >= 0);
System.Diagnostics.Debug.Assert(startIndex >= 0);
System.Diagnostics.Debug.Assert(endIndex >= 0);
this.pool = pool;
this.endIndex = endIndex;
level = 0;
bufferUpto = startIndex / DocumentsWriter.BYTE_BLOCK_SIZE;
bufferOffset = bufferUpto * DocumentsWriter.BYTE_BLOCK_SIZE;
buffer = pool.buffers[bufferUpto];
upto = startIndex & DocumentsWriter.BYTE_BLOCK_MASK;
int firstSize = ByteBlockPool.levelSizeArray[0];
if (startIndex + firstSize >= endIndex)
{
// There is only this one slice to read
limit = endIndex & DocumentsWriter.BYTE_BLOCK_MASK;
}
else
{
limit = upto + firstSize - 4;
}
}
public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHash termsHash, TermsHash nextTermsHash, FieldInfo fieldInfo)
{
IntPool = termsHash.IntPool;
BytePool = termsHash.BytePool;
TermBytePool = termsHash.TermBytePool;
DocState = termsHash.DocState;
this.TermsHash = termsHash;
BytesUsed = termsHash.BytesUsed;
FieldState = docInverterPerField.FieldState;
this.Consumer = termsHash.Consumer.AddField(this, fieldInfo);
PostingsBytesStartArray byteStarts = new PostingsBytesStartArray(this, BytesUsed);
BytesHash = new BytesRefHash(TermBytePool, HASH_INIT_SIZE, byteStarts);
StreamCount = Consumer.StreamCount;
NumPostingInt = 2 * StreamCount;
this.FieldInfo = fieldInfo;
if (nextTermsHash != null)
{
NextPerField = (TermsHashPerField)nextTermsHash.AddField(docInverterPerField, fieldInfo);
}
else
{
NextPerField = null;
}
}
public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHashPerThread perThread, TermsHashPerThread nextPerThread, FieldInfo fieldInfo)
{
this.perThread = perThread;
intPool = perThread.intPool;
charPool = perThread.charPool;
bytePool = perThread.bytePool;
docState = perThread.docState;
fieldState = docInverterPerField.fieldState;
this.consumer = perThread.consumer.addField(this, fieldInfo);
streamCount = consumer.getStreamCount();
numPostingInt = 2 * streamCount;
this.fieldInfo = fieldInfo;
if (nextPerThread != null)
nextPerField = (TermsHashPerField)nextPerThread.addField(docInverterPerField, fieldInfo);
else
nextPerField = null;
}
public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHashPerThread perThread, TermsHashPerThread nextPerThread, FieldInfo fieldInfo)
{
this.perThread = perThread;
intPool = perThread.intPool;
charPool = perThread.charPool;
bytePool = perThread.bytePool;
docState = perThread.docState;
fieldState = docInverterPerField.fieldState;
this.consumer = perThread.consumer.addField(this, fieldInfo);
streamCount = consumer.getStreamCount();
numPostingInt = 2 * streamCount;
this.fieldInfo = fieldInfo;
if (nextPerThread != null)
{
nextPerField = (TermsHashPerField)nextPerThread.addField(docInverterPerField, fieldInfo);
}
else
{
nextPerField = null;
}
}
public TermsHash(DocumentsWriterPerThread docWriter, TermsHashConsumer consumer, bool trackAllocations, TermsHash nextTermsHash)
{
this.docState = docWriter.docState;
this.consumer = consumer;
this.trackAllocations = trackAllocations;
this.nextTermsHash = nextTermsHash;
this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.NewCounter();
intPool = new Int32BlockPool(docWriter.intBlockAllocator);
bytePool = new ByteBlockPool(docWriter.byteBlockAllocator);
if (nextTermsHash != null)
{
// We are primary
primary = true;
termBytePool = bytePool;
nextTermsHash.termBytePool = bytePool;
}
else
{
primary = false;
}
}
public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHash termsHash, TermsHash nextTermsHash, FieldInfo fieldInfo)
{
IntPool = termsHash.IntPool;
BytePool = termsHash.BytePool;
TermBytePool = termsHash.TermBytePool;
DocState = termsHash.DocState;
this.TermsHash = termsHash;
BytesUsed = termsHash.BytesUsed;
FieldState = docInverterPerField.FieldState;
this.Consumer = termsHash.Consumer.AddField(this, fieldInfo);
PostingsBytesStartArray byteStarts = new PostingsBytesStartArray(this, BytesUsed);
BytesHash = new BytesRefHash(TermBytePool, HASH_INIT_SIZE, byteStarts);
StreamCount = Consumer.StreamCount;
NumPostingInt = 2 * StreamCount;
this.FieldInfo = fieldInfo;
if (nextTermsHash != null)
{
NextPerField = (TermsHashPerField)nextTermsHash.AddField(docInverterPerField, fieldInfo);
}
else
{
NextPerField = null;
}
}
public ByteSliceWriter(ByteBlockPool pool)
{
this.pool = pool;
}
internal void FinishDocument()
{
Debug.Assert(docState.TestPoint("TermVectorsTermsWriterPerField.finish start"));
int numPostings = termsHashPerField.bytesHash.Count;
BytesRef flushTerm = termsWriter.flushTerm;
Debug.Assert(numPostings >= 0);
if (numPostings > maxNumPostings)
{
maxNumPostings = numPostings;
}
// this is called once, after inverting all occurrences
// of a given field in the doc. At this point we flush
// our hash into the DocWriter.
Debug.Assert(termsWriter.VectorFieldsInOrder(fieldInfo));
TermVectorsPostingsArray postings = (TermVectorsPostingsArray)termsHashPerField.postingsArray;
TermVectorsWriter tv = termsWriter.writer;
int[] termIDs = termsHashPerField.SortPostings(tv.Comparer);
tv.StartField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets, hasPayloads);
ByteSliceReader posReader = doVectorPositions ? termsWriter.vectorSliceReaderPos : null;
ByteSliceReader offReader = doVectorOffsets ? termsWriter.vectorSliceReaderOff : null;
ByteBlockPool termBytePool = termsHashPerField.termBytePool;
for (int j = 0; j < numPostings; j++)
{
int termID = termIDs[j];
int freq = postings.freqs[termID];
// Get BytesRef
termBytePool.SetBytesRef(flushTerm, postings.textStarts[termID]);
tv.StartTerm(flushTerm, freq);
if (doVectorPositions || doVectorOffsets)
{
if (posReader != null)
{
termsHashPerField.InitReader(posReader, termID, 0);
}
if (offReader != null)
{
termsHashPerField.InitReader(offReader, termID, 1);
}
tv.AddProx(freq, posReader, offReader);
}
tv.FinishTerm();
}
tv.FinishField();
termsHashPerField.Reset();
fieldInfo.SetStoreTermVectors();
}
private void InitBlock(DocumentsWriter enclosingInstance)
{
this.enclosingInstance = enclosingInstance;
allFieldDataArray = new FieldData[10];
postingsPool = new ByteBlockPool(true, enclosingInstance);
vectorsPool = new ByteBlockPool(false, enclosingInstance);
charPool = new CharBlockPool(enclosingInstance);
}
public virtual void TestBasic()
{
ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random().Next(100)));
int NUM_STREAM = AtLeast(100);
ByteSliceWriter writer = new ByteSliceWriter(pool);
int[] starts = new int[NUM_STREAM];
int[] uptos = new int[NUM_STREAM];
int[] counters = new int[NUM_STREAM];
ByteSliceReader reader = new ByteSliceReader();
for (int ti = 0; ti < 100; ti++)
{
for (int stream = 0; stream < NUM_STREAM; stream++)
{
starts[stream] = -1;
counters[stream] = 0;
}
int num = AtLeast(3000);
for (int iter = 0; iter < num; iter++)
{
int stream;
if (Random().NextBoolean())
{
stream = Random().Next(3);
}
else
{
stream = Random().Next(NUM_STREAM);
}
if (VERBOSE)
{
Console.WriteLine("write stream=" + stream);
}
if (starts[stream] == -1)
{
int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
starts[stream] = uptos[stream] = spot + pool.ByteOffset;
if (VERBOSE)
{
Console.WriteLine(" init to " + starts[stream]);
}
}
writer.Init(uptos[stream]);
int numValue;
if (Random().Next(10) == 3)
{
numValue = Random().Next(100);
}
else if (Random().Next(5) == 3)
{
numValue = Random().Next(3);
}
else
{
numValue = Random().Next(20);
}
for (int j = 0; j < numValue; j++)
{
if (VERBOSE)
{
Console.WriteLine(" write " + (counters[stream] + j));
}
// write some large (incl. negative) ints:
writer.WriteVInt(Random().Next());
writer.WriteVInt(counters[stream] + j);
}
counters[stream] += numValue;
uptos[stream] = writer.Address;
if (VERBOSE)
{
Console.WriteLine(" addr now " + uptos[stream]);
}
}
for (int stream = 0; stream < NUM_STREAM; stream++)
{
if (VERBOSE)
{
Console.WriteLine(" stream=" + stream + " count=" + counters[stream]);
}
if (starts[stream] != -1 && starts[stream] != uptos[stream])
{
reader.Init(pool, starts[stream], uptos[stream]);
for (int j = 0; j < counters[stream]; j++)
{
reader.ReadVInt();
Assert.AreEqual(j, reader.ReadVInt());
}
}
}
pool.Reset();
}
}
public virtual void TestBasic()
{
ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random().Next(100)));
int NUM_STREAM = AtLeast(100);
ByteSliceWriter writer = new ByteSliceWriter(pool);
int[] starts = new int[NUM_STREAM];
int[] uptos = new int[NUM_STREAM];
int[] counters = new int[NUM_STREAM];
ByteSliceReader reader = new ByteSliceReader();
for (int ti = 0; ti < 100; ti++)
{
for (int stream = 0; stream < NUM_STREAM; stream++)
{
starts[stream] = -1;
counters[stream] = 0;
}
int num = AtLeast(3000);
for (int iter = 0; iter < num; iter++)
{
int stream;
if (Random().NextBoolean())
{
stream = Random().Next(3);
}
else
{
stream = Random().Next(NUM_STREAM);
}
if (VERBOSE)
{
Console.WriteLine("write stream=" + stream);
}
if (starts[stream] == -1)
{
int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
starts[stream] = uptos[stream] = spot + pool.ByteOffset;
if (VERBOSE)
{
Console.WriteLine(" init to " + starts[stream]);
}
}
writer.Init(uptos[stream]);
int numValue;
if (Random().Next(10) == 3)
{
numValue = Random().Next(100);
}
else if (Random().Next(5) == 3)
{
numValue = Random().Next(3);
}
else
{
numValue = Random().Next(20);
}
for (int j = 0; j < numValue; j++)
{
if (VERBOSE)
{
Console.WriteLine(" write " + (counters[stream] + j));
}
// write some large (incl. negative) ints:
writer.WriteVInt(Random().Next());
writer.WriteVInt(counters[stream] + j);
}
counters[stream] += numValue;
uptos[stream] = writer.Address;
if (VERBOSE)
{
Console.WriteLine(" addr now " + uptos[stream]);
}
}
for (int stream = 0; stream < NUM_STREAM; stream++)
{
if (VERBOSE)
{
Console.WriteLine(" stream=" + stream + " count=" + counters[stream]);
}
if (starts[stream] != -1 && starts[stream] != uptos[stream])
{
reader.Init(pool, starts[stream], uptos[stream]);
for (int j = 0; j < counters[stream]; j++)
{
reader.ReadVInt();
Assert.AreEqual(j, reader.ReadVInt());
}
}
}
pool.Reset();
}
}
public ByteSliceWriter(ByteBlockPool pool)
{
this.pool = pool;
}
public virtual void TestBasic()
{
// LUCENENET specific: NUnit will crash with an OOM if we do the full test
// with verbosity enabled. So, making this a manual setting that can be
// turned on if, and only if, needed for debugging. If the setting is turned
// on, we are decresing the number of iterations by 1/3, which seems to
// keep it from crashing.
bool isVerbose = false;
if (!isVerbose)
{
Console.WriteLine("Verbosity disabled to keep NUnit from running out of memory - enable manually");
}
ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random.Next(100)));
int NUM_STREAM = AtLeast(100);
ByteSliceWriter writer = new ByteSliceWriter(pool);
int[] starts = new int[NUM_STREAM];
int[] uptos = new int[NUM_STREAM];
int[] counters = new int[NUM_STREAM];
ByteSliceReader reader = new ByteSliceReader();
for (int ti = 0; ti < 100; ti++)
{
for (int stream = 0; stream < NUM_STREAM; stream++)
{
starts[stream] = -1;
counters[stream] = 0;
}
// LUCENENET NOTE: Since upgrading to NUnit 3, this test
// will crash if VERBOSE is true because of an OutOfMemoryException.
// This not only keeps this test from finishing, it crashes NUnit
// and no other tests will run.
// So, we need to allocate a smaller size to ensure this
// doesn't happen with verbosity enabled.
int num = isVerbose ? AtLeast(2000) : AtLeast(3000);
for (int iter = 0; iter < num; iter++)
{
int stream;
if (Random.NextBoolean())
{
stream = Random.Next(3);
}
else
{
stream = Random.Next(NUM_STREAM);
}
if (isVerbose)
{
Console.WriteLine("write stream=" + stream);
}
if (starts[stream] == -1)
{
int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE);
starts[stream] = uptos[stream] = spot + pool.ByteOffset;
if (isVerbose)
{
Console.WriteLine(" init to " + starts[stream]);
}
}
writer.Init(uptos[stream]);
int numValue;
if (Random.Next(10) == 3)
{
numValue = Random.Next(100);
}
else if (Random.Next(5) == 3)
{
numValue = Random.Next(3);
}
else
{
numValue = Random.Next(20);
}
for (int j = 0; j < numValue; j++)
{
if (isVerbose)
{
Console.WriteLine(" write " + (counters[stream] + j));
}
// write some large (incl. negative) ints:
writer.WriteVInt32(Random.Next());
writer.WriteVInt32(counters[stream] + j);
}
counters[stream] += numValue;
uptos[stream] = writer.Address;
if (isVerbose)
{
Console.WriteLine(" addr now " + uptos[stream]);
}
}
for (int stream = 0; stream < NUM_STREAM; stream++)
{
if (isVerbose)
{
Console.WriteLine(" stream=" + stream + " count=" + counters[stream]);
}
if (starts[stream] != -1 && starts[stream] != uptos[stream])
{
reader.Init(pool, starts[stream], uptos[stream]);
for (int j = 0; j < counters[stream]; j++)
{
reader.ReadVInt32();
Assert.AreEqual(j, reader.ReadVInt32());
}
}
}
pool.Reset();
}
}
