public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHashPerThread perThread, TermsHashPerThread nextPerThread, FieldInfo fieldInfo) { InitBlock(); this.perThread = perThread; intPool = perThread.intPool; charPool = perThread.charPool; bytePool = perThread.bytePool; docState = perThread.docState; fieldState = docInverterPerField.fieldState; // Sorter requires the char pool. _sorter = new Sorter <RawPostingList, PostingComparer>(new PostingComparer(this)); this.consumer = perThread.consumer.AddField(this, fieldInfo); streamCount = consumer.GetStreamCount(); numPostingInt = 2 * streamCount; this.fieldInfo = fieldInfo; if (nextPerThread != null) { nextPerField = (TermsHashPerField)nextPerThread.AddField(docInverterPerField, fieldInfo); } else { nextPerField = null; } }
public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHash termsHash, TermsHash nextTermsHash, FieldInfo fieldInfo) { intPool = termsHash.intPool; bytePool = termsHash.bytePool; termBytePool = termsHash.termBytePool; docState = termsHash.docState; this.termsHash = termsHash; bytesUsed = termsHash.bytesUsed; fieldState = docInverterPerField.fieldState; this.consumer = termsHash.consumer.AddField(this, fieldInfo); PostingsBytesStartArray byteStarts = new PostingsBytesStartArray(this, bytesUsed); bytesHash = new BytesRefHash(termBytePool, HASH_INIT_SIZE, byteStarts); streamCount = consumer.StreamCount; numPostingInt = 2 * streamCount; this.fieldInfo = fieldInfo; if (nextTermsHash != null) { nextPerField = (TermsHashPerField)nextTermsHash.AddField(docInverterPerField, fieldInfo); } else { nextPerField = null; } }
public TermsHashPerThread(DocInverterPerThread docInverterPerThread, TermsHash termsHash, TermsHash nextTermsHash, TermsHashPerThread primaryPerThread) { docState = docInverterPerThread.docState; this.termsHash = termsHash; this.consumer = termsHash.consumer.AddThread(this); if (nextTermsHash != null) { // We are primary charPool = new CharBlockPool(termsHash.docWriter); primary = true; } else { charPool = primaryPerThread.charPool; primary = false; } intPool = new IntBlockPool(termsHash.docWriter, termsHash.trackAllocations); bytePool = new ByteBlockPool(termsHash.docWriter.byteBlockAllocator, termsHash.trackAllocations); if (nextTermsHash != null) { nextPerThread = nextTermsHash.AddThread(docInverterPerThread, this); } else { nextPerThread = null; } }
public void Init(ByteBlockPool pool, int startIndex, int endIndex) { System.Diagnostics.Debug.Assert(endIndex - startIndex >= 0); System.Diagnostics.Debug.Assert(startIndex >= 0); System.Diagnostics.Debug.Assert(endIndex >= 0); this.pool = pool; this.endIndex = endIndex; level = 0; bufferUpto = startIndex / DocumentsWriter.BYTE_BLOCK_SIZE; bufferOffset = bufferUpto * DocumentsWriter.BYTE_BLOCK_SIZE; buffer = pool.buffers[bufferUpto]; upto = startIndex & DocumentsWriter.BYTE_BLOCK_MASK; int firstSize = ByteBlockPool.levelSizeArray[0]; if (startIndex + firstSize >= endIndex) { // There is only this one slice to read limit = endIndex & DocumentsWriter.BYTE_BLOCK_MASK; } else limit = upto + firstSize - 4; }
public void Init(ByteBlockPool pool, int startIndex, int endIndex) { Debug.Assert(endIndex - startIndex >= 0); Debug.Assert(startIndex >= 0); Debug.Assert(endIndex >= 0); this.Pool = pool; this.EndIndex = endIndex; Level = 0; BufferUpto = startIndex / ByteBlockPool.BYTE_BLOCK_SIZE; BufferOffset = BufferUpto * ByteBlockPool.BYTE_BLOCK_SIZE; Buffer = pool.Buffers[BufferUpto]; Upto = startIndex & ByteBlockPool.BYTE_BLOCK_MASK; int firstSize = ByteBlockPool.LEVEL_SIZE_ARRAY[0]; if (startIndex + firstSize >= endIndex) { // There is only this one slice to read Limit = endIndex & ByteBlockPool.BYTE_BLOCK_MASK; } else { Limit = Upto + firstSize - 4; } }
public TermsHashPerThread(DocInverterPerThread docInverterPerThread, TermsHash termsHash, TermsHash nextTermsHash, TermsHashPerThread primaryPerThread) { docState = docInverterPerThread.docState; this.termsHash = termsHash; this.consumer = termsHash.consumer.AddThread(this); if (nextTermsHash != null) { // We are primary charPool = new CharBlockPool(termsHash.docWriter); primary = true; } else { charPool = primaryPerThread.charPool; primary = false; } intPool = new IntBlockPool(termsHash.docWriter, termsHash.trackAllocations); bytePool = new ByteBlockPool(termsHash.docWriter.byteBlockAllocator, termsHash.trackAllocations); if (nextTermsHash != null) nextPerThread = nextTermsHash.AddThread(docInverterPerThread, this); else nextPerThread = null; }
} // LUCENENET specific - made constructor internal since this class was meant to be internal public void Init(ByteBlockPool pool, int startIndex, int endIndex) { if (Debugging.AssertsEnabled) { Debugging.Assert(endIndex - startIndex >= 0); Debugging.Assert(startIndex >= 0); Debugging.Assert(endIndex >= 0); } this.pool = pool; this.EndIndex = endIndex; level = 0; bufferUpto = startIndex / ByteBlockPool.BYTE_BLOCK_SIZE; BufferOffset = bufferUpto * ByteBlockPool.BYTE_BLOCK_SIZE; buffer = pool.Buffers[bufferUpto]; upto = startIndex & ByteBlockPool.BYTE_BLOCK_MASK; int firstSize = ByteBlockPool.LEVEL_SIZE_ARRAY[0]; if (startIndex + firstSize >= endIndex) { // There is only this one slice to read limit = endIndex & ByteBlockPool.BYTE_BLOCK_MASK; } else { limit = upto + firstSize - 4; } }
public void Init(ByteBlockPool pool, int startIndex, int endIndex) { System.Diagnostics.Debug.Assert(endIndex - startIndex >= 0); System.Diagnostics.Debug.Assert(startIndex >= 0); System.Diagnostics.Debug.Assert(endIndex >= 0); this.pool = pool; this.endIndex = endIndex; level = 0; bufferUpto = startIndex / DocumentsWriter.BYTE_BLOCK_SIZE; bufferOffset = bufferUpto * DocumentsWriter.BYTE_BLOCK_SIZE; buffer = pool.buffers[bufferUpto]; upto = startIndex & DocumentsWriter.BYTE_BLOCK_MASK; int firstSize = ByteBlockPool.levelSizeArray[0]; if (startIndex + firstSize >= endIndex) { // There is only this one slice to read limit = endIndex & DocumentsWriter.BYTE_BLOCK_MASK; } else { limit = upto + firstSize - 4; } }
public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHash termsHash, TermsHash nextTermsHash, FieldInfo fieldInfo) { IntPool = termsHash.IntPool; BytePool = termsHash.BytePool; TermBytePool = termsHash.TermBytePool; DocState = termsHash.DocState; this.TermsHash = termsHash; BytesUsed = termsHash.BytesUsed; FieldState = docInverterPerField.FieldState; this.Consumer = termsHash.Consumer.AddField(this, fieldInfo); PostingsBytesStartArray byteStarts = new PostingsBytesStartArray(this, BytesUsed); BytesHash = new BytesRefHash(TermBytePool, HASH_INIT_SIZE, byteStarts); StreamCount = Consumer.StreamCount; NumPostingInt = 2 * StreamCount; this.FieldInfo = fieldInfo; if (nextTermsHash != null) { NextPerField = (TermsHashPerField)nextTermsHash.AddField(docInverterPerField, fieldInfo); } else { NextPerField = null; } }
public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHashPerThread perThread, TermsHashPerThread nextPerThread, FieldInfo fieldInfo) { this.perThread = perThread; intPool = perThread.intPool; charPool = perThread.charPool; bytePool = perThread.bytePool; docState = perThread.docState; fieldState = docInverterPerField.fieldState; this.consumer = perThread.consumer.addField(this, fieldInfo); streamCount = consumer.getStreamCount(); numPostingInt = 2 * streamCount; this.fieldInfo = fieldInfo; if (nextPerThread != null) nextPerField = (TermsHashPerField)nextPerThread.addField(docInverterPerField, fieldInfo); else nextPerField = null; }
public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHashPerThread perThread, TermsHashPerThread nextPerThread, FieldInfo fieldInfo) { this.perThread = perThread; intPool = perThread.intPool; charPool = perThread.charPool; bytePool = perThread.bytePool; docState = perThread.docState; fieldState = docInverterPerField.fieldState; this.consumer = perThread.consumer.addField(this, fieldInfo); streamCount = consumer.getStreamCount(); numPostingInt = 2 * streamCount; this.fieldInfo = fieldInfo; if (nextPerThread != null) { nextPerField = (TermsHashPerField)nextPerThread.addField(docInverterPerField, fieldInfo); } else { nextPerField = null; } }
public TermsHash(DocumentsWriterPerThread docWriter, TermsHashConsumer consumer, bool trackAllocations, TermsHash nextTermsHash) { this.docState = docWriter.docState; this.consumer = consumer; this.trackAllocations = trackAllocations; this.nextTermsHash = nextTermsHash; this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.NewCounter(); intPool = new Int32BlockPool(docWriter.intBlockAllocator); bytePool = new ByteBlockPool(docWriter.byteBlockAllocator); if (nextTermsHash != null) { // We are primary primary = true; termBytePool = bytePool; nextTermsHash.termBytePool = bytePool; } else { primary = false; } }
public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHash termsHash, TermsHash nextTermsHash, FieldInfo fieldInfo) { IntPool = termsHash.IntPool; BytePool = termsHash.BytePool; TermBytePool = termsHash.TermBytePool; DocState = termsHash.DocState; this.TermsHash = termsHash; BytesUsed = termsHash.BytesUsed; FieldState = docInverterPerField.FieldState; this.Consumer = termsHash.Consumer.AddField(this, fieldInfo); PostingsBytesStartArray byteStarts = new PostingsBytesStartArray(this, BytesUsed); BytesHash = new BytesRefHash(TermBytePool, HASH_INIT_SIZE, byteStarts); StreamCount = Consumer.StreamCount; NumPostingInt = 2 * StreamCount; this.FieldInfo = fieldInfo; if (nextTermsHash != null) { NextPerField = (TermsHashPerField)nextTermsHash.AddField(docInverterPerField, fieldInfo); } else { NextPerField = null; } }
public ByteSliceWriter(ByteBlockPool pool) { this.pool = pool; }
internal void FinishDocument() { Debug.Assert(docState.TestPoint("TermVectorsTermsWriterPerField.finish start")); int numPostings = termsHashPerField.bytesHash.Count; BytesRef flushTerm = termsWriter.flushTerm; Debug.Assert(numPostings >= 0); if (numPostings > maxNumPostings) { maxNumPostings = numPostings; } // this is called once, after inverting all occurrences // of a given field in the doc. At this point we flush // our hash into the DocWriter. Debug.Assert(termsWriter.VectorFieldsInOrder(fieldInfo)); TermVectorsPostingsArray postings = (TermVectorsPostingsArray)termsHashPerField.postingsArray; TermVectorsWriter tv = termsWriter.writer; int[] termIDs = termsHashPerField.SortPostings(tv.Comparer); tv.StartField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets, hasPayloads); ByteSliceReader posReader = doVectorPositions ? termsWriter.vectorSliceReaderPos : null; ByteSliceReader offReader = doVectorOffsets ? termsWriter.vectorSliceReaderOff : null; ByteBlockPool termBytePool = termsHashPerField.termBytePool; for (int j = 0; j < numPostings; j++) { int termID = termIDs[j]; int freq = postings.freqs[termID]; // Get BytesRef termBytePool.SetBytesRef(flushTerm, postings.textStarts[termID]); tv.StartTerm(flushTerm, freq); if (doVectorPositions || doVectorOffsets) { if (posReader != null) { termsHashPerField.InitReader(posReader, termID, 0); } if (offReader != null) { termsHashPerField.InitReader(offReader, termID, 1); } tv.AddProx(freq, posReader, offReader); } tv.FinishTerm(); } tv.FinishField(); termsHashPerField.Reset(); fieldInfo.SetStoreTermVectors(); }
private void InitBlock(DocumentsWriter enclosingInstance) { this.enclosingInstance = enclosingInstance; allFieldDataArray = new FieldData[10]; postingsPool = new ByteBlockPool(true, enclosingInstance); vectorsPool = new ByteBlockPool(false, enclosingInstance); charPool = new CharBlockPool(enclosingInstance); }
public virtual void TestBasic() { ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random().Next(100))); int NUM_STREAM = AtLeast(100); ByteSliceWriter writer = new ByteSliceWriter(pool); int[] starts = new int[NUM_STREAM]; int[] uptos = new int[NUM_STREAM]; int[] counters = new int[NUM_STREAM]; ByteSliceReader reader = new ByteSliceReader(); for (int ti = 0; ti < 100; ti++) { for (int stream = 0; stream < NUM_STREAM; stream++) { starts[stream] = -1; counters[stream] = 0; } int num = AtLeast(3000); for (int iter = 0; iter < num; iter++) { int stream; if (Random().NextBoolean()) { stream = Random().Next(3); } else { stream = Random().Next(NUM_STREAM); } if (VERBOSE) { Console.WriteLine("write stream=" + stream); } if (starts[stream] == -1) { int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE); starts[stream] = uptos[stream] = spot + pool.ByteOffset; if (VERBOSE) { Console.WriteLine(" init to " + starts[stream]); } } writer.Init(uptos[stream]); int numValue; if (Random().Next(10) == 3) { numValue = Random().Next(100); } else if (Random().Next(5) == 3) { numValue = Random().Next(3); } else { numValue = Random().Next(20); } for (int j = 0; j < numValue; j++) { if (VERBOSE) { Console.WriteLine(" write " + (counters[stream] + j)); } // write some large (incl. negative) ints: writer.WriteVInt(Random().Next()); writer.WriteVInt(counters[stream] + j); } counters[stream] += numValue; uptos[stream] = writer.Address; if (VERBOSE) { Console.WriteLine(" addr now " + uptos[stream]); } } for (int stream = 0; stream < NUM_STREAM; stream++) { if (VERBOSE) { Console.WriteLine(" stream=" + stream + " count=" + counters[stream]); } if (starts[stream] != -1 && starts[stream] != uptos[stream]) { reader.Init(pool, starts[stream], uptos[stream]); for (int j = 0; j < counters[stream]; j++) { reader.ReadVInt(); Assert.AreEqual(j, reader.ReadVInt()); } } } pool.Reset(); } }
public virtual void TestBasic() { ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random().Next(100))); int NUM_STREAM = AtLeast(100); ByteSliceWriter writer = new ByteSliceWriter(pool); int[] starts = new int[NUM_STREAM]; int[] uptos = new int[NUM_STREAM]; int[] counters = new int[NUM_STREAM]; ByteSliceReader reader = new ByteSliceReader(); for (int ti = 0; ti < 100; ti++) { for (int stream = 0; stream < NUM_STREAM; stream++) { starts[stream] = -1; counters[stream] = 0; } int num = AtLeast(3000); for (int iter = 0; iter < num; iter++) { int stream; if (Random().NextBoolean()) { stream = Random().Next(3); } else { stream = Random().Next(NUM_STREAM); } if (VERBOSE) { Console.WriteLine("write stream=" + stream); } if (starts[stream] == -1) { int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE); starts[stream] = uptos[stream] = spot + pool.ByteOffset; if (VERBOSE) { Console.WriteLine(" init to " + starts[stream]); } } writer.Init(uptos[stream]); int numValue; if (Random().Next(10) == 3) { numValue = Random().Next(100); } else if (Random().Next(5) == 3) { numValue = Random().Next(3); } else { numValue = Random().Next(20); } for (int j = 0; j < numValue; j++) { if (VERBOSE) { Console.WriteLine(" write " + (counters[stream] + j)); } // write some large (incl. negative) ints: writer.WriteVInt(Random().Next()); writer.WriteVInt(counters[stream] + j); } counters[stream] += numValue; uptos[stream] = writer.Address; if (VERBOSE) { Console.WriteLine(" addr now " + uptos[stream]); } } for (int stream = 0; stream < NUM_STREAM; stream++) { if (VERBOSE) { Console.WriteLine(" stream=" + stream + " count=" + counters[stream]); } if (starts[stream] != -1 && starts[stream] != uptos[stream]) { reader.Init(pool, starts[stream], uptos[stream]); for (int j = 0; j < counters[stream]; j++) { reader.ReadVInt(); Assert.AreEqual(j, reader.ReadVInt()); } } } pool.Reset(); } }
public ByteSliceWriter(ByteBlockPool pool) { this.pool = pool; }
public virtual void TestBasic() { // LUCENENET specific: NUnit will crash with an OOM if we do the full test // with verbosity enabled. So, making this a manual setting that can be // turned on if, and only if, needed for debugging. If the setting is turned // on, we are decresing the number of iterations by 1/3, which seems to // keep it from crashing. bool isVerbose = false; if (!isVerbose) { Console.WriteLine("Verbosity disabled to keep NUnit from running out of memory - enable manually"); } ByteBlockPool pool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, Random.Next(100))); int NUM_STREAM = AtLeast(100); ByteSliceWriter writer = new ByteSliceWriter(pool); int[] starts = new int[NUM_STREAM]; int[] uptos = new int[NUM_STREAM]; int[] counters = new int[NUM_STREAM]; ByteSliceReader reader = new ByteSliceReader(); for (int ti = 0; ti < 100; ti++) { for (int stream = 0; stream < NUM_STREAM; stream++) { starts[stream] = -1; counters[stream] = 0; } // LUCENENET NOTE: Since upgrading to NUnit 3, this test // will crash if VERBOSE is true because of an OutOfMemoryException. // This not only keeps this test from finishing, it crashes NUnit // and no other tests will run. // So, we need to allocate a smaller size to ensure this // doesn't happen with verbosity enabled. int num = isVerbose ? AtLeast(2000) : AtLeast(3000); for (int iter = 0; iter < num; iter++) { int stream; if (Random.NextBoolean()) { stream = Random.Next(3); } else { stream = Random.Next(NUM_STREAM); } if (isVerbose) { Console.WriteLine("write stream=" + stream); } if (starts[stream] == -1) { int spot = pool.NewSlice(ByteBlockPool.FIRST_LEVEL_SIZE); starts[stream] = uptos[stream] = spot + pool.ByteOffset; if (isVerbose) { Console.WriteLine(" init to " + starts[stream]); } } writer.Init(uptos[stream]); int numValue; if (Random.Next(10) == 3) { numValue = Random.Next(100); } else if (Random.Next(5) == 3) { numValue = Random.Next(3); } else { numValue = Random.Next(20); } for (int j = 0; j < numValue; j++) { if (isVerbose) { Console.WriteLine(" write " + (counters[stream] + j)); } // write some large (incl. negative) ints: writer.WriteVInt32(Random.Next()); writer.WriteVInt32(counters[stream] + j); } counters[stream] += numValue; uptos[stream] = writer.Address; if (isVerbose) { Console.WriteLine(" addr now " + uptos[stream]); } } for (int stream = 0; stream < NUM_STREAM; stream++) { if (isVerbose) { Console.WriteLine(" stream=" + stream + " count=" + counters[stream]); } if (starts[stream] != -1 && starts[stream] != uptos[stream]) { reader.Init(pool, starts[stream], uptos[stream]); for (int j = 0; j < counters[stream]; j++) { reader.ReadVInt32(); Assert.AreEqual(j, reader.ReadVInt32()); } } } pool.Reset(); } }