public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHash termsHash, TermsHash nextTermsHash, FieldInfo fieldInfo)
{
intPool = termsHash.intPool;
bytePool = termsHash.bytePool;
termBytePool = termsHash.termBytePool;
docState = termsHash.docState;
this.termsHash = termsHash;
bytesUsed = termsHash.bytesUsed;
fieldState = docInverterPerField.fieldState;
this.consumer = termsHash.consumer.AddField(this, fieldInfo);
PostingsBytesStartArray byteStarts = new PostingsBytesStartArray(this, bytesUsed);
bytesHash = new BytesRefHash(termBytePool, HASH_INIT_SIZE, byteStarts);
streamCount = consumer.StreamCount;
numPostingInt = 2 * streamCount;
this.fieldInfo = fieldInfo;
if (nextTermsHash != null)
{
nextPerField = (TermsHashPerField)nextTermsHash.AddField(docInverterPerField, fieldInfo);
}
else
{
nextPerField = null;
}
}
public virtual void TestSingleWriterReader()
{
Counter bytesUsed = Util.Counter.NewCounter();
Int32BlockPool pool = new Int32BlockPool(new ByteTrackingAllocator(bytesUsed));
for (int j = 0; j < 2; j++)
{
Int32BlockPool.SliceWriter writer = new Int32BlockPool.SliceWriter(pool);
int start = writer.StartNewSlice();
int num = AtLeast(100);
for (int i = 0; i < num; i++)
{
writer.WriteInt32(i);
}
int upto = writer.CurrentOffset;
Int32BlockPool.SliceReader reader = new Int32BlockPool.SliceReader(pool);
reader.Reset(start, upto);
for (int i = 0; i < num; i++)
{
Assert.AreEqual(i, reader.ReadInt32());
}
Assert.IsTrue(reader.IsEndOfSlice);
if (Random.NextBoolean())
{
pool.Reset(true, false);
Assert.AreEqual(0, bytesUsed);
}
else
{
pool.Reset(true, true);
Assert.AreEqual(Int32BlockPool.INT32_BLOCK_SIZE * RamUsageEstimator.NUM_BYTES_INT32, bytesUsed);
}
}
}
public virtual void TestMultipleWriterReader()
{
Counter bytesUsed = Util.Counter.NewCounter();
Int32BlockPool pool = new Int32BlockPool(new ByteTrackingAllocator(bytesUsed));
for (int j = 0; j < 2; j++)
{
IList <StartEndAndValues> holders = new JCG.List <StartEndAndValues>();
int num = AtLeast(4);
for (int i = 0; i < num; i++)
{
holders.Add(new StartEndAndValues(Random.Next(1000)));
}
Int32BlockPool.SliceWriter writer = new Int32BlockPool.SliceWriter(pool);
Int32BlockPool.SliceReader reader = new Int32BlockPool.SliceReader(pool);
int numValuesToWrite = AtLeast(10000);
for (int i = 0; i < numValuesToWrite; i++)
{
StartEndAndValues values = holders[Random.Next(holders.Count)];
if (values.valueCount == 0)
{
values.start = writer.StartNewSlice();
}
else
{
writer.Reset(values.end);
}
writer.WriteInt32(values.NextValue());
values.end = writer.CurrentOffset;
if (Random.Next(5) == 0)
{
// pick one and reader the ints
AssertReader(reader, holders[Random.Next(holders.Count)]);
}
}
while (holders.Count > 0)
{
int randIndex = Random.Next(holders.Count);
StartEndAndValues values = holders[randIndex];
holders.RemoveAt(randIndex);
AssertReader(reader, values);
}
if (Random.NextBoolean())
{
pool.Reset(true, false);
Assert.AreEqual(0, bytesUsed);
}
else
{
pool.Reset(true, true);
Assert.AreEqual(Int32BlockPool.INT32_BLOCK_SIZE * RamUsageEstimator.NUM_BYTES_INT32, bytesUsed);
}
}
}
/// <summary>
/// Expert: This constructor accepts an upper limit for the number of bytes that should be reused if this instance is <see cref="Reset()"/>.
/// </summary>
/// <param name="storeOffsets"> <c>true</c> if offsets should be stored </param>
/// <param name="maxReusedBytes"> the number of bytes that should remain in the internal memory pools after <see cref="Reset()"/> is called </param>
internal MemoryIndex(bool storeOffsets, long maxReusedBytes)
{
this.storeOffsets = storeOffsets;
this.bytesUsed = Counter.NewCounter();
int maxBufferedByteBlocks = (int)((maxReusedBytes / 2) / ByteBlockPool.BYTE_BLOCK_SIZE);
int maxBufferedIntBlocks = (int)((maxReusedBytes - (maxBufferedByteBlocks * ByteBlockPool.BYTE_BLOCK_SIZE)) / (Int32BlockPool.INT32_BLOCK_SIZE * RamUsageEstimator.NUM_BYTES_INT32));
Debug.Assert((maxBufferedByteBlocks * ByteBlockPool.BYTE_BLOCK_SIZE) + (maxBufferedIntBlocks * Int32BlockPool.INT32_BLOCK_SIZE * RamUsageEstimator.NUM_BYTES_INT32) <= maxReusedBytes);
byteBlockPool = new ByteBlockPool(new RecyclingByteBlockAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, maxBufferedByteBlocks, bytesUsed));
intBlockPool = new Int32BlockPool(new RecyclingInt32BlockAllocator(Int32BlockPool.INT32_BLOCK_SIZE, maxBufferedIntBlocks, bytesUsed));
postingsWriter = new Int32BlockPool.SliceWriter(intBlockPool);
}
public TermsHash(DocumentsWriterPerThread docWriter, TermsHashConsumer consumer, bool trackAllocations, TermsHash nextTermsHash)
{
this.docState = docWriter.docState;
this.consumer = consumer;
this.trackAllocations = trackAllocations;
this.nextTermsHash = nextTermsHash;
this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.NewCounter();
intPool = new Int32BlockPool(docWriter.intBlockAllocator);
bytePool = new ByteBlockPool(docWriter.byteBlockAllocator);
if (nextTermsHash != null)
{
// We are primary
primary = true;
termBytePool = bytePool;
nextTermsHash.termBytePool = bytePool;
}
else
{
primary = false;
}
}
