/// <summary> /// Called for each document addition on the given <see cref="ThreadState"/>s /// <see cref="DocumentsWriterPerThread"/>. /// <para/> /// Note: this method is synchronized by the given /// <see cref="DocumentsWriterFlushControl"/> and it is guaranteed that the calling /// thread holds the lock on the given <see cref="ThreadState"/> /// </summary> public abstract void OnInsert(DocumentsWriterFlushControl control, ThreadState state);
public IteratorAnonymousClass(DocumentsWriterFlushControl outerInstance, int upto)
{
this.outerInstance = outerInstance;
this.upto = upto;
i = 0;
}
/// <summary>
/// Called for each document update on the given <see cref="ThreadState"/>'s
/// <see cref="DocumentsWriterPerThread"/>.
/// <para/>
/// Note: this method is called synchronized on the given
/// <see cref="DocumentsWriterFlushControl"/> and it is guaranteed that the calling
/// thread holds the lock on the given <see cref="ThreadState"/>
/// </summary>
public virtual void OnUpdate(DocumentsWriterFlushControl control, ThreadState state)
{
OnInsert(control, state);
OnDelete(control, state);
}
/// <summary>
/// Marks the most ram consuming active <see cref="DocumentsWriterPerThread"/> flush
/// pending
/// </summary>
protected virtual void MarkLargestWriterPending(DocumentsWriterFlushControl control, ThreadState perThreadState, long currentBytesPerThread)
{
control.SetFlushPending(FindLargestNonPendingWriter(control, perThreadState));
}
protected internal virtual void RunFlushByRam(int numThreads, double maxRamMB, bool ensureNotStalled)
{
int numDocumentsToIndex = 10 + AtLeast(30);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetRAMBufferSizeMB(maxRamMB);
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsFalse(flushPolicy.FlushOnDocCount);
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms);
Assert.IsTrue(flushPolicy.FlushOnRAM);
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
if (flushPolicy.HasMarkedPending)
{
Assert.IsTrue(maxRAMBytes < flushControl.peakActiveBytes);
}
if (ensureNotStalled)
{
Assert.IsFalse(docsWriter.flushControl.stallControl.WasStalled);
}
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes);
dir.Dispose();
}
public virtual void TestRandom()
{
int numThreads = 1 + Random.Next(8);
int numDocumentsToIndex = 50 + AtLeast(70);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
iwc.SetFlushPolicy(flushPolicy);
int numDWPT = 1 + Random.Next(8);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, true);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (flushPolicy.FlushOnRAM && !flushPolicy.FlushOnDocCount && !flushPolicy.FlushOnDeleteTerms)
{
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
if (flushPolicy.HasMarkedPending)
{
assertTrue("max: " + maxRAMBytes + " " + flushControl.peakActiveBytes, maxRAMBytes <= flushControl.peakActiveBytes);
}
}
AssertActiveBytesAfter(flushControl);
writer.Commit();
Assert.AreEqual(0, flushControl.ActiveBytes);
IndexReader r = DirectoryReader.Open(dir);
Assert.AreEqual(numDocumentsToIndex, r.NumDocs);
Assert.AreEqual(numDocumentsToIndex, r.MaxDoc);
if (!flushPolicy.FlushOnRAM)
{
assertFalse("never stall if we don't flush on RAM", docsWriter.flushControl.stallControl.WasStalled);
assertFalse("never block if we don't flush on RAM", docsWriter.flushControl.stallControl.HasBlocked);
}
r.Dispose();
writer.Dispose();
dir.Dispose();
}
public IteratorAnonymousInnerClassHelper(DocumentsWriterFlushControl outerInstance, int upto)
{
this.OuterInstance = outerInstance;
this.Upto = upto;
i = 0;
}