public override object Clone()
{
ThreadAffinityDocumentsWriterThreadPool clone = (ThreadAffinityDocumentsWriterThreadPool)base.Clone();
clone.threadBindings = new ConcurrentDictionary <Thread, ThreadState>();
return(clone);
}
protected internal virtual void RunFlushByRam(int numThreads, double maxRamMB, bool ensureNotStalled)
{
int numDocumentsToIndex = 10 + AtLeast(30);
AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetRAMBufferSizeMB(maxRamMB);
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsFalse(flushPolicy.FlushOnDocCount());
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms());
Assert.IsTrue(flushPolicy.FlushOnRAM());
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.FlushControl;
Assert.AreEqual(0, flushControl.FlushBytes(), " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.AreEqual(0, flushControl.FlushBytes(), " all flushes must be due numThreads=" + numThreads);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs());
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
if (flushPolicy.HasMarkedPending)
{
Assert.IsTrue(maxRAMBytes < flushControl.PeakActiveBytes);
}
if (ensureNotStalled)
{
Assert.IsFalse(docsWriter.FlushControl.StallControl.WasStalled());
}
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes());
dir.Dispose();
}
[Slow] // LUCENENET: occasionally
public virtual void TestStallControl()
{
// LUCENENET specific - disable the test if asserts are not enabled
AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled);
int[] numThreads = new int[] { 4 + Random.Next(8), 1 };
int numDocumentsToIndex = 50 + Random.Next(50);
for (int i = 0; i < numThreads.Length; i++)
{
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
MockDirectoryWrapper dir = NewMockDirectory();
// mock a very slow harddisk sometimes here so that flushing is very slow
dir.Throttling = Throttling.SOMETIMES;
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy();
iwc.SetFlushPolicy(flushPolicy);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numThreads[i] == 1 ? 1 : 2);
iwc.SetIndexerThreadPool(threadPool);
// with such a small ram buffer we should be stalled quiet quickly
iwc.SetRAMBufferSizeMB(0.25);
IndexWriter writer = new IndexWriter(dir, iwc);
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(numDocs, writer, lineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (numThreads[i] == 1)
{
assertFalse("single thread must not block numThreads: " + numThreads[i], docsWriter.flushControl.stallControl.HasBlocked);
}
if (docsWriter.flushControl.peakNetBytes > (2d * iwc.RAMBufferSizeMB * 1024d * 1024d))
{
Assert.IsTrue(docsWriter.flushControl.stallControl.WasStalled);
}
AssertActiveBytesAfter(flushControl);
writer.Dispose(true);
dir.Dispose();
}
}
public virtual void TestFlushDocCount()
{
// LUCENENET specific - disable the test if asserts are not enabled
AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled);
int[] numThreads = new int[] { 2 + AtLeast(1), 1 };
for (int i = 0; i < numThreads.Length; i++)
{
int numDocumentsToIndex = 50 + AtLeast(30);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetMaxBufferedDocs(2 + AtLeast(10));
iwc.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsTrue(flushPolicy.FlushOnDocCount);
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms);
Assert.IsFalse(flushPolicy.FlushOnRAM);
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(numDocs, writer, lineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads[i]);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.peakDocCountWithoutFlush <= iwc.MaxBufferedDocs, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes);
dir.Dispose();
}
}
protected internal virtual void RunFlushByRam(int numThreads, double maxRamMB, bool ensureNotStalled)
{
int numDocumentsToIndex = 10 + AtLeast(30);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetRAMBufferSizeMB(maxRamMB);
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsFalse(flushPolicy.FlushOnDocCount);
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms);
Assert.IsTrue(flushPolicy.FlushOnRAM);
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
if (flushPolicy.HasMarkedPending)
{
Assert.IsTrue(maxRAMBytes < flushControl.peakActiveBytes);
}
if (ensureNotStalled)
{
Assert.IsFalse(docsWriter.flushControl.stallControl.WasStalled);
}
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes);
dir.Dispose();
}
public virtual void TestRandom()
{
int numThreads = 1 + Random.Next(8);
int numDocumentsToIndex = 50 + AtLeast(70);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
iwc.SetFlushPolicy(flushPolicy);
int numDWPT = 1 + Random.Next(8);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, true);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (flushPolicy.FlushOnRAM && !flushPolicy.FlushOnDocCount && !flushPolicy.FlushOnDeleteTerms)
{
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
if (flushPolicy.HasMarkedPending)
{
assertTrue("max: " + maxRAMBytes + " " + flushControl.peakActiveBytes, maxRAMBytes <= flushControl.peakActiveBytes);
}
}
AssertActiveBytesAfter(flushControl);
writer.Commit();
Assert.AreEqual(0, flushControl.ActiveBytes);
IndexReader r = DirectoryReader.Open(dir);
Assert.AreEqual(numDocumentsToIndex, r.NumDocs);
Assert.AreEqual(numDocumentsToIndex, r.MaxDoc);
if (!flushPolicy.FlushOnRAM)
{
assertFalse("never stall if we don't flush on RAM", docsWriter.flushControl.stallControl.WasStalled);
assertFalse("never block if we don't flush on RAM", docsWriter.flushControl.stallControl.HasBlocked);
}
r.Dispose();
writer.Dispose();
dir.Dispose();
}
public virtual void TestStallControl()
{
int[] numThreads = new int[] { 4 + Random().Next(8), 1 };
int numDocumentsToIndex = 50 + Random().Next(50);
for (int i = 0; i < numThreads.Length; i++)
{
AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex);
MockDirectoryWrapper dir = NewMockDirectory();
// mock a very slow harddisk sometimes here so that flushing is very slow
dir.Throttling = MockDirectoryWrapper.Throttling_e.SOMETIMES;
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy();
iwc.SetFlushPolicy(flushPolicy);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numThreads[i] == 1 ? 1 : 2);
iwc.SetIndexerThreadPool(threadPool);
// with such a small ram buffer we should be stalled quiet quickly
iwc.SetRAMBufferSizeMB(0.25);
IndexWriter writer = new IndexWriter(dir, iwc);
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads[i], writer, LineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.FlushControl;
Assert.AreEqual(0, flushControl.FlushBytes(), " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs());
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (numThreads[i] == 1)
{
assertFalse("single thread must not block numThreads: " + numThreads[i], docsWriter.FlushControl.StallControl.HasBlocked());
}
if (docsWriter.FlushControl.PeakNetBytes > (2d * iwc.RAMBufferSizeMB * 1024d * 1024d))
{
Assert.IsTrue(docsWriter.FlushControl.StallControl.WasStalled());
}
AssertActiveBytesAfter(flushControl);
writer.Dispose(true);
dir.Dispose();
}
}
public virtual void TestRandom()
{
int numThreads = 1 + Random().Next(8);
int numDocumentsToIndex = 50 + AtLeast(70);
AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex);
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
iwc.SetFlushPolicy(flushPolicy);
int numDWPT = 1 + Random().Next(8);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.FlushControl;
Assert.AreEqual(0, flushControl.FlushBytes(), " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, true);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes(), " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs());
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (flushPolicy.FlushOnRAM() && !flushPolicy.FlushOnDocCount() && !flushPolicy.FlushOnDeleteTerms())
{
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
if (flushPolicy.HasMarkedPending)
{
assertTrue("max: " + maxRAMBytes + " " + flushControl.PeakActiveBytes, maxRAMBytes <= flushControl.PeakActiveBytes);
}
}
AssertActiveBytesAfter(flushControl);
writer.Commit();
Assert.AreEqual(0, flushControl.ActiveBytes());
IndexReader r = DirectoryReader.Open(dir);
Assert.AreEqual(numDocumentsToIndex, r.NumDocs);
Assert.AreEqual(numDocumentsToIndex, r.MaxDoc);
if (!flushPolicy.FlushOnRAM())
{
assertFalse("never stall if we don't flush on RAM", docsWriter.FlushControl.StallControl.WasStalled());
assertFalse("never block if we don't flush on RAM", docsWriter.FlushControl.StallControl.HasBlocked());
}
r.Dispose();
writer.Dispose();
dir.Dispose();
}
public virtual void TestFlushDocCount()
{
int[] numThreads = new int[] { 2 + AtLeast(1), 1 };
for (int i = 0; i < numThreads.Length; i++)
{
int numDocumentsToIndex = 50 + AtLeast(30);
AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetMaxBufferedDocs(2 + AtLeast(10));
iwc.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsTrue(flushPolicy.FlushOnDocCount());
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms());
Assert.IsFalse(flushPolicy.FlushOnRAM());
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.FlushControl;
Assert.AreEqual(0, flushControl.FlushBytes(), " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads[i], writer, LineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes(), " all flushes must be due numThreads=" + numThreads[i]);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs());
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.PeakDocCountWithoutFlush <= iwc.MaxBufferedDocs, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes());
dir.Dispose();
}
}
