|
public SetMaxBufferedDeleteTerms ( int maxBufferedDeleteTerms ) : |
||
| maxBufferedDeleteTerms | int | |
| return | ||
[Slow] // LUCENENET: occasionally
public virtual void TestStallControl()
{
// LUCENENET specific - disable the test if asserts are not enabled
AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled);
int[] numThreads = new int[] { 4 + Random.Next(8), 1 };
int numDocumentsToIndex = 50 + Random.Next(50);
for (int i = 0; i < numThreads.Length; i++)
{
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
MockDirectoryWrapper dir = NewMockDirectory();
// mock a very slow harddisk sometimes here so that flushing is very slow
dir.Throttling = Throttling.SOMETIMES;
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy();
iwc.SetFlushPolicy(flushPolicy);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numThreads[i] == 1 ? 1 : 2);
iwc.SetIndexerThreadPool(threadPool);
// with such a small ram buffer we should be stalled quiet quickly
iwc.SetRAMBufferSizeMB(0.25);
IndexWriter writer = new IndexWriter(dir, iwc);
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(numDocs, writer, lineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (numThreads[i] == 1)
{
assertFalse("single thread must not block numThreads: " + numThreads[i], docsWriter.flushControl.stallControl.HasBlocked);
}
if (docsWriter.flushControl.peakNetBytes > (2d * iwc.RAMBufferSizeMB * 1024d * 1024d))
{
Assert.IsTrue(docsWriter.flushControl.stallControl.WasStalled);
}
AssertActiveBytesAfter(flushControl);
writer.Dispose(true);
dir.Dispose();
}
}
public virtual void TestFlushDocCount()
{
// LUCENENET specific - disable the test if asserts are not enabled
AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled);
int[] numThreads = new int[] { 2 + AtLeast(1), 1 };
for (int i = 0; i < numThreads.Length; i++)
{
int numDocumentsToIndex = 50 + AtLeast(30);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetMaxBufferedDocs(2 + AtLeast(10));
iwc.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsTrue(flushPolicy.FlushOnDocCount);
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms);
Assert.IsFalse(flushPolicy.FlushOnRAM);
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(numDocs, writer, lineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads[i]);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.peakDocCountWithoutFlush <= iwc.MaxBufferedDocs, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes);
dir.Dispose();
}
}
public virtual void TestInvalidValues()
{
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
// Test IndexDeletionPolicy
Assert.AreEqual(typeof(KeepOnlyLastCommitDeletionPolicy), conf.DelPolicy.GetType());
conf.SetIndexDeletionPolicy(new SnapshotDeletionPolicy(null));
Assert.AreEqual(typeof(SnapshotDeletionPolicy), conf.DelPolicy.GetType());
try
{
conf.SetIndexDeletionPolicy(null);
Assert.Fail();
}
catch (System.ArgumentException e)
{
// ok
}
// Test MergeScheduler
Assert.AreEqual(typeof(ConcurrentMergeScheduler), conf.MergeScheduler.GetType());
conf.SetMergeScheduler(new SerialMergeScheduler());
Assert.AreEqual(typeof(SerialMergeScheduler), conf.MergeScheduler.GetType());
try
{
conf.SetMergeScheduler(null);
Assert.Fail();
}
catch (System.ArgumentException e)
{
// ok
}
// Test Similarity:
// we shouldnt assert what the default is, just that its not null.
Assert.IsTrue(IndexSearcher.DefaultSimilarity == conf.Similarity);
conf.SetSimilarity(new MySimilarity());
Assert.AreEqual(typeof(MySimilarity), conf.Similarity.GetType());
try
{
conf.SetSimilarity(null);
Assert.Fail();
}
catch (System.ArgumentException e)
{
// ok
}
// Test IndexingChain
Assert.IsTrue(DocumentsWriterPerThread.DefaultIndexingChain == conf.IndexingChain);
conf.SetIndexingChain(new MyIndexingChain());
Assert.AreEqual(typeof(MyIndexingChain), conf.IndexingChain.GetType());
try
{
conf.SetIndexingChain(null);
Assert.Fail();
}
catch (System.ArgumentException e)
{
// ok
}
try
{
conf.SetMaxBufferedDeleteTerms(0);
Assert.Fail("should not have succeeded to set maxBufferedDeleteTerms to 0");
}
catch (System.ArgumentException e)
{
// this is expected
}
try
{
conf.SetMaxBufferedDocs(1);
Assert.Fail("should not have succeeded to set maxBufferedDocs to 1");
}
catch (System.ArgumentException e)
{
// this is expected
}
try
{
// Disable both MAX_BUF_DOCS and RAM_SIZE_MB
conf.SetMaxBufferedDocs(4);
conf.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
conf.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
Assert.Fail("should not have succeeded to disable maxBufferedDocs when ramBufferSizeMB is disabled as well");
}
catch (System.ArgumentException e)
{
// this is expected
}
conf.SetRAMBufferSizeMB(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
conf.SetMaxBufferedDocs(IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS);
try
{
conf.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
Assert.Fail("should not have succeeded to disable ramBufferSizeMB when maxBufferedDocs is disabled as well");
}
catch (System.ArgumentException e)
{
// this is expected
}
// Test setReaderTermsIndexDivisor
try
{
conf.SetReaderTermsIndexDivisor(0);
Assert.Fail("should not have succeeded to set termsIndexDivisor to 0");
}
catch (System.ArgumentException e)
{
// this is expected
}
// Setting to -1 is ok
conf.SetReaderTermsIndexDivisor(-1);
try
{
conf.SetReaderTermsIndexDivisor(-2);
Assert.Fail("should not have succeeded to set termsIndexDivisor to < -1");
}
catch (System.ArgumentException e)
{
// this is expected
}
try
{
conf.SetRAMPerThreadHardLimitMB(2048);
Assert.Fail("should not have succeeded to set RAMPerThreadHardLimitMB to >= 2048");
}
catch (System.ArgumentException e)
{
// this is expected
}
try
{
conf.SetRAMPerThreadHardLimitMB(0);
Assert.Fail("should not have succeeded to set RAMPerThreadHardLimitMB to 0");
}
catch (System.ArgumentException e)
{
// this is expected
}
// Test MergePolicy
Assert.AreEqual(typeof(TieredMergePolicy), conf.MergePolicy.GetType());
conf.SetMergePolicy(new LogDocMergePolicy());
Assert.AreEqual(typeof(LogDocMergePolicy), conf.MergePolicy.GetType());
try
{
conf.SetMergePolicy(null);
Assert.Fail();
}
catch (System.ArgumentException e)
{
// ok
}
}
protected internal virtual void RunFlushByRam(int numThreads, double maxRamMB, bool ensureNotStalled)
{
int numDocumentsToIndex = 10 + AtLeast(30);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetRAMBufferSizeMB(maxRamMB);
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsFalse(flushPolicy.FlushOnDocCount);
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms);
Assert.IsTrue(flushPolicy.FlushOnRAM);
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
if (flushPolicy.HasMarkedPending)
{
Assert.IsTrue(maxRAMBytes < flushControl.peakActiveBytes);
}
if (ensureNotStalled)
{
Assert.IsFalse(docsWriter.flushControl.stallControl.WasStalled);
}
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes);
dir.Dispose();
}
public virtual void TestInvalidValues()
{
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
// Test IndexDeletionPolicy
Assert.AreEqual(typeof(KeepOnlyLastCommitDeletionPolicy), conf.IndexDeletionPolicy.GetType());
conf.SetIndexDeletionPolicy(new SnapshotDeletionPolicy(null));
Assert.AreEqual(typeof(SnapshotDeletionPolicy), conf.IndexDeletionPolicy.GetType());
try
{
conf.SetIndexDeletionPolicy(null);
Assert.Fail();
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// ok
}
// Test MergeScheduler
#if !FEATURE_CONCURRENTMERGESCHEDULER
Assert.AreEqual(typeof(TaskMergeScheduler), conf.MergeScheduler.GetType());
#else
Assert.AreEqual(typeof(ConcurrentMergeScheduler), conf.MergeScheduler.GetType());
#endif
conf.SetMergeScheduler(new SerialMergeScheduler());
Assert.AreEqual(typeof(SerialMergeScheduler), conf.MergeScheduler.GetType());
try
{
conf.SetMergeScheduler(null);
Assert.Fail();
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// ok
}
// Test Similarity:
// we shouldnt assert what the default is, just that its not null.
Assert.IsTrue(IndexSearcher.DefaultSimilarity == conf.Similarity);
conf.SetSimilarity(new MySimilarity());
Assert.AreEqual(typeof(MySimilarity), conf.Similarity.GetType());
try
{
conf.SetSimilarity(null);
Assert.Fail();
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// ok
}
// Test IndexingChain
Assert.IsTrue(DocumentsWriterPerThread.DefaultIndexingChain == conf.IndexingChain);
conf.SetIndexingChain(new MyIndexingChain());
Assert.AreEqual(typeof(MyIndexingChain), conf.IndexingChain.GetType());
try
{
conf.SetIndexingChain(null);
Assert.Fail();
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// ok
}
try
{
conf.SetMaxBufferedDeleteTerms(0);
Assert.Fail("should not have succeeded to set maxBufferedDeleteTerms to 0");
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// this is expected
}
try
{
conf.SetMaxBufferedDocs(1);
Assert.Fail("should not have succeeded to set maxBufferedDocs to 1");
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// this is expected
}
try
{
// Disable both MAX_BUF_DOCS and RAM_SIZE_MB
conf.SetMaxBufferedDocs(4);
conf.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
conf.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
Assert.Fail("should not have succeeded to disable maxBufferedDocs when ramBufferSizeMB is disabled as well");
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// this is expected
}
conf.SetRAMBufferSizeMB(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
conf.SetMaxBufferedDocs(IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS);
try
{
conf.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
Assert.Fail("should not have succeeded to disable ramBufferSizeMB when maxBufferedDocs is disabled as well");
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// this is expected
}
// Test setReaderTermsIndexDivisor
try
{
conf.SetReaderTermsIndexDivisor(0);
Assert.Fail("should not have succeeded to set termsIndexDivisor to 0");
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// this is expected
}
// Setting to -1 is ok
conf.SetReaderTermsIndexDivisor(-1);
try
{
conf.SetReaderTermsIndexDivisor(-2);
Assert.Fail("should not have succeeded to set termsIndexDivisor to < -1");
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// this is expected
}
try
{
conf.SetRAMPerThreadHardLimitMB(2048);
Assert.Fail("should not have succeeded to set RAMPerThreadHardLimitMB to >= 2048");
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// this is expected
}
try
{
conf.SetRAMPerThreadHardLimitMB(0);
Assert.Fail("should not have succeeded to set RAMPerThreadHardLimitMB to 0");
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// this is expected
}
// Test MergePolicy
Assert.AreEqual(typeof(TieredMergePolicy), conf.MergePolicy.GetType());
conf.SetMergePolicy(new LogDocMergePolicy());
Assert.AreEqual(typeof(LogDocMergePolicy), conf.MergePolicy.GetType());
try
{
conf.SetMergePolicy(null);
Assert.Fail();
}
#pragma warning disable 168
catch (System.ArgumentException e)
#pragma warning restore 168
{
// ok
}
}