public SetMaxBufferedDeleteTerms ( int maxBufferedDeleteTerms ) : |
||
maxBufferedDeleteTerms | int | |
Résultat |
[Slow] // LUCENENET: occasionally public virtual void TestStallControl() { // LUCENENET specific - disable the test if asserts are not enabled AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled); int[] numThreads = new int[] { 4 + Random.Next(8), 1 }; int numDocumentsToIndex = 50 + Random.Next(50); for (int i = 0; i < numThreads.Length; i++) { AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex); MockDirectoryWrapper dir = NewMockDirectory(); // mock a very slow harddisk sometimes here so that flushing is very slow dir.Throttling = Throttling.SOMETIMES; IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)); iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy(); iwc.SetFlushPolicy(flushPolicy); DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numThreads[i] == 1 ? 1 : 2); iwc.SetIndexerThreadPool(threadPool); // with such a small ram buffer we should be stalled quiet quickly iwc.SetRAMBufferSizeMB(0.25); IndexWriter writer = new IndexWriter(dir, iwc); IndexThread[] threads = new IndexThread[numThreads[i]]; for (int x = 0; x < threads.Length; x++) { threads[x] = new IndexThread(numDocs, writer, lineDocFile, false); threads[x].Start(); } for (int x = 0; x < threads.Length; x++) { threads[x].Join(); } DocumentsWriter docsWriter = writer.DocsWriter; Assert.IsNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.flushControl; Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due"); Assert.AreEqual(numDocumentsToIndex, writer.NumDocs); Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc); if (numThreads[i] == 1) { assertFalse("single thread must not block numThreads: " + numThreads[i], docsWriter.flushControl.stallControl.HasBlocked); } if (docsWriter.flushControl.peakNetBytes > (2d * iwc.RAMBufferSizeMB * 1024d * 1024d)) { Assert.IsTrue(docsWriter.flushControl.stallControl.WasStalled); } AssertActiveBytesAfter(flushControl); writer.Dispose(true); dir.Dispose(); } }
public virtual void TestFlushDocCount() { // LUCENENET specific - disable the test if asserts are not enabled AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled); int[] numThreads = new int[] { 2 + AtLeast(1), 1 }; for (int i = 0; i < numThreads.Length; i++) { int numDocumentsToIndex = 50 + AtLeast(30); AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex); Directory dir = NewDirectory(); MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetFlushPolicy(flushPolicy); int numDWPT = 1 + AtLeast(2); DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numDWPT); iwc.SetIndexerThreadPool(threadPool); iwc.SetMaxBufferedDocs(2 + AtLeast(10)); iwc.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); IndexWriter writer = new IndexWriter(dir, iwc); flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy; Assert.IsTrue(flushPolicy.FlushOnDocCount); Assert.IsFalse(flushPolicy.FlushOnDeleteTerms); Assert.IsFalse(flushPolicy.FlushOnRAM); DocumentsWriter docsWriter = writer.DocsWriter; Assert.IsNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.flushControl; Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init"); IndexThread[] threads = new IndexThread[numThreads[i]]; for (int x = 0; x < threads.Length; x++) { threads[x] = new IndexThread(numDocs, writer, lineDocFile, false); threads[x].Start(); } for (int x = 0; x < threads.Length; x++) { threads[x].Join(); } Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads[i]); Assert.AreEqual(numDocumentsToIndex, writer.NumDocs); Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc); Assert.IsTrue(flushPolicy.peakDocCountWithoutFlush <= iwc.MaxBufferedDocs, "peak bytes without flush exceeded watermark"); AssertActiveBytesAfter(flushControl); writer.Dispose(); Assert.AreEqual(0, flushControl.ActiveBytes); dir.Dispose(); } }
public virtual void TestInvalidValues() { IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); // Test IndexDeletionPolicy Assert.AreEqual(typeof(KeepOnlyLastCommitDeletionPolicy), conf.DelPolicy.GetType()); conf.SetIndexDeletionPolicy(new SnapshotDeletionPolicy(null)); Assert.AreEqual(typeof(SnapshotDeletionPolicy), conf.DelPolicy.GetType()); try { conf.SetIndexDeletionPolicy(null); Assert.Fail(); } catch (System.ArgumentException e) { // ok } // Test MergeScheduler Assert.AreEqual(typeof(ConcurrentMergeScheduler), conf.MergeScheduler.GetType()); conf.SetMergeScheduler(new SerialMergeScheduler()); Assert.AreEqual(typeof(SerialMergeScheduler), conf.MergeScheduler.GetType()); try { conf.SetMergeScheduler(null); Assert.Fail(); } catch (System.ArgumentException e) { // ok } // Test Similarity: // we shouldnt assert what the default is, just that its not null. Assert.IsTrue(IndexSearcher.DefaultSimilarity == conf.Similarity); conf.SetSimilarity(new MySimilarity()); Assert.AreEqual(typeof(MySimilarity), conf.Similarity.GetType()); try { conf.SetSimilarity(null); Assert.Fail(); } catch (System.ArgumentException e) { // ok } // Test IndexingChain Assert.IsTrue(DocumentsWriterPerThread.DefaultIndexingChain == conf.IndexingChain); conf.SetIndexingChain(new MyIndexingChain()); Assert.AreEqual(typeof(MyIndexingChain), conf.IndexingChain.GetType()); try { conf.SetIndexingChain(null); Assert.Fail(); } catch (System.ArgumentException e) { // ok } try { conf.SetMaxBufferedDeleteTerms(0); Assert.Fail("should not have succeeded to set maxBufferedDeleteTerms to 0"); } catch (System.ArgumentException e) { // this is expected } try { conf.SetMaxBufferedDocs(1); Assert.Fail("should not have succeeded to set maxBufferedDocs to 1"); } catch (System.ArgumentException e) { // this is expected } try { // Disable both MAX_BUF_DOCS and RAM_SIZE_MB conf.SetMaxBufferedDocs(4); conf.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); conf.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); Assert.Fail("should not have succeeded to disable maxBufferedDocs when ramBufferSizeMB is disabled as well"); } catch (System.ArgumentException e) { // this is expected } conf.SetRAMBufferSizeMB(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB); conf.SetMaxBufferedDocs(IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS); try { conf.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); Assert.Fail("should not have succeeded to disable ramBufferSizeMB when maxBufferedDocs is disabled as well"); } catch (System.ArgumentException e) { // this is expected } // Test setReaderTermsIndexDivisor try { conf.SetReaderTermsIndexDivisor(0); Assert.Fail("should not have succeeded to set termsIndexDivisor to 0"); } catch (System.ArgumentException e) { // this is expected } // Setting to -1 is ok conf.SetReaderTermsIndexDivisor(-1); try { conf.SetReaderTermsIndexDivisor(-2); Assert.Fail("should not have succeeded to set termsIndexDivisor to < -1"); } catch (System.ArgumentException e) { // this is expected } try { conf.SetRAMPerThreadHardLimitMB(2048); Assert.Fail("should not have succeeded to set RAMPerThreadHardLimitMB to >= 2048"); } catch (System.ArgumentException e) { // this is expected } try { conf.SetRAMPerThreadHardLimitMB(0); Assert.Fail("should not have succeeded to set RAMPerThreadHardLimitMB to 0"); } catch (System.ArgumentException e) { // this is expected } // Test MergePolicy Assert.AreEqual(typeof(TieredMergePolicy), conf.MergePolicy.GetType()); conf.SetMergePolicy(new LogDocMergePolicy()); Assert.AreEqual(typeof(LogDocMergePolicy), conf.MergePolicy.GetType()); try { conf.SetMergePolicy(null); Assert.Fail(); } catch (System.ArgumentException e) { // ok } }
protected internal virtual void RunFlushByRam(int numThreads, double maxRamMB, bool ensureNotStalled) { int numDocumentsToIndex = 10 + AtLeast(30); AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex); Directory dir = NewDirectory(); MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); MockAnalyzer analyzer = new MockAnalyzer(Random); analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetFlushPolicy(flushPolicy); int numDWPT = 1 + AtLeast(2); DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT); iwc.SetIndexerThreadPool(threadPool); iwc.SetRAMBufferSizeMB(maxRamMB); iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); IndexWriter writer = new IndexWriter(dir, iwc); flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy; Assert.IsFalse(flushPolicy.FlushOnDocCount); Assert.IsFalse(flushPolicy.FlushOnDeleteTerms); Assert.IsTrue(flushPolicy.FlushOnRAM); DocumentsWriter docsWriter = writer.DocsWriter; Assert.IsNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.flushControl; Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init"); IndexThread[] threads = new IndexThread[numThreads]; for (int x = 0; x < threads.Length; x++) { threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, false); threads[x].Start(); } for (int x = 0; x < threads.Length; x++) { threads[x].Join(); } long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0); Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads); Assert.AreEqual(numDocumentsToIndex, writer.NumDocs); Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc); Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark"); AssertActiveBytesAfter(flushControl); if (flushPolicy.HasMarkedPending) { Assert.IsTrue(maxRAMBytes < flushControl.peakActiveBytes); } if (ensureNotStalled) { Assert.IsFalse(docsWriter.flushControl.stallControl.WasStalled); } writer.Dispose(); Assert.AreEqual(0, flushControl.ActiveBytes); dir.Dispose(); }
public virtual void TestInvalidValues() { IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); // Test IndexDeletionPolicy Assert.AreEqual(typeof(KeepOnlyLastCommitDeletionPolicy), conf.IndexDeletionPolicy.GetType()); conf.SetIndexDeletionPolicy(new SnapshotDeletionPolicy(null)); Assert.AreEqual(typeof(SnapshotDeletionPolicy), conf.IndexDeletionPolicy.GetType()); try { conf.SetIndexDeletionPolicy(null); Assert.Fail(); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // ok } // Test MergeScheduler #if !FEATURE_CONCURRENTMERGESCHEDULER Assert.AreEqual(typeof(TaskMergeScheduler), conf.MergeScheduler.GetType()); #else Assert.AreEqual(typeof(ConcurrentMergeScheduler), conf.MergeScheduler.GetType()); #endif conf.SetMergeScheduler(new SerialMergeScheduler()); Assert.AreEqual(typeof(SerialMergeScheduler), conf.MergeScheduler.GetType()); try { conf.SetMergeScheduler(null); Assert.Fail(); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // ok } // Test Similarity: // we shouldnt assert what the default is, just that its not null. Assert.IsTrue(IndexSearcher.DefaultSimilarity == conf.Similarity); conf.SetSimilarity(new MySimilarity()); Assert.AreEqual(typeof(MySimilarity), conf.Similarity.GetType()); try { conf.SetSimilarity(null); Assert.Fail(); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // ok } // Test IndexingChain Assert.IsTrue(DocumentsWriterPerThread.DefaultIndexingChain == conf.IndexingChain); conf.SetIndexingChain(new MyIndexingChain()); Assert.AreEqual(typeof(MyIndexingChain), conf.IndexingChain.GetType()); try { conf.SetIndexingChain(null); Assert.Fail(); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // ok } try { conf.SetMaxBufferedDeleteTerms(0); Assert.Fail("should not have succeeded to set maxBufferedDeleteTerms to 0"); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // this is expected } try { conf.SetMaxBufferedDocs(1); Assert.Fail("should not have succeeded to set maxBufferedDocs to 1"); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // this is expected } try { // Disable both MAX_BUF_DOCS and RAM_SIZE_MB conf.SetMaxBufferedDocs(4); conf.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); conf.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); Assert.Fail("should not have succeeded to disable maxBufferedDocs when ramBufferSizeMB is disabled as well"); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // this is expected } conf.SetRAMBufferSizeMB(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB); conf.SetMaxBufferedDocs(IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS); try { conf.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); Assert.Fail("should not have succeeded to disable ramBufferSizeMB when maxBufferedDocs is disabled as well"); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // this is expected } // Test setReaderTermsIndexDivisor try { conf.SetReaderTermsIndexDivisor(0); Assert.Fail("should not have succeeded to set termsIndexDivisor to 0"); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // this is expected } // Setting to -1 is ok conf.SetReaderTermsIndexDivisor(-1); try { conf.SetReaderTermsIndexDivisor(-2); Assert.Fail("should not have succeeded to set termsIndexDivisor to < -1"); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // this is expected } try { conf.SetRAMPerThreadHardLimitMB(2048); Assert.Fail("should not have succeeded to set RAMPerThreadHardLimitMB to >= 2048"); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // this is expected } try { conf.SetRAMPerThreadHardLimitMB(0); Assert.Fail("should not have succeeded to set RAMPerThreadHardLimitMB to 0"); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // this is expected } // Test MergePolicy Assert.AreEqual(typeof(TieredMergePolicy), conf.MergePolicy.GetType()); conf.SetMergePolicy(new LogDocMergePolicy()); Assert.AreEqual(typeof(LogDocMergePolicy), conf.MergePolicy.GetType()); try { conf.SetMergePolicy(null); Assert.Fail(); } #pragma warning disable 168 catch (System.ArgumentException e) #pragma warning restore 168 { // ok } }