public virtual void TestRandomExceptionsThreads() { random = new Random((int)(DateTime.Now.Ticks & 0x7fffffff)); MockRAMDirectory dir = new MockRAMDirectory(); MockIndexWriter writer = new MockIndexWriter(this, dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); ((ConcurrentMergeScheduler)writer.MergeScheduler).SetSuppressExceptions(); //writer.setMaxBufferedDocs(10); writer.SetRAMBufferSizeMB(0.2); if (DEBUG) { System.IO.StreamWriter temp_writer; temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding); temp_writer.AutoFlush = true; writer.SetInfoStream(temp_writer, null); } int NUM_THREADS = 4; IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(this, i, writer); threads[i].Start(); } for (int i = 0; i < NUM_THREADS; i++) { threads[i].Join(); } for (int i = 0; i < NUM_THREADS; i++) { Assert.IsNull(threads[i].failure, "thread " + threads[i].Name + ": hit unexpected failure"); } writer.Commit(null); try { writer.Close(); } catch (System.Exception t) { System.Console.Out.WriteLine("exception during close:"); System.Console.Out.WriteLine(t.StackTrace); writer.Rollback(null); } // Confirm that when doc hits exception partway through tokenization, it's deleted: IndexReader r2 = IndexReader.Open((Directory)dir, true, null); int count = r2.DocFreq(new Term("content4", "aaa"), null); int count2 = r2.DocFreq(new Term("content4", "ddd"), null); Assert.AreEqual(count, count2); r2.Close(); _TestUtil.CheckIndex(dir); }
public static void Main(System.String[] args) { bool readOnly = false; bool add = false; for (int i = 0; i < args.Length; i++) { if ("-ro".Equals(args[i])) { readOnly = true; } if ("-add".Equals(args[i])) { add = true; } } System.IO.FileInfo indexDir = new System.IO.FileInfo("index"); bool tmpBool; if (System.IO.File.Exists(indexDir.FullName)) { tmpBool = true; } else { tmpBool = System.IO.Directory.Exists(indexDir.FullName); } if (!tmpBool) { System.IO.Directory.CreateDirectory(indexDir.FullName); } IndexReader.Unlock(FSDirectory.GetDirectory(indexDir, false)); if (!readOnly) { IndexWriter writer = new IndexWriter(indexDir, ANALYZER, !add); SupportClass.ThreadClass indexerThread = new IndexerThread(writer); indexerThread.Start(); System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000)); } SearcherThread searcherThread1 = new SearcherThread(false); searcherThread1.Start(); SEARCHER = new IndexSearcher(indexDir.ToString()); SearcherThread searcherThread2 = new SearcherThread(true); searcherThread2.Start(); SearcherThread searcherThread3 = new SearcherThread(true); searcherThread3.Start(); }
/* * Run one indexer and 2 searchers against single index as * stress test. */ public virtual void RunTest(Directory directory) { TimedThread[] threads = new TimedThread[4]; IndexWriter writer = new MockIndexWriter(this, directory, true, ANALYZER, true); writer.SetMaxBufferedDocs(7); writer.SetMergeFactor(3); // Establish a base index of 100 docs: for (int i = 0; i < 100; i++) { Document d = new Document(); d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED)); d.Add(new Field("contents", English.IntToEnglish(i), Field.Store.NO, Field.Index.ANALYZED)); writer.AddDocument(d); } writer.Commit(); IndexReader r = IndexReader.Open(directory); Assert.AreEqual(100, r.NumDocs()); r.Close(); IndexerThread indexerThread = new IndexerThread(writer, threads); threads[0] = indexerThread; indexerThread.Start(); IndexerThread indexerThread2 = new IndexerThread(writer, threads); threads[1] = indexerThread2; indexerThread2.Start(); SearcherThread searcherThread1 = new SearcherThread(directory, threads); threads[2] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(directory, threads); threads[3] = searcherThread2; searcherThread2.Start(); indexerThread.Join(); indexerThread2.Join(); searcherThread1.Join(); searcherThread2.Join(); writer.Close(); Assert.IsTrue(!indexerThread.failed, "hit unexpected exception in indexer"); Assert.IsTrue(!indexerThread2.failed, "hit unexpected exception in indexer2"); Assert.IsTrue(!searcherThread1.failed, "hit unexpected exception in search1"); Assert.IsTrue(!searcherThread2.failed, "hit unexpected exception in search2"); //System.out.println(" Writer: " + indexerThread.count + " iterations"); //System.out.println("Searcher 1: " + searcherThread1.count + " searchers created"); //System.out.println("Searcher 2: " + searcherThread2.count + " searchers created"); }
/* Run one indexer and 2 searchers against single index as stress test. */ public virtual void RunTest(Directory directory) { TimedThread[] threads = new TimedThread[4]; IndexWriterConfig conf = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMaxBufferedDocs(7); ((TieredMergePolicy)conf.MergePolicy).MaxMergeAtOnce = 3; IndexWriter writer = RandomIndexWriter.MockIndexWriter(directory, conf, Random()); // Establish a base index of 100 docs: for (int i = 0; i < 100; i++) { Documents.Document d = new Documents.Document(); d.Add(NewStringField("id", Convert.ToString(i), Field.Store.YES)); d.Add(NewTextField("contents", English.IntToEnglish(i), Field.Store.NO)); if ((i - 1) % 7 == 0) { writer.Commit(); } writer.AddDocument(d); } writer.Commit(); IndexReader r = DirectoryReader.Open(directory); Assert.AreEqual(100, r.NumDocs); r.Dispose(); IndexerThread indexerThread = new IndexerThread(writer, threads); threads[0] = indexerThread; indexerThread.Start(); IndexerThread indexerThread2 = new IndexerThread(writer, threads); threads[1] = indexerThread2; indexerThread2.Start(); SearcherThread searcherThread1 = new SearcherThread(directory, threads); threads[2] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(directory, threads); threads[3] = searcherThread2; searcherThread2.Start(); indexerThread.Join(); indexerThread2.Join(); searcherThread1.Join(); searcherThread2.Join(); writer.Dispose(); Assert.IsTrue(!indexerThread.Failed, "hit unexpected exception in indexer"); Assert.IsTrue(!indexerThread2.Failed, "hit unexpected exception in indexer2"); Assert.IsTrue(!searcherThread1.Failed, "hit unexpected exception in search1"); Assert.IsTrue(!searcherThread2.Failed, "hit unexpected exception in search2"); //System.out.println(" Writer: " + indexerThread.count + " iterations"); //System.out.println("Searcher 1: " + searcherThread1.count + " searchers created"); //System.out.println("Searcher 2: " + searcherThread2.count + " searchers created"); }
public virtual void TestTransactions_Mem( [ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler1, [ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler2) { Console.WriteLine("Start test"); // we cant use non-ramdir on windows, because this test needs to double-write. MockDirectoryWrapper dir1 = new MockDirectoryWrapper(Random(), new RAMDirectory()); MockDirectoryWrapper dir2 = new MockDirectoryWrapper(Random(), new RAMDirectory()); dir1.PreventDoubleWrite = false; dir2.PreventDoubleWrite = false; dir1.FailOn(new RandomFailure(this)); dir2.FailOn(new RandomFailure(this)); dir1.FailOnOpenInput = false; dir2.FailOnOpenInput = false; // We throw exceptions in deleteFile, which creates // leftover files: dir1.AssertNoUnrefencedFilesOnClose = false; dir2.AssertNoUnrefencedFilesOnClose = false; InitIndex(dir1); InitIndex(dir2); TimedThread[] threads = new TimedThread[3]; int numThread = 0; IndexerThread indexerThread = new IndexerThread(this, this, dir1, dir2, newScheduler1, newScheduler2, threads); threads[numThread++] = indexerThread; indexerThread.Start(); SearcherThread searcherThread1 = new SearcherThread(this, dir1, dir2, threads); threads[numThread++] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(this, dir1, dir2, threads); threads[numThread++] = searcherThread2; searcherThread2.Start(); for (int i = 0; i < numThread; i++) { threads[i].Join(); } for (int i = 0; i < numThread; i++) { Assert.IsTrue(!threads[i].Failed); } dir1.Dispose(); dir2.Dispose(); Console.WriteLine("End test"); }
/* * Run one indexer and 2 searchers against single index as * stress test. */ public virtual void RunStressTest(Directory directory, MergeScheduler mergeScheduler) { IndexWriter modifier = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED, null); modifier.SetMaxBufferedDocs(10); TimedThread[] threads = new TimedThread[4]; int numThread = 0; if (mergeScheduler != null) { modifier.SetMergeScheduler(mergeScheduler, null); } // One modifier that writes 10 docs then removes 5, over // and over: IndexerThread indexerThread = new IndexerThread(this, modifier, threads); threads[numThread++] = indexerThread; indexerThread.Start(); IndexerThread indexerThread2 = new IndexerThread(this, modifier, threads); threads[numThread++] = indexerThread2; indexerThread2.Start(); // Two searchers that constantly just re-instantiate the // searcher: SearcherThread searcherThread1 = new SearcherThread(directory, threads); threads[numThread++] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(directory, threads); threads[numThread++] = searcherThread2; searcherThread2.Start(); for (int i = 0; i < numThread; i++) { threads[i].Join(); } modifier.Close(); for (int i = 0; i < numThread; i++) { Assert.IsTrue(!((TimedThread)threads[i]).failed); } //System.out.println(" Writer: " + indexerThread.count + " iterations"); //System.out.println("Searcher 1: " + searcherThread1.count + " searchers created"); //System.out.println("Searcher 2: " + searcherThread2.count + " searchers created"); }
public virtual void TestImmediateDiskFullWithThreads([ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler) { int NUM_THREADS = 3; int numIterations = TEST_NIGHTLY ? 10 : 3; for (int iter = 0; iter < numIterations; iter++) { if (VERBOSE) { Console.WriteLine("\nTEST: iter=" + iter); } MockDirectoryWrapper dir = NewMockDirectory(); var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)) .SetMaxBufferedDocs(2) .SetMergeScheduler(newScheduler()) .SetMergePolicy(NewLogMergePolicy(4)); IndexWriter writer = new IndexWriter(dir, config); var scheduler = config.mergeScheduler as IConcurrentMergeScheduler; if (scheduler != null) { scheduler.SetSuppressExceptions(); } dir.MaxSizeInBytes = 4 * 1024 + 20 * iter; IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(writer, true, NewField); } for (int i = 0; i < NUM_THREADS; i++) { threads[i].Start(); } for (int i = 0; i < NUM_THREADS; i++) { // Without fix for LUCENE-1130: one of the // threads will hang threads[i].Join(); Assert.IsTrue(threads[i].Error == null, "hit unexpected Throwable"); } // Make sure once disk space is avail again, we can // cleanly close: dir.MaxSizeInBytes = 0; writer.Dispose(false); dir.Dispose(); } }
public virtual void TestRandomExceptions() { MockRAMDirectory dir = new MockRAMDirectory(); MockIndexWriter writer = new MockIndexWriter(this, dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); ((ConcurrentMergeScheduler)writer.GetMergeScheduler()).SetSuppressExceptions(); //writer.setMaxBufferedDocs(10); writer.SetRAMBufferSizeMB(0.1); if (DEBUG) { System.IO.StreamWriter temp_writer; temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding); temp_writer.AutoFlush = true; writer.SetInfoStream(temp_writer); } IndexerThread thread = new IndexerThread(this, 0, writer); thread.Run(); if (thread.failure != null) { System.Console.Out.WriteLine(thread.failure.StackTrace); Assert.Fail("thread " + thread.Name + ": hit unexpected failure"); } writer.Commit(); try { writer.Close(); } catch (System.Exception t) { System.Console.Out.WriteLine("exception during close:"); System.Console.Out.WriteLine(t.StackTrace); writer.Rollback(); } // Confirm that when doc hits exception partway through tokenization, it's deleted: IndexReader r2 = IndexReader.Open(dir); int count = r2.DocFreq(new Term("content4", "aaa")); int count2 = r2.DocFreq(new Term("content4", "ddd")); Assert.AreEqual(count, count2); r2.Close(); _TestUtil.CheckIndex(dir); }
public virtual void TestImmediateDiskFullWithThreads() { int NUM_THREADS = 3; int numIterations = TestNightly ? 10 : 3; for (int iter = 0; iter < numIterations; iter++) { if (Verbose) { Console.WriteLine("\nTEST: iter=" + iter); } MockDirectoryWrapper dir = NewMockDirectory(); IndexWriter writer = new IndexWriter( dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)) .SetMaxBufferedDocs(2) .SetMergeScheduler(new ConcurrentMergeScheduler()) .SetMergePolicy(NewLogMergePolicy(4))); ((IConcurrentMergeScheduler)writer.Config.MergeScheduler).SetSuppressExceptions(); dir.MaxSizeInBytes = 4 * 1024 + 20 * iter; IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(writer, true, NewField); } for (int i = 0; i < NUM_THREADS; i++) { threads[i].Start(); } for (int i = 0; i < NUM_THREADS; i++) { // Without fix for LUCENE-1130: one of the // threads will hang threads[i].Join(); Assert.IsTrue(threads[i].error is null, "hit unexpected Throwable"); } // Make sure once disk space is avail again, we can // cleanly close: dir.MaxSizeInBytes = 0; writer.Dispose(false); dir.Dispose(); } }
/* * Run one indexer and 2 searchers against single index as * stress test. */ public virtual void RunStressTest(Directory directory, IConcurrentMergeScheduler mergeScheduler) { IndexWriter modifier = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetMaxBufferedDocs(10).SetMergeScheduler(mergeScheduler)); modifier.Commit(); TimedThread[] threads = new TimedThread[4]; int numThread = 0; // One modifier that writes 10 docs then removes 5, over // and over: IndexerThread indexerThread = new IndexerThread(modifier, threads, NewStringField, NewTextField); threads[numThread++] = indexerThread; indexerThread.Start(); IndexerThread indexerThread2 = new IndexerThread(modifier, threads, NewStringField, NewTextField); threads[numThread++] = indexerThread2; indexerThread2.Start(); // Two searchers that constantly just re-instantiate the // searcher: SearcherThread searcherThread1 = new SearcherThread(directory, threads, this); threads[numThread++] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(directory, threads, this); threads[numThread++] = searcherThread2; searcherThread2.Start(); for (int i = 0; i < numThread; i++) { threads[i].Join(); } modifier.Dispose(); for (int i = 0; i < numThread; i++) { Assert.IsTrue(!threads[i].failed); } //System.out.println(" Writer: " + indexerThread.count + " iterations"); //System.out.println("Searcher 1: " + searcherThread1.count + " searchers created"); //System.out.println("Searcher 2: " + searcherThread2.count + " searchers created"); }
/* Run one indexer and 2 searchers against single index as stress test. */ public virtual void RunStressTest(Directory directory, MergeScheduler mergeScheduler) { IndexWriter modifier = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode_e.CREATE).SetMaxBufferedDocs(10).SetMergeScheduler(mergeScheduler)); modifier.Commit(); TimedThread[] threads = new TimedThread[4]; int numThread = 0; // One modifier that writes 10 docs then removes 5, over // and over: IndexerThread indexerThread = new IndexerThread(modifier, threads); threads[numThread++] = indexerThread; indexerThread.Start(); IndexerThread indexerThread2 = new IndexerThread(modifier, threads); threads[numThread++] = indexerThread2; indexerThread2.Start(); // Two searchers that constantly just re-instantiate the // searcher: SearcherThread searcherThread1 = new SearcherThread(directory, threads); threads[numThread++] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(directory, threads); threads[numThread++] = searcherThread2; searcherThread2.Start(); for (int i = 0; i < numThread; i++) { threads[i].Join(); } modifier.Dispose(); for (int i = 0; i < numThread; i++) { Assert.IsTrue(!threads[i].Failed); } //System.out.println(" Writer: " + indexerThread.count + " iterations"); //System.out.println("Searcher 1: " + searcherThread1.count + " searchers created"); //System.out.println("Searcher 2: " + searcherThread2.count + " searchers created"); }
public virtual void TestTransactions_Rename() { RANDOM = NewRandom(); MockRAMDirectory dir1 = new MockRAMDirectory(); MockRAMDirectory dir2 = new MockRAMDirectory(); dir1.SetPreventDoubleWrite(false); dir2.SetPreventDoubleWrite(false); dir1.FailOn(new RandomFailure(this)); dir2.FailOn(new RandomFailure(this)); InitIndex(dir1); InitIndex(dir2); TimedThread[] threads = new TimedThread[3]; int numThread = 0; IndexerThread indexerThread = new IndexerThread(this, this, dir1, dir2, threads); threads[numThread++] = indexerThread; indexerThread.Start(); SearcherThread searcherThread1 = new SearcherThread(this, dir1, dir2, threads); threads[numThread++] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(this, dir1, dir2, threads); threads[numThread++] = searcherThread2; searcherThread2.Start(); for (int i = 0; i < numThread; i++) { threads[i].Join(); } for (int i = 0; i < numThread; i++) { Assert.IsTrue(!((TimedThread)threads[i]).failed); } }
public static void Main(System.String[] args) { bool readOnly = false; bool add = false; for (int i = 0; i < args.Length; i++) { if ("-ro".Equals(args[i])) readOnly = true; if ("-add".Equals(args[i])) add = true; } System.IO.FileInfo indexDir = new System.IO.FileInfo("index"); bool tmpBool; if (System.IO.File.Exists(indexDir.FullName)) tmpBool = true; else tmpBool = System.IO.Directory.Exists(indexDir.FullName); if (!tmpBool) { System.IO.Directory.CreateDirectory(indexDir.FullName); } IndexReader.Unlock(FSDirectory.GetDirectory(indexDir, false)); if (!readOnly) { IndexWriter writer = new IndexWriter(indexDir, ANALYZER, !add); SupportClass.ThreadClass indexerThread = new IndexerThread(writer); indexerThread.Start(); System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000)); } SearcherThread searcherThread1 = new SearcherThread(false); searcherThread1.Start(); SEARCHER = new IndexSearcher(indexDir.ToString()); SearcherThread searcherThread2 = new SearcherThread(true); searcherThread2.Start(); SearcherThread searcherThread3 = new SearcherThread(true); searcherThread3.Start(); }
// Runs test, with multiple threads, using the specific // failure to trigger an IOException public virtual void _testMultipleThreadsFailure(MockRAMDirectory.Failure failure) { int NUM_THREADS = 3; for (int iter = 0; iter < 5; iter++) { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer()); ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); // We expect disk full exceptions in the merge threads cms.SetSuppressExceptions_ForNUnitTest(); writer.SetMergeScheduler(cms); writer.SetMaxBufferedDocs(2); writer.SetMergeFactor(4); IndexerThread[] threads = new IndexerThread[NUM_THREADS]; //bool diskFull = false; for (int i = 0; i < NUM_THREADS; i++) threads[i] = new IndexerThread(this, writer, true); for (int i = 0; i < NUM_THREADS; i++) threads[i].Start(); try { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 10)); } catch (System.Threading.ThreadInterruptedException) { SupportClass.ThreadClass.Current().Interrupt(); } dir.FailOn(failure); failure.SetDoFail(); for (int i = 0; i < NUM_THREADS; i++) { while (true) { try { threads[i].Join(); break; } catch (System.Threading.ThreadInterruptedException) { SupportClass.ThreadClass.Current().Interrupt(); } } if (threads[i].IsAlive) Assert.Fail("thread seems to be hung"); else Assert.IsTrue(threads[i].error == null, "hit unexpected Throwable"); } bool success = false; try { writer.Close(false); success = true; } catch (System.IO.IOException) { } if (success) { IndexReader reader = IndexReader.Open(dir); for (int j = 0; j < reader.MaxDoc(); j++) { if (!reader.IsDeleted(j)) { reader.Document(j); reader.GetTermFreqVectors(j); } } reader.Close(); } dir.Close(); } }
public virtual void TestImmediateDiskFullWithThreads() { int NUM_THREADS = 3; for (int iter = 0; iter < 10; iter++) { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, true, new WhitespaceAnalyzer()); ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); // We expect disk full exceptions in the merge threads cms.SetSuppressExceptions(); writer.SetMergeScheduler(cms); writer.SetMaxBufferedDocs(2); writer.SetMergeFactor(4); dir.SetMaxSizeInBytes(4 * 1024 + 20 * iter); IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) threads[i] = new IndexerThread(this, writer, true); for (int i = 0; i < NUM_THREADS; i++) threads[i].Start(); for (int i = 0; i < NUM_THREADS; i++) { // Without fix for LUCENE-1130: one of the // threads will hang threads[i].Join(); Assert.IsTrue(threads[i].error == null, "hit unexpected Throwable"); } try { writer.Close(false); } catch (System.IO.IOException ioe) { } dir.Close(); } }
public virtual void TestCloseWithThreads() { int NUM_THREADS = 3; for (int iter = 0; iter < 50; iter++) { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer()); ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); writer.SetMergeScheduler(cms); writer.SetMaxBufferedDocs(10); writer.SetMergeFactor(4); IndexerThread[] threads = new IndexerThread[NUM_THREADS]; //bool diskFull = false; for (int i = 0; i < NUM_THREADS; i++) threads[i] = new IndexerThread(this, writer, false); for (int i = 0; i < NUM_THREADS; i++) threads[i].Start(); try { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 50)); } catch (System.Threading.ThreadInterruptedException) { SupportClass.ThreadClass.Current().Interrupt(); } writer.Close(false); // Make sure threads that are adding docs are not hung: for (int i = 0; i < NUM_THREADS; i++) { while (true) { try { // Without fix for LUCENE-1130: one of the // threads will hang threads[i].Join(); break; } catch (System.Threading.ThreadInterruptedException) { SupportClass.ThreadClass.Current().Interrupt(); } } if (threads[i].IsAlive) Assert.Fail("thread seems to be hung"); } // Quick test to make sure index is not corrupt: IndexReader reader = IndexReader.Open(dir); TermDocs tdocs = reader.TermDocs(new Term("field", "aaa")); int count = 0; while (tdocs.Next()) { count++; } Assert.IsTrue(count > 0); reader.Close(); dir.Close(); } }
public virtual void TestImmediateDiskFullWithThreads() { int NUM_THREADS = 3; for (int iter = 0; iter < 10; iter++) { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer()); ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); // We expect disk full exceptions in the merge threads cms.SetSuppressExceptions_ForNUnitTest(); writer.SetMergeScheduler(cms); writer.SetMaxBufferedDocs(2); writer.SetMergeFactor(4); dir.SetMaxSizeInBytes(4 * 1024 + 20 * iter); IndexerThread[] threads = new IndexerThread[NUM_THREADS]; //bool diskFull = false; for (int i = 0; i < NUM_THREADS; i++) threads[i] = new IndexerThread(this, writer, true); for (int i = 0; i < NUM_THREADS; i++) threads[i].Start(); for (int i = 0; i < NUM_THREADS; i++) { while (true) { try { // Without fix for LUCENE-1130: one of the // threads will hang threads[i].Join(); break; } catch (System.Threading.ThreadInterruptedException) { SupportClass.ThreadClass.Current().Interrupt(); } } if (threads[i].IsAlive) Assert.Fail("thread seems to be hung"); else Assert.IsTrue(threads[i].error == null, "hit unexpected Throwable"); } try { writer.Close(false); } catch (System.IO.IOException) { } dir.Close(); } }
/* Run one indexer and 2 searchers against single index as stress test. */ public virtual void RunStressTest(Directory directory, bool autoCommit, MergeScheduler mergeScheduler) { IndexWriter modifier = new IndexWriter(directory, autoCommit, ANALYZER, true); modifier.SetMaxBufferedDocs(10); TimedThread[] threads = new TimedThread[4]; int numThread = 0; if (mergeScheduler != null) modifier.SetMergeScheduler(mergeScheduler); // One modifier that writes 10 docs then removes 5, over // and over: IndexerThread indexerThread = new IndexerThread(this, modifier, threads); threads[numThread++] = indexerThread; indexerThread.Start(); IndexerThread indexerThread2 = new IndexerThread(this, modifier, threads); threads[numThread++] = indexerThread2; indexerThread2.Start(); // Two searchers that constantly just re-instantiate the // searcher: SearcherThread searcherThread1 = new SearcherThread(directory, threads); threads[numThread++] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(directory, threads); threads[numThread++] = searcherThread2; searcherThread2.Start(); for (int i = 0; i < numThread; i++) threads[i].Join(); modifier.Close(); for (int i = 0; i < numThread; i++) Assert.IsTrue(!((TimedThread) threads[i]).failed); //System.out.println(" Writer: " + indexerThread.count + " iterations"); //System.out.println("Searcher 1: " + searcherThread1.count + " searchers created"); //System.out.println("Searcher 2: " + searcherThread2.count + " searchers created"); }
public virtual void TestCloseWithThreads() { int NUM_THREADS = 3; int numIterations = TEST_NIGHTLY ? 7 : 3; for (int iter = 0; iter < numIterations; iter++) { if (VERBOSE) { Console.WriteLine("\nTEST: iter=" + iter); } Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(10).SetMergeScheduler(new ConcurrentMergeScheduler()).SetMergePolicy(NewLogMergePolicy(4))); ((ConcurrentMergeScheduler)writer.Config.MergeScheduler).SetSuppressExceptions(); IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(writer, false); } for (int i = 0; i < NUM_THREADS; i++) { threads[i].Start(); } bool done = false; while (!done) { Thread.Sleep(100); for (int i = 0; i < NUM_THREADS; i++) // only stop when at least one thread has added a doc { if (threads[i].AddCount > 0) { done = true; break; } else if (!threads[i].IsAlive) { Assert.Fail("thread failed before indexing a single document"); } } } if (VERBOSE) { Console.WriteLine("\nTEST: now close"); } writer.Dispose(false); // Make sure threads that are adding docs are not hung: for (int i = 0; i < NUM_THREADS; i++) { // Without fix for LUCENE-1130: one of the // threads will hang threads[i].Join(); if (threads[i].IsAlive) { Assert.Fail("thread seems to be hung"); } } // Quick test to make sure index is not corrupt: IndexReader reader = DirectoryReader.Open(dir); DocsEnum tdocs = TestUtil.Docs(Random(), reader, "field", new BytesRef("aaa"), MultiFields.GetLiveDocs(reader), null, 0); int count = 0; while (tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS) { count++; } Assert.IsTrue(count > 0); reader.Dispose(); dir.Dispose(); } }
public virtual void TestTransactions_Mem() { Console.WriteLine("Start test"); // we cant use non-ramdir on windows, because this test needs to double-write. MockDirectoryWrapper dir1 = new MockDirectoryWrapper(Random(), new RAMDirectory()); MockDirectoryWrapper dir2 = new MockDirectoryWrapper(Random(), new RAMDirectory()); dir1.PreventDoubleWrite = false; dir2.PreventDoubleWrite = false; dir1.FailOn(new RandomFailure(this)); dir2.FailOn(new RandomFailure(this)); dir1.FailOnOpenInput = false; dir2.FailOnOpenInput = false; // We throw exceptions in deleteFile, which creates // leftover files: dir1.AssertNoUnrefencedFilesOnClose = false; dir2.AssertNoUnrefencedFilesOnClose = false; InitIndex(dir1); InitIndex(dir2); TimedThread[] threads = new TimedThread[3]; int numThread = 0; IndexerThread indexerThread = new IndexerThread(this, this, dir1, dir2, threads); threads[numThread++] = indexerThread; indexerThread.Start(); SearcherThread searcherThread1 = new SearcherThread(this, dir1, dir2, threads); threads[numThread++] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(this, dir1, dir2, threads); threads[numThread++] = searcherThread2; searcherThread2.Start(); for (int i = 0; i < numThread; i++) { threads[i].Join(); } for (int i = 0; i < numThread; i++) { Assert.IsTrue(!threads[i].Failed); } dir1.Dispose(); dir2.Dispose(); Console.WriteLine("End test"); }
/* Run one indexer and 2 searchers against single index as stress test. */ public virtual void RunTest(Directory directory) { TimedThread[] threads = new TimedThread[4]; IndexWriter writer = new MockIndexWriter(this, directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED); writer.SetMaxBufferedDocs(7); writer.MergeFactor = 3; // Establish a base index of 100 docs: for (int i = 0; i < 100; i++) { Document d = new Document(); d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED)); d.Add(new Field("contents", English.IntToEnglish(i), Field.Store.NO, Field.Index.ANALYZED)); if ((i - 1) % 7 == 0) { writer.Commit(); } writer.AddDocument(d); } writer.Commit(); IndexReader r = IndexReader.Open(directory, true); Assert.AreEqual(100, r.NumDocs()); r.Close(); IndexerThread indexerThread = new IndexerThread(writer, threads); threads[0] = indexerThread; indexerThread.Start(); IndexerThread indexerThread2 = new IndexerThread(writer, threads); threads[1] = indexerThread2; indexerThread2.Start(); SearcherThread searcherThread1 = new SearcherThread(directory, threads); threads[2] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(directory, threads); threads[3] = searcherThread2; searcherThread2.Start(); indexerThread.Join(); indexerThread2.Join(); searcherThread1.Join(); searcherThread2.Join(); writer.Close(); Assert.IsTrue(!indexerThread.failed, "hit unexpected exception in indexer"); Assert.IsTrue(!indexerThread2.failed, "hit unexpected exception in indexer2"); Assert.IsTrue(!searcherThread1.failed, "hit unexpected exception in search1"); Assert.IsTrue(!searcherThread2.failed, "hit unexpected exception in search2"); //System.out.println(" Writer: " + indexerThread.count + " iterations"); //System.out.println("Searcher 1: " + searcherThread1.count + " searchers created"); //System.out.println("Searcher 2: " + searcherThread2.count + " searchers created"); }
public virtual void Test_Directory() // LUCENENET specific - name collides with property of LuceneTestCase { Store.Directory indexDir = NewDirectory(); Store.Directory taxoDir = NewDirectory(); IndexWriter w = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))); var tw = new DirectoryTaxonomyWriter(taxoDir); // first empty commit w.Commit(); tw.Commit(); var mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null); FacetsConfig config = new FacetsConfig(); config.SetMultiValued("field", true); AtomicBoolean stop = new AtomicBoolean(); // How many unique facets to index before stopping: //int ordLimit = TestNightly ? 100000 : 6000; // LUCENENET specific: 100000 facets takes about 2-3 hours. To keep it under // the 1 hour free limit of Azure DevOps, this was reduced to 30000. int ordLimit = TestNightly ? 30000 : 6000; var indexer = new IndexerThread(w, config, tw, mgr, ordLimit, stop); indexer.Start(); try { while (!stop) { SearcherAndTaxonomy pair = mgr.Acquire(); try { //System.out.println("search maxOrd=" + pair.taxonomyReader.getSize()); FacetsCollector sfc = new FacetsCollector(); pair.Searcher.Search(new MatchAllDocsQuery(), sfc); Facets facets = GetTaxonomyFacetCounts(pair.TaxonomyReader, config, sfc); FacetResult result = facets.GetTopChildren(10, "field"); if (pair.Searcher.IndexReader.NumDocs > 0) { //System.out.println(pair.taxonomyReader.getSize()); Assert.IsTrue(result.ChildCount > 0); Assert.IsTrue(result.LabelValues.Length > 0); } //if (VERBOSE) { //System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0))); //} } finally { mgr.Release(pair); } } } finally { indexer.Join(); } if (Verbose) { Console.WriteLine("TEST: now stop"); } IOUtils.Dispose(mgr, tw, w, taxoDir, indexDir); }
/* Run one indexer and 2 searchers against single index as stress test. */ public virtual void RunStressTest(Directory directory, bool autoCommit, MergeScheduler mergeScheduler) { IndexWriter modifier = new IndexWriter(directory, autoCommit, ANALYZER, true); modifier.SetMaxBufferedDocs(10); TimedThread[] threads = new TimedThread[4]; if (mergeScheduler != null) modifier.SetMergeScheduler(mergeScheduler); // One modifier that writes 10 docs then removes 5, over // and over: IndexerThread indexerThread = new IndexerThread(modifier, threads); threads[0] = indexerThread; indexerThread.Start(); IndexerThread indexerThread2 = new IndexerThread(modifier, threads); threads[2] = indexerThread2; indexerThread2.Start(); // Two searchers that constantly just re-instantiate the // searcher: SearcherThread searcherThread1 = new SearcherThread(directory, threads); threads[3] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(directory, threads); threads[3] = searcherThread2; searcherThread2.Start(); indexerThread.Join(); indexerThread2.Join(); searcherThread1.Join(); searcherThread2.Join(); modifier.Close(); Assert.IsTrue(!indexerThread.failed, "hit unexpected exception in indexer"); Assert.IsTrue(!indexerThread2.failed, "hit unexpected exception in indexer2"); Assert.IsTrue(!searcherThread1.failed, "hit unexpected exception in search1"); Assert.IsTrue(!searcherThread2.failed, "hit unexpected exception in search2"); //System.out.println(" Writer: " + indexerThread.count + " iterations"); //System.out.println("Searcher 1: " + searcherThread1.count + " searchers created"); //System.out.println("Searcher 2: " + searcherThread2.count + " searchers created"); }
public virtual void TestDirectory() { Store.Directory indexDir = NewDirectory(); Store.Directory taxoDir = NewDirectory(); IndexWriter w = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); var tw = new DirectoryTaxonomyWriter(taxoDir); // first empty commit w.Commit(); tw.Commit(); var mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null); FacetsConfig config = new FacetsConfig(); config.SetMultiValued("field", true); AtomicBoolean stop = new AtomicBoolean(); // How many unique facets to index before stopping: int ordLimit = TEST_NIGHTLY ? 100000 : 6000; var indexer = new IndexerThread(w, config, tw, mgr, ordLimit, stop); indexer.Start(); try { while (!stop.Get()) { SearcherAndTaxonomy pair = mgr.Acquire(); try { //System.out.println("search maxOrd=" + pair.taxonomyReader.getSize()); FacetsCollector sfc = new FacetsCollector(); pair.searcher.Search(new MatchAllDocsQuery(), sfc); Facets facets = GetTaxonomyFacetCounts(pair.taxonomyReader, config, sfc); FacetResult result = facets.GetTopChildren(10, "field"); if (pair.searcher.IndexReader.NumDocs > 0) { //System.out.println(pair.taxonomyReader.getSize()); Assert.True(result.ChildCount > 0); Assert.True(result.LabelValues.Length > 0); } //if (VERBOSE) { //System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0))); //} } finally { mgr.Release(pair); } } } finally { indexer.Join(); } if (VERBOSE) { Console.WriteLine("TEST: now stop"); } IOUtils.Close(mgr, tw, w, taxoDir, indexDir); }
public virtual void TestNrt() { Store.Directory dir = NewDirectory(); Store.Directory taxoDir = NewDirectory(); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); // Don't allow tiny maxBufferedDocs; it can make this // test too slow: iwc.SetMaxBufferedDocs(Math.Max(500, iwc.MaxBufferedDocs)); // MockRandom/AlcololicMergePolicy are too slow: TieredMergePolicy tmp = new TieredMergePolicy(); tmp.FloorSegmentMB = .001; iwc.SetMergePolicy(tmp); IndexWriter w = new IndexWriter(dir, iwc); var tw = new DirectoryTaxonomyWriter(taxoDir); FacetsConfig config = new FacetsConfig(); config.SetMultiValued("field", true); AtomicBoolean stop = new AtomicBoolean(); // How many unique facets to index before stopping: int ordLimit = TEST_NIGHTLY ? 100000 : 6000; var indexer = new IndexerThread(w, config, tw, null, ordLimit, stop); var mgr = new SearcherTaxonomyManager(w, true, null, tw); var reopener = new ThreadAnonymousInnerClassHelper(this, stop, mgr); reopener.Name = "reopener"; reopener.Start(); indexer.Name = "indexer"; indexer.Start(); try { while (!stop.Get()) { SearcherAndTaxonomy pair = mgr.Acquire(); try { //System.out.println("search maxOrd=" + pair.taxonomyReader.getSize()); FacetsCollector sfc = new FacetsCollector(); pair.searcher.Search(new MatchAllDocsQuery(), sfc); Facets facets = GetTaxonomyFacetCounts(pair.taxonomyReader, config, sfc); FacetResult result = facets.GetTopChildren(10, "field"); if (pair.searcher.IndexReader.NumDocs > 0) { //System.out.println(pair.taxonomyReader.getSize()); Assert.True(result.ChildCount > 0); Assert.True(result.LabelValues.Length > 0); } //if (VERBOSE) { //System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0))); //} } finally { mgr.Release(pair); } } } finally { indexer.Join(); reopener.Join(); } if (VERBOSE) { Console.WriteLine("TEST: now stop"); } IOUtils.Close(mgr, tw, w, taxoDir, dir); }
public virtual void TestRandomExceptionsThreads() { MockRAMDirectory dir = new MockRAMDirectory(); MockIndexWriter writer = new MockIndexWriter(this, dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); ((ConcurrentMergeScheduler) writer.GetMergeScheduler()).SetSuppressExceptions(); //writer.setMaxBufferedDocs(10); writer.SetRAMBufferSizeMB(0.2); if (DEBUG) { System.IO.StreamWriter temp_writer; temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding); temp_writer.AutoFlush = true; writer.SetInfoStream(temp_writer); } int NUM_THREADS = 4; IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(this, i, writer); threads[i].Start(); } for (int i = 0; i < NUM_THREADS; i++) threads[i].Join(); for (int i = 0; i < NUM_THREADS; i++) if (threads[i].failure != null) Assert.Fail("thread " + threads[i].Name + ": hit unexpected failure"); writer.Commit(); try { writer.Close(); } catch (System.Exception t) { System.Console.Out.WriteLine("exception during close:"); System.Console.Out.WriteLine(t.StackTrace); writer.Rollback(); } // Confirm that when doc hits exception partway through tokenization, it's deleted: IndexReader r2 = IndexReader.Open(dir); int count = r2.DocFreq(new Term("content4", "aaa")); int count2 = r2.DocFreq(new Term("content4", "ddd")); Assert.AreEqual(count, count2); r2.Close(); _TestUtil.CheckIndex(dir); }
public virtual void TestCloseWithThreads() { int NUM_THREADS = 3; int numIterations = TEST_NIGHTLY ? 7 : 3; for (int iter = 0; iter < numIterations; iter++) { if (VERBOSE) { Console.WriteLine("\nTEST: iter=" + iter); } Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(10).SetMergeScheduler(new ConcurrentMergeScheduler()).SetMergePolicy(NewLogMergePolicy(4))); ((ConcurrentMergeScheduler)writer.Config.MergeScheduler).SetSuppressExceptions(); IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(this, writer, false); } for (int i = 0; i < NUM_THREADS; i++) { threads[i].Start(); } bool done = false; while (!done) { Thread.Sleep(100); for (int i = 0; i < NUM_THREADS; i++) // only stop when at least one thread has added a doc { if (threads[i].AddCount > 0) { done = true; break; } else if (!threads[i].IsAlive) { Assert.Fail("thread failed before indexing a single document"); } } } if (VERBOSE) { Console.WriteLine("\nTEST: now close"); } writer.Dispose(false); // Make sure threads that are adding docs are not hung: for (int i = 0; i < NUM_THREADS; i++) { // Without fix for LUCENE-1130: one of the // threads will hang threads[i].Join(); if (threads[i].IsAlive) { Assert.Fail("thread seems to be hung"); } } // Quick test to make sure index is not corrupt: IndexReader reader = DirectoryReader.Open(dir); DocsEnum tdocs = TestUtil.Docs(Random(), reader, "field", new BytesRef("aaa"), MultiFields.GetLiveDocs(reader), null, 0); int count = 0; while (tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS) { count++; } Assert.IsTrue(count > 0); reader.Dispose(); dir.Dispose(); } }
public virtual void TestCloseWithThreads([ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler) { int NUM_THREADS = 3; int numIterations = TEST_NIGHTLY ? 7 : 3; for (int iter = 0; iter < numIterations; iter++) { if (VERBOSE) { Console.WriteLine("\nTEST: iter=" + iter); } Directory dir = NewDirectory(); var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)) .SetMaxBufferedDocs(10) .SetMergeScheduler(newScheduler()) .SetMergePolicy(NewLogMergePolicy(4)); IndexWriter writer = new IndexWriter(dir, config); var scheduler = config.mergeScheduler as IConcurrentMergeScheduler; if (scheduler != null) { scheduler.SetSuppressExceptions(); } IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(writer, false, NewField) // LUCENENET NOTE - ConcurrentMergeScheduler // used to take too long for this test to index a single document // so, increased the time from 200 to 300 ms. // But it has now been restored to 200 ms like Lucene. { TimeToRunInMilliseconds = 200 }; } for (int i = 0; i < NUM_THREADS; i++) { threads[i].Start(); } bool done = false; while (!done) { Thread.Sleep(100); for (int i = 0; i < NUM_THREADS; i++) // only stop when at least one thread has added a doc { if (threads[i].AddCount > 0) { done = true; break; } else if (!threads[i].IsAlive) { Assert.Fail("thread failed before indexing a single document"); } } } if (VERBOSE) { Console.WriteLine("\nTEST: now close"); } writer.Dispose(false); // Make sure threads that are adding docs are not hung: for (int i = 0; i < NUM_THREADS; i++) { // Without fix for LUCENE-1130: one of the // threads will hang threads[i].Join(); if (threads[i].IsAlive) { Assert.Fail("thread seems to be hung"); } } // Quick test to make sure index is not corrupt: IndexReader reader = DirectoryReader.Open(dir); DocsEnum tdocs = TestUtil.Docs(Random, reader, "field", new BytesRef("aaa"), MultiFields.GetLiveDocs(reader), null, 0); int count = 0; while (tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS) { count++; } Assert.IsTrue(count > 0); reader.Dispose(); dir.Dispose(); } }
public virtual void TestNrt() { Store.Directory dir = NewDirectory(); Store.Directory taxoDir = NewDirectory(); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); // Don't allow tiny maxBufferedDocs; it can make this // test too slow: iwc.SetMaxBufferedDocs(Math.Max(500, iwc.MaxBufferedDocs)); // MockRandom/AlcololicMergePolicy are too slow: TieredMergePolicy tmp = new TieredMergePolicy(); tmp.FloorSegmentMB = .001; iwc.SetMergePolicy(tmp); IndexWriter w = new IndexWriter(dir, iwc); var tw = new DirectoryTaxonomyWriter(taxoDir); FacetsConfig config = new FacetsConfig(); config.SetMultiValued("field", true); AtomicBoolean stop = new AtomicBoolean(); // How many unique facets to index before stopping: int ordLimit = TEST_NIGHTLY ? 100000 : 6000; var indexer = new IndexerThread(w, config, tw, null, ordLimit, stop); var mgr = new SearcherTaxonomyManager(w, true, null, tw); var reopener = new ThreadAnonymousInnerClassHelper(this, stop, mgr); reopener.Name = "reopener"; reopener.Start(); indexer.Name = "indexer"; indexer.Start(); try { while (!stop.Get()) { SearcherAndTaxonomy pair = mgr.Acquire(); try { //System.out.println("search maxOrd=" + pair.taxonomyReader.getSize()); FacetsCollector sfc = new FacetsCollector(); pair.searcher.Search(new MatchAllDocsQuery(), sfc); Facets facets = GetTaxonomyFacetCounts(pair.taxonomyReader, config, sfc); FacetResult result = facets.GetTopChildren(10, "field"); if (pair.searcher.IndexReader.NumDocs > 0) { //System.out.println(pair.taxonomyReader.getSize()); Assert.True(result.ChildCount > 0); Assert.True(result.LabelValues.Length > 0); } //if (VERBOSE) { //System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0))); //} } finally { mgr.Release(pair); } } } finally { indexer.Join(); reopener.Join(); } if (VERBOSE) { Console.WriteLine("TEST: now stop"); } IOUtils.Close(mgr, tw, w, taxoDir, dir); }
public virtual void TestCloseWithThreads() { int NUM_THREADS = 3; for (int iter = 0; iter < 20; iter++) { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); // We expect AlreadyClosedException cms.SetSuppressExceptions(); writer.SetMergeScheduler(cms); writer.SetMaxBufferedDocs(10); writer.SetMergeFactor(4); IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) threads[i] = new IndexerThread(this, writer, false); for (int i = 0; i < NUM_THREADS; i++) threads[i].Start(); bool done = false; while (!done) { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 100)); for (int i = 0; i < NUM_THREADS; i++) // only stop when at least one thread has added a doc if (threads[i].addCount > 0) { done = true; break; } } writer.Close(false); // Make sure threads that are adding docs are not hung: for (int i = 0; i < NUM_THREADS; i++) { // Without fix for LUCENE-1130: one of the // threads will hang threads[i].Join(); if (threads[i].IsAlive) Assert.Fail("thread seems to be hung"); } // Quick test to make sure index is not corrupt: IndexReader reader = IndexReader.Open(dir); TermDocs tdocs = reader.TermDocs(new Term("field", "aaa")); int count = 0; while (tdocs.Next()) { count++; } Assert.IsTrue(count > 0); reader.Close(); dir.Close(); } }
public virtual void TestTransactions_Rename() { RANDOM = NewRandom(); MockRAMDirectory dir1 = new MockRAMDirectory(); MockRAMDirectory dir2 = new MockRAMDirectory(); dir1.SetPreventDoubleWrite(false); dir2.SetPreventDoubleWrite(false); dir1.FailOn(new RandomFailure(this)); dir2.FailOn(new RandomFailure(this)); InitIndex(dir1); InitIndex(dir2); TimedThread[] threads = new TimedThread[3]; int numThread = 0; IndexerThread indexerThread = new IndexerThread(this, this, dir1, dir2, threads); threads[numThread++] = indexerThread; indexerThread.Start(); SearcherThread searcherThread1 = new SearcherThread(this, dir1, dir2, threads); threads[numThread++] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(this, dir1, dir2, threads); threads[numThread++] = searcherThread2; searcherThread2.Start(); for (int i = 0; i < numThread; i++) threads[i].Join(); for (int i = 0; i < numThread; i++) Assert.IsTrue(!((TimedThread) threads[i]).failed); }
// Runs test, with multiple threads, using the specific // failure to trigger an IOException public virtual void _testMultipleThreadsFailure(MockRAMDirectory.Failure failure) { int NUM_THREADS = 3; for (int iter = 0; iter < 5; iter++) { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); // We expect disk full exceptions in the merge threads cms.SetSuppressExceptions(); writer.SetMergeScheduler(cms); writer.SetMaxBufferedDocs(2); writer.SetMergeFactor(4); IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) threads[i] = new IndexerThread(this, writer, true); for (int i = 0; i < NUM_THREADS; i++) threads[i].Start(); System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 10)); dir.FailOn(failure); failure.SetDoFail(); for (int i = 0; i < NUM_THREADS; i++) { threads[i].Join(); Assert.IsTrue(threads[i].error == null, "hit unexpected Throwable"); } bool success = false; try { writer.Close(false); success = true; } catch (System.IO.IOException ioe) { failure.ClearDoFail(); writer.Close(false); } if (success) { IndexReader reader = IndexReader.Open(dir); for (int j = 0; j < reader.MaxDoc(); j++) { if (!reader.IsDeleted(j)) { reader.Document(j); reader.GetTermFreqVectors(j); } } reader.Close(); } dir.Close(); } }
/* * Run one indexer and 2 searchers against single index as * stress test. */ public virtual void RunTest(Directory directory) { TimedThread[] threads = new TimedThread[4]; IndexWriterConfig conf = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMaxBufferedDocs(7); ((TieredMergePolicy)conf.MergePolicy).MaxMergeAtOnce = 3; IndexWriter writer = RandomIndexWriter.MockIndexWriter(directory, conf, Random()); // Establish a base index of 100 docs: for (int i = 0; i < 100; i++) { Documents.Document d = new Documents.Document(); d.Add(NewStringField("id", Convert.ToString(i), Field.Store.YES)); d.Add(NewTextField("contents", English.IntToEnglish(i), Field.Store.NO)); if ((i - 1) % 7 == 0) { writer.Commit(); } writer.AddDocument(d); } writer.Commit(); IndexReader r = DirectoryReader.Open(directory); Assert.AreEqual(100, r.NumDocs); r.Dispose(); IndexerThread indexerThread = new IndexerThread(writer, threads); threads[0] = indexerThread; indexerThread.Start(); IndexerThread indexerThread2 = new IndexerThread(writer, threads); threads[1] = indexerThread2; indexerThread2.Start(); SearcherThread searcherThread1 = new SearcherThread(directory, threads); threads[2] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(directory, threads); threads[3] = searcherThread2; searcherThread2.Start(); indexerThread.Join(); indexerThread2.Join(); searcherThread1.Join(); searcherThread2.Join(); writer.Dispose(); Assert.IsTrue(!indexerThread.Failed, "hit unexpected exception in indexer"); Assert.IsTrue(!indexerThread2.Failed, "hit unexpected exception in indexer2"); Assert.IsTrue(!searcherThread1.Failed, "hit unexpected exception in search1"); Assert.IsTrue(!searcherThread2.Failed, "hit unexpected exception in search2"); //System.out.println(" Writer: " + indexerThread.count + " iterations"); //System.out.println("Searcher 1: " + searcherThread1.count + " searchers created"); //System.out.println("Searcher 2: " + searcherThread2.count + " searchers created"); }
public virtual void TestImmediateDiskFullWithThreads() { int NUM_THREADS = 3; int numIterations = TEST_NIGHTLY ? 10 : 3; for (int iter = 0; iter < numIterations; iter++) { if (VERBOSE) { Console.WriteLine("\nTEST: iter=" + iter); } MockDirectoryWrapper dir = NewMockDirectory(); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergeScheduler(new ConcurrentMergeScheduler()).SetMergePolicy(NewLogMergePolicy(4))); ((ConcurrentMergeScheduler)writer.Config.MergeScheduler).SetSuppressExceptions(); dir.MaxSizeInBytes = 4 * 1024 + 20 * iter; IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(writer, true); } for (int i = 0; i < NUM_THREADS; i++) { threads[i].Start(); } for (int i = 0; i < NUM_THREADS; i++) { // Without fix for LUCENE-1130: one of the // threads will hang threads[i].Join(); Assert.IsTrue(threads[i].Error == null, "hit unexpected Throwable"); } // Make sure once disk space is avail again, we can // cleanly close: dir.MaxSizeInBytes = 0; writer.Dispose(false); dir.Dispose(); } }
// Runs test, with multiple threads, using the specific // failure to trigger an IOException public virtual void TestMultipleThreadsFailure(Func <IConcurrentMergeScheduler> newScheduler, Failure failure) { int NUM_THREADS = 3; for (int iter = 0; iter < 2; iter++) { if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter); } MockDirectoryWrapper dir = NewMockDirectory(); var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)) .SetMaxBufferedDocs(2) .SetMergeScheduler(newScheduler()) .SetMergePolicy(NewLogMergePolicy(4)); IndexWriter writer = new IndexWriter(dir, config); var scheduler = config.mergeScheduler as IConcurrentMergeScheduler; if (scheduler != null) { scheduler.SetSuppressExceptions(); } IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(writer, true, NewField); } for (int i = 0; i < NUM_THREADS; i++) { threads[i].Start(); } Thread.Sleep(10); dir.FailOn(failure); failure.SetDoFail(); for (int i = 0; i < NUM_THREADS; i++) { threads[i].Join(); Assert.IsTrue(threads[i].Error == null, "hit unexpected Throwable"); } bool success = false; try { writer.Dispose(false); success = true; } catch (IOException) { failure.ClearDoFail(); writer.Dispose(false); } if (VERBOSE) { Console.WriteLine("TEST: success=" + success); } if (success) { IndexReader reader = DirectoryReader.Open(dir); IBits delDocs = MultiFields.GetLiveDocs(reader); for (int j = 0; j < reader.MaxDoc; j++) { if (delDocs == null || !delDocs.Get(j)) { reader.Document(j); reader.GetTermVectors(j); } } reader.Dispose(); } dir.Dispose(); } }
// Runs test, with multiple threads, using the specific // failure to trigger an IOException public virtual void _testMultipleThreadsFailure(MockDirectoryWrapper.Failure failure) { int NUM_THREADS = 3; for (int iter = 0; iter < 2; iter++) { if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter); } MockDirectoryWrapper dir = NewMockDirectory(); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergeScheduler(new ConcurrentMergeScheduler()).SetMergePolicy(NewLogMergePolicy(4))); ((ConcurrentMergeScheduler)writer.Config.MergeScheduler).SetSuppressExceptions(); IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(writer, true); } for (int i = 0; i < NUM_THREADS; i++) { threads[i].Start(); } Thread.Sleep(10); dir.FailOn(failure); failure.SetDoFail(); for (int i = 0; i < NUM_THREADS; i++) { threads[i].Join(); Assert.IsTrue(threads[i].Error == null, "hit unexpected Throwable"); } bool success = false; try { writer.Dispose(false); success = true; } catch (IOException ioe) { failure.ClearDoFail(); writer.Dispose(false); } if (VERBOSE) { Console.WriteLine("TEST: success=" + success); } if (success) { IndexReader reader = DirectoryReader.Open(dir); Bits delDocs = MultiFields.GetLiveDocs(reader); for (int j = 0; j < reader.MaxDoc; j++) { if (delDocs == null || !delDocs.Get(j)) { reader.Document(j); reader.GetTermVectors(j); } } reader.Dispose(); } dir.Dispose(); } }
public virtual void TestDirectory() { Store.Directory indexDir = NewDirectory(); Store.Directory taxoDir = NewDirectory(); IndexWriter w = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); var tw = new DirectoryTaxonomyWriter(taxoDir); // first empty commit w.Commit(); tw.Commit(); var mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null); FacetsConfig config = new FacetsConfig(); config.SetMultiValued("field", true); AtomicBoolean stop = new AtomicBoolean(); // How many unique facets to index before stopping: int ordLimit = TEST_NIGHTLY ? 100000 : 6000; var indexer = new IndexerThread(w, config, tw, mgr, ordLimit, stop); indexer.Start(); try { while (!stop.Get()) { SearcherAndTaxonomy pair = mgr.Acquire(); try { //System.out.println("search maxOrd=" + pair.taxonomyReader.getSize()); FacetsCollector sfc = new FacetsCollector(); pair.searcher.Search(new MatchAllDocsQuery(), sfc); Facets facets = GetTaxonomyFacetCounts(pair.taxonomyReader, config, sfc); FacetResult result = facets.GetTopChildren(10, "field"); if (pair.searcher.IndexReader.NumDocs > 0) { //System.out.println(pair.taxonomyReader.getSize()); Assert.True(result.ChildCount > 0); Assert.True(result.LabelValues.Length > 0); } //if (VERBOSE) { //System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0))); //} } finally { mgr.Release(pair); } } } finally { indexer.Join(); } if (VERBOSE) { Console.WriteLine("TEST: now stop"); } IOUtils.Close(mgr, tw, w, taxoDir, indexDir); }
/* Run one indexer and 2 searchers against single index as stress test. */ public virtual void RunTest(Directory directory) { TimedThread[] threads = new TimedThread[4]; IndexWriter writer = new IndexWriter(directory, ANALYZER, true); // Establish a base index of 100 docs: for (int i = 0; i < 100; i++) { Document d = new Document(); d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.UN_TOKENIZED)); d.Add(new Field("contents", English.IntToEnglish(i), Field.Store.NO, Field.Index.TOKENIZED)); writer.AddDocument(d); } writer.Flush(); IndexerThread indexerThread = new IndexerThread(writer, threads); threads[0] = indexerThread; indexerThread.Start(); IndexerThread indexerThread2 = new IndexerThread(writer, threads); threads[1] = indexerThread2; indexerThread2.Start(); SearcherThread searcherThread1 = new SearcherThread(directory, threads); threads[2] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(directory, threads); threads[3] = searcherThread2; searcherThread2.Start(); indexerThread.Join(); indexerThread2.Join(); searcherThread1.Join(); searcherThread2.Join(); writer.Close(); Assert.IsTrue(!indexerThread.failed, "hit unexpected exception in indexer"); Assert.IsTrue(!indexerThread2.failed, "hit unexpected exception in indexer2"); Assert.IsTrue(!searcherThread1.failed, "hit unexpected exception in search1"); Assert.IsTrue(!searcherThread2.failed, "hit unexpected exception in search2"); //System.out.println(" Writer: " + indexerThread.count + " iterations"); //System.out.println("Searcher 1: " + searcherThread1.count + " searchers created"); //System.out.println("Searcher 2: " + searcherThread2.count + " searchers created"); }