public void TestSubclassConcurrentMergeScheduler() { MockDirectoryWrapper dir = NewMockDirectory(); dir.FailOn(new FailOnlyOnMerge()); Document doc = new Document(); Field idField = NewStringField("id", "", Field.Store.YES); doc.Add(idField); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergeScheduler(new MyMergeScheduler(this)).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetMergePolicy(NewLogMergePolicy())); LogMergePolicy logMP = (LogMergePolicy)writer.Config.MergePolicy; logMP.MergeFactor = 10; for (int i = 0; i < 20; i++) { writer.AddDocument(doc); } ((MyMergeScheduler)writer.Config.MergeScheduler).Sync(); writer.Dispose(); Assert.IsTrue(MergeThreadCreated); Assert.IsTrue(MergeCalled); Assert.IsTrue(ExcCalled); dir.Dispose(); }
// Runs test, with one thread, using the specific failure // to trigger an IOException public virtual void TestSingleThreadFailure(Func <IConcurrentMergeScheduler> newScheduler, Failure failure) { MockDirectoryWrapper dir = NewMockDirectory(); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(2).SetMergeScheduler(newScheduler())); Document doc = new Document(); FieldType customType = new FieldType(TextField.TYPE_STORED); customType.StoreTermVectors = true; customType.StoreTermVectorPositions = true; customType.StoreTermVectorOffsets = true; doc.Add(NewField("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj", customType)); for (int i = 0; i < 6; i++) { writer.AddDocument(doc); } dir.FailOn(failure); failure.SetDoFail(); try { writer.AddDocument(doc); writer.AddDocument(doc); writer.Commit(); Assert.Fail("did not hit exception"); } catch (IOException) { } failure.ClearDoFail(); writer.AddDocument(doc); writer.Dispose(false); dir.Dispose(); }
public void TestExceptionOnBackgroundThreadIsPropagatedToCallingThread() { using MockDirectoryWrapper dir = NewMockDirectory(); dir.FailOn(new FailOnlyOnMerge()); Document doc = new Document(); Field idField = NewStringField("id", "", Field.Store.YES); doc.Add(idField); var mergeScheduler = new ConcurrentMergeScheduler(); using IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergeScheduler(mergeScheduler).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetMergePolicy(NewLogMergePolicy())); LogMergePolicy logMP = (LogMergePolicy)writer.Config.MergePolicy; logMP.MergeFactor = 10; for (int i = 0; i < 20; i++) { writer.AddDocument(doc); } bool exceptionHit = false; try { mergeScheduler.Sync(); } catch (MergePolicy.MergeException) { exceptionHit = true; } assertTrue(exceptionHit); }
public virtual void TestExceptionDuringSave() { MockDirectoryWrapper dir = NewMockDirectory(); dir.FailOn(new FailureAnonymousInnerClassHelper(this, dir)); IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.CREATE_OR_APPEND))); writer.AddDocument(new Document()); writer.Commit(); PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy; try { psdp.Snapshot(); } catch (IOException ioe) { if (ioe.Message.Equals("now fail on purpose", StringComparison.Ordinal)) { // ok } else { throw ioe; } } Assert.AreEqual(0, psdp.SnapshotCount); writer.Dispose(); Assert.AreEqual(1, DirectoryReader.ListCommits(dir).Count); dir.Dispose(); }
public virtual void TestExceptionDuringSave() { MockDirectoryWrapper dir = NewMockDirectory(); dir.FailOn(new FailureAnonymousClass(this, dir)); IndexWriter writer = new IndexWriter(dir, GetConfig(Random, new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.CREATE_OR_APPEND))); writer.AddDocument(new Document()); writer.Commit(); PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy; try { psdp.Snapshot(); } catch (Exception ioe) when(ioe.IsIOException()) { if (ioe.Message.Equals("now fail on purpose", StringComparison.Ordinal)) { // ok } else { throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details) } } Assert.AreEqual(0, psdp.SnapshotCount); writer.Dispose(); Assert.AreEqual(1, DirectoryReader.ListCommits(dir).Count); dir.Dispose(); }
public virtual void TestTransactions_Mem( [ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler1, [ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler2) { Console.WriteLine("Start test"); // we cant use non-ramdir on windows, because this test needs to double-write. MockDirectoryWrapper dir1 = new MockDirectoryWrapper(Random(), new RAMDirectory()); MockDirectoryWrapper dir2 = new MockDirectoryWrapper(Random(), new RAMDirectory()); dir1.PreventDoubleWrite = false; dir2.PreventDoubleWrite = false; dir1.FailOn(new RandomFailure(this)); dir2.FailOn(new RandomFailure(this)); dir1.FailOnOpenInput = false; dir2.FailOnOpenInput = false; // We throw exceptions in deleteFile, which creates // leftover files: dir1.AssertNoUnrefencedFilesOnClose = false; dir2.AssertNoUnrefencedFilesOnClose = false; InitIndex(dir1); InitIndex(dir2); TimedThread[] threads = new TimedThread[3]; int numThread = 0; IndexerThread indexerThread = new IndexerThread(this, this, dir1, dir2, newScheduler1, newScheduler2, threads); threads[numThread++] = indexerThread; indexerThread.Start(); SearcherThread searcherThread1 = new SearcherThread(this, dir1, dir2, threads); threads[numThread++] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(this, dir1, dir2, threads); threads[numThread++] = searcherThread2; searcherThread2.Start(); for (int i = 0; i < numThread; i++) { threads[i].Join(); } for (int i = 0; i < numThread; i++) { Assert.IsTrue(!threads[i].Failed); } dir1.Dispose(); dir2.Dispose(); Console.WriteLine("End test"); }
public virtual void TestCorruptionAfterDiskFullDuringMerge() { MockDirectoryWrapper dir = NewMockDirectory(); //IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setReaderPooling(true)); IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(new SerialMergeScheduler()).SetReaderPooling(true).SetMergePolicy(NewLogMergePolicy(2))); // we can do this because we add/delete/add (and dont merge to "nothing") w.KeepFullyDeletedSegments = true; Document doc = new Document(); doc.Add(NewTextField("f", "doctor who", Field.Store.NO)); w.AddDocument(doc); w.Commit(); w.DeleteDocuments(new Term("f", "who")); w.AddDocument(doc); // disk fills up! FailTwiceDuringMerge ftdm = new FailTwiceDuringMerge(); ftdm.SetDoFail(); dir.FailOn(ftdm); try { w.Commit(); Assert.Fail("fake disk full IOExceptions not hit"); } #pragma warning disable 168 catch (IOException ioe) #pragma warning restore 168 { // expected Assert.IsTrue(ftdm.DidFail1 || ftdm.DidFail2); } TestUtil.CheckIndex(dir); ftdm.ClearDoFail(); w.AddDocument(doc); w.Dispose(); dir.Dispose(); }
public virtual void TestFlushExceptions() { MockDirectoryWrapper directory = NewMockDirectory(); FailOnlyOnFlush failure = new FailOnlyOnFlush(this); directory.FailOn(failure); IndexWriter writer = new IndexWriter(directory, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2)); Document doc = new Document(); Field idField = NewStringField("id", "", Field.Store.YES); doc.Add(idField); int extraCount = 0; for (int i = 0; i < 10; i++) { if (VERBOSE) { Console.WriteLine("TEST: iter=" + i); } for (int j = 0; j < 20; j++) { idField.StringValue = Convert.ToString(i * 20 + j); writer.AddDocument(doc); } // must cycle here because sometimes the merge flushes // the doc we just added and so there's nothing to // flush, and we don't hit the exception while (true) { writer.AddDocument(doc); failure.SetDoFail(); try { writer.Flush(true, true); if (failure.HitExc) { Assert.Fail("failed to hit IOException"); } extraCount++; } catch (IOException ioe) { if (VERBOSE) { Console.WriteLine(ioe.StackTrace); } failure.ClearDoFail(); break; } } Assert.AreEqual(20 * (i + 1) + extraCount, writer.NumDocs()); } writer.Dispose(); IndexReader reader = DirectoryReader.Open(directory); Assert.AreEqual(200 + extraCount, reader.NumDocs); reader.Dispose(); directory.Dispose(); }
// Runs test, with multiple threads, using the specific // failure to trigger an IOException public virtual void TestMultipleThreadsFailure(Func <IConcurrentMergeScheduler> newScheduler, Failure failure) { int NUM_THREADS = 3; for (int iter = 0; iter < 2; iter++) { if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter); } MockDirectoryWrapper dir = NewMockDirectory(); var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)) .SetMaxBufferedDocs(2) .SetMergeScheduler(newScheduler()) .SetMergePolicy(NewLogMergePolicy(4)); IndexWriter writer = new IndexWriter(dir, config); var scheduler = config.mergeScheduler as IConcurrentMergeScheduler; if (scheduler != null) { scheduler.SetSuppressExceptions(); } IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(writer, true, NewField); } for (int i = 0; i < NUM_THREADS; i++) { threads[i].Start(); } Thread.Sleep(10); dir.FailOn(failure); failure.SetDoFail(); for (int i = 0; i < NUM_THREADS; i++) { threads[i].Join(); Assert.IsTrue(threads[i].Error == null, "hit unexpected Throwable"); } bool success = false; try { writer.Dispose(false); success = true; } catch (IOException) { failure.ClearDoFail(); writer.Dispose(false); } if (VERBOSE) { Console.WriteLine("TEST: success=" + success); } if (success) { IndexReader reader = DirectoryReader.Open(dir); IBits delDocs = MultiFields.GetLiveDocs(reader); for (int j = 0; j < reader.MaxDoc; j++) { if (delDocs == null || !delDocs.Get(j)) { reader.Document(j); reader.GetTermVectors(j); } } reader.Dispose(); } dir.Dispose(); } }
public virtual void TestTransactions_Mem() { // we cant use non-ramdir on windows, because this test needs to double-write. MockDirectoryWrapper dir1 = new MockDirectoryWrapper(Random(), new RAMDirectory()); MockDirectoryWrapper dir2 = new MockDirectoryWrapper(Random(), new RAMDirectory()); dir1.PreventDoubleWrite = false; dir2.PreventDoubleWrite = false; dir1.FailOn(new RandomFailure(this)); dir2.FailOn(new RandomFailure(this)); dir1.FailOnOpenInput = false; dir2.FailOnOpenInput = false; // We throw exceptions in deleteFile, which creates // leftover files: dir1.AssertNoUnrefencedFilesOnClose = false; dir2.AssertNoUnrefencedFilesOnClose = false; InitIndex(dir1); InitIndex(dir2); TimedThread[] threads = new TimedThread[3]; int numThread = 0; IndexerThread indexerThread = new IndexerThread(this, this, dir1, dir2, threads); threads[numThread++] = indexerThread; indexerThread.Start(); SearcherThread searcherThread1 = new SearcherThread(this, dir1, dir2, threads); threads[numThread++] = searcherThread1; searcherThread1.Start(); SearcherThread searcherThread2 = new SearcherThread(this, dir1, dir2, threads); threads[numThread++] = searcherThread2; searcherThread2.Start(); for (int i = 0; i < numThread; i++) { threads[i].Join(); } for (int i = 0; i < numThread; i++) { Assert.IsTrue(!threads[i].Failed); } dir1.Dispose(); dir2.Dispose(); }