public virtual void TestOpenIfChangedNRTToCommit() { Directory dir = NewDirectory(); // Can't use RIW because it randomly commits: IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); Document doc = new Document(); doc.Add(NewStringField("field", "value", Field.Store.NO)); w.AddDocument(doc); w.Commit(); IList <IndexCommit> commits = DirectoryReader.ListCommits(dir); Assert.AreEqual(1, commits.Count); w.AddDocument(doc); DirectoryReader r = DirectoryReader.Open(w, true); Assert.AreEqual(2, r.NumDocs); IndexReader r2 = DirectoryReader.OpenIfChanged(r, commits[0]); Assert.IsNotNull(r2); r.Dispose(); Assert.AreEqual(1, r2.NumDocs); w.Dispose(); r2.Dispose(); dir.Dispose(); }
public override void Run() { try { DirectoryReader open = null; for (int i = 0; i < Num; i++) { Documents.Document doc = new Documents.Document(); // docs.NextDoc(); doc.Add(NewStringField("id", "test", Field.Store.NO)); Writer.UpdateDocument(new Term("id", "test"), doc); if (Random().Next(3) == 0) { if (open == null) { open = DirectoryReader.Open(Writer, true); } DirectoryReader reader = DirectoryReader.OpenIfChanged(open); if (reader != null) { open.Dispose(); open = reader; } Assert.AreEqual(1, open.NumDocs, "iter: " + i + " numDocs: " + open.NumDocs + " del: " + open.NumDeletedDocs + " max: " + open.MaxDoc); } } if (open != null) { open.Dispose(); } } catch (Exception e) { throw new Exception(e.Message, e); } }
public virtual void TestPrepareCommit() { Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy(5))); writer.Commit(); for (int i = 0; i < 23; i++) { AddDoc(writer); } DirectoryReader reader = DirectoryReader.Open(dir); Assert.AreEqual(0, reader.NumDocs); writer.PrepareCommit(); IndexReader reader2 = DirectoryReader.Open(dir); Assert.AreEqual(0, reader2.NumDocs); writer.Commit(); IndexReader reader3 = DirectoryReader.OpenIfChanged(reader); Assert.IsNotNull(reader3); Assert.AreEqual(0, reader.NumDocs); Assert.AreEqual(0, reader2.NumDocs); Assert.AreEqual(23, reader3.NumDocs); reader.Dispose(); reader2.Dispose(); for (int i = 0; i < 17; i++) { AddDoc(writer); } Assert.AreEqual(23, reader3.NumDocs); reader3.Dispose(); reader = DirectoryReader.Open(dir); Assert.AreEqual(23, reader.NumDocs); reader.Dispose(); writer.PrepareCommit(); reader = DirectoryReader.Open(dir); Assert.AreEqual(23, reader.NumDocs); reader.Dispose(); writer.Commit(); reader = DirectoryReader.Open(dir); Assert.AreEqual(40, reader.NumDocs); reader.Dispose(); writer.Dispose(); dir.Dispose(); }
internal virtual ReaderCouple RefreshReader(DirectoryReader reader, TestReopen test, int modify, bool hasChanges) { UninterruptableMonitor.Enter(createReaderMutex); try { DirectoryReader r = null; if (test != null) { test.ModifyIndex(modify); r = test.OpenReader(); } DirectoryReader refreshed = null; try { refreshed = DirectoryReader.OpenIfChanged(reader); if (refreshed == null) { refreshed = reader; } } finally { if (refreshed == null && r != null) { // Hit exception -- close opened reader r.Dispose(); } } if (hasChanges) { if (refreshed == reader) { Assert.Fail("No new DirectoryReader instance created during refresh."); } } else { if (refreshed != reader) { Assert.Fail("New DirectoryReader instance created during refresh even though index had no changes."); } } return(new ReaderCouple(r, refreshed)); } finally { UninterruptableMonitor.Exit(createReaderMutex); } }
public override void Run() { Random rnd = LuceneTestCase.Random; while (!stopped) { if (index % 2 == 0) { // refresh reader synchronized ReaderCouple c = (outerInstance.RefreshReader(r, test, index, true)); readersToClose.Add(c.newReader); readersToClose.Add(c.refreshedReader); readers.Add(c); // prevent too many readers break; } else { // not synchronized DirectoryReader refreshed = DirectoryReader.OpenIfChanged(r); if (refreshed == null) { refreshed = r; } IndexSearcher searcher = #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION outerInstance. #endif NewSearcher(refreshed); ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("field1", "a" + rnd.Next(refreshed.MaxDoc))), null, 1000).ScoreDocs; if (hits.Length > 0) { searcher.Doc(hits[0].Doc); } if (refreshed != r) { refreshed.Dispose(); } } UninterruptableMonitor.Enter(this); try { UninterruptableMonitor.Wait(this, TimeSpan.FromMilliseconds(TestUtil.NextInt32(Random, 1, 100))); } finally { UninterruptableMonitor.Exit(this); } } }
public override void Run() { Random rnd = LuceneTestCase.Random(); while (!Stopped) { if (Index % 2 == 0) { // refresh reader synchronized ReaderCouple c = (OuterInstance.RefreshReader(r, Test, Index, true)); ReadersToClose.Add(c.NewReader); ReadersToClose.Add(c.RefreshedReader); Readers.Add(c); // prevent too many readers break; } else { // not synchronized DirectoryReader refreshed = DirectoryReader.OpenIfChanged(r); if (refreshed == null) { refreshed = r; } IndexSearcher searcher = OuterInstance.NewSearcher(refreshed); ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("field1", "a" + rnd.Next(refreshed.MaxDoc))), null, 1000).ScoreDocs; if (hits.Length > 0) { searcher.Doc(hits[0].Doc); } if (refreshed != r) { refreshed.Dispose(); } } lock (this) { Monitor.Wait(this, TimeSpan.FromMilliseconds(TestUtil.NextInt(Random(), 1, 100))); } } }
public override void Run() { try { Document doc = new Document(); DirectoryReader r = DirectoryReader.Open(dir); Field f = newStringField("f", "", Field.Store.NO); doc.Add(f); int count = 0; do { if (failed) { break; } for (int j = 0; j < 10; j++) { string s = finalI + "_" + Convert.ToString(count++); f.SetStringValue(s); w.AddDocument(doc); w.Commit(); DirectoryReader r2 = DirectoryReader.OpenIfChanged(r); Assert.IsNotNull(r2); Assert.IsTrue(!r2.Equals(r)); r.Dispose(); r = r2; Assert.AreEqual(1, r.DocFreq(new Term("f", s)), "term=f:" + s + "; r=" + r); } } while ((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) < endTime); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results r.Dispose(); } catch (Exception t) when(t.IsThrowable()) { failed.Value = (true); throw RuntimeException.Create(t); } }
public override void Run() { try { Document doc = new Document(); DirectoryReader r = DirectoryReader.Open(Dir); Field f = NewStringField("f", "", Field.Store.NO); doc.Add(f); int count = 0; do { if (Failed.Get()) { break; } for (int j = 0; j < 10; j++) { string s = FinalI + "_" + Convert.ToString(count++); f.SetStringValue(s); w.AddDocument(doc); w.Commit(); DirectoryReader r2 = DirectoryReader.OpenIfChanged(r); Assert.IsNotNull(r2); Assert.IsTrue(!r2.Equals(r)); r.Dispose(); r = r2; Assert.AreEqual(1, r.DocFreq(new Term("f", s)), "term=f:" + s + "; r=" + r); } } while (Environment.TickCount < EndTime); r.Dispose(); } catch (Exception t) { Failed.Set(true); throw new Exception(t.Message, t); } }
protected override void DoSearching(TaskScheduler es, long stopTime) { bool anyOpenDelFiles = false; DirectoryReader r = DirectoryReader.Open(m_writer, true); while (Environment.TickCount < stopTime && !m_failed) { if (Random.NextBoolean()) { if (Verbose) { Console.WriteLine("TEST: now reopen r=" + r); } DirectoryReader r2 = DirectoryReader.OpenIfChanged(r); if (r2 != null) { r.Dispose(); r = r2; } } else { if (Verbose) { Console.WriteLine("TEST: now close reader=" + r); } r.Dispose(); m_writer.Commit(); ICollection <string> openDeletedFiles = ((MockDirectoryWrapper)m_dir).GetOpenDeletedFiles(); if (openDeletedFiles.Count > 0) { Console.WriteLine("OBD files: " + openDeletedFiles); } anyOpenDelFiles |= openDeletedFiles.Count > 0; //Assert.AreEqual("open but deleted: " + openDeletedFiles, 0, openDeletedFiles.Size()); if (Verbose) { Console.WriteLine("TEST: now open"); } r = DirectoryReader.Open(m_writer, true); } if (Verbose) { Console.WriteLine("TEST: got new reader=" + r); } //System.out.println("numDocs=" + r.NumDocs + " //openDelFileCount=" + dir.openDeleteFileCount()); if (r.NumDocs > 0) { fixedSearcher = new IndexSearcher(r, es); SmokeTestSearcher(fixedSearcher); RunSearchThreads(Environment.TickCount + 500); } } r.Dispose(); //System.out.println("numDocs=" + r.NumDocs + " openDelFileCount=" + dir.openDeleteFileCount()); ICollection <string> openDeletedFiles_ = ((MockDirectoryWrapper)m_dir).GetOpenDeletedFiles(); if (openDeletedFiles_.Count > 0) { Console.WriteLine("OBD files: " + openDeletedFiles_); } anyOpenDelFiles |= openDeletedFiles_.Count > 0; Assert.IsFalse(anyOpenDelFiles, "saw non-zero open-but-deleted count"); }
public virtual void Test() { MockDirectoryWrapper dir = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors")); dir.PreventDoubleWrite = false; double rate = Random.NextDouble() * 0.01; //System.out.println("rate=" + rate); dir.RandomIOExceptionRateOnOpen = rate; int iters = AtLeast(20); LineFileDocs docs = new LineFileDocs(Random, DefaultCodecSupportsDocValues); IndexReader r = null; DirectoryReader r2 = null; bool any = false; MockDirectoryWrapper dirCopy = null; int lastNumDocs = 0; for (int iter = 0; iter < iters; iter++) { IndexWriter w = null; if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter); } try { MockAnalyzer analyzer = new MockAnalyzer(Random); analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); if (VERBOSE) { // Do this ourselves instead of relying on LTC so // we see incrementing messageID: iwc.SetInfoStream(new TextWriterInfoStream(Console.Out)); } var ms = iwc.MergeScheduler; if (ms is IConcurrentMergeScheduler) { ((IConcurrentMergeScheduler)ms).SetSuppressExceptions(); } w = new IndexWriter(dir, iwc); if (r != null && Random.Next(5) == 3) { if (Random.NextBoolean()) { if (VERBOSE) { Console.WriteLine("TEST: addIndexes IR[]"); } w.AddIndexes(new IndexReader[] { r }); } else { if (VERBOSE) { Console.WriteLine("TEST: addIndexes Directory[]"); } w.AddIndexes(new Directory[] { dirCopy }); } } else { if (VERBOSE) { Console.WriteLine("TEST: addDocument"); } w.AddDocument(docs.NextDoc()); } dir.RandomIOExceptionRateOnOpen = 0.0; w.Dispose(); w = null; // NOTE: this is O(N^2)! Only enable for temporary debugging: //dir.setRandomIOExceptionRateOnOpen(0.0); //TestUtil.CheckIndex(dir); //dir.setRandomIOExceptionRateOnOpen(rate); // Verify numDocs only increases, to catch IndexWriter // accidentally deleting the index: dir.RandomIOExceptionRateOnOpen = 0.0; Assert.IsTrue(DirectoryReader.IndexExists(dir)); if (r2 == null) { r2 = DirectoryReader.Open(dir); } else { DirectoryReader r3 = DirectoryReader.OpenIfChanged(r2); if (r3 != null) { r2.Dispose(); r2 = r3; } } Assert.IsTrue(r2.NumDocs >= lastNumDocs, "before=" + lastNumDocs + " after=" + r2.NumDocs); lastNumDocs = r2.NumDocs; //System.out.println("numDocs=" + lastNumDocs); dir.RandomIOExceptionRateOnOpen = rate; any = true; if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter + ": success"); } } catch (IOException ioe) { if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter + ": exception"); Console.WriteLine(ioe.ToString()); Console.Write(ioe.StackTrace); } if (w != null) { // NOTE: leave random IO exceptions enabled here, // to verify that rollback does not try to write // anything: w.Rollback(); } } if (any && r == null && Random.NextBoolean()) { // Make a copy of a non-empty index so we can use // it to addIndexes later: dir.RandomIOExceptionRateOnOpen = 0.0; r = DirectoryReader.Open(dir); dirCopy = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors.copy")); HashSet <string> files = new HashSet <string>(); foreach (string file in dir.ListAll()) { dir.Copy(dirCopy, file, file, IOContext.DEFAULT); files.Add(file); } dirCopy.Sync(files); // Have IW kiss the dir so we remove any leftover // files ... we can easily have leftover files at // the time we take a copy because we are holding // open a reader: (new IndexWriter(dirCopy, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)))).Dispose(); dirCopy.RandomIOExceptionRate = rate; dir.RandomIOExceptionRateOnOpen = rate; } } if (r2 != null) { r2.Dispose(); } if (r != null) { r.Dispose(); dirCopy.Dispose(); } dir.Dispose(); }
public override void Run() { try { while (Operations.Get() > 0) { int oper = rand.Next(100); if (oper < CommitPercent) { if (NumCommitting.IncrementAndGet() <= MaxConcurrentCommits) { IDictionary <int, long> newCommittedModel; long version; DirectoryReader oldReader; lock (OuterInstance) { newCommittedModel = new Dictionary <int, long>(OuterInstance.Model); // take a snapshot version = OuterInstance.SnapshotCount++; oldReader = OuterInstance.Reader; oldReader.IncRef(); // increment the reference since we will use this for reopening } DirectoryReader newReader; if (rand.Next(100) < SoftCommitPercent) { // assertU(h.Commit("softCommit","true")); if (Random().NextBoolean()) { if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": call writer.getReader"); } newReader = Writer.GetReader(true); } else { if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": reopen reader=" + oldReader + " version=" + version); } newReader = DirectoryReader.OpenIfChanged(oldReader, Writer.w, true); } } else { // assertU(commit()); if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": commit+reopen reader=" + oldReader + " version=" + version); } Writer.Commit(); if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": now reopen after commit"); } newReader = DirectoryReader.OpenIfChanged(oldReader); } // Code below assumes newReader comes w/ // extra ref: if (newReader == null) { oldReader.IncRef(); newReader = oldReader; } oldReader.DecRef(); lock (OuterInstance) { // install the new reader if it's newest (and check the current version since another reader may have already been installed) //System.out.println(Thread.currentThread().getName() + ": newVersion=" + newReader.getVersion()); Debug.Assert(newReader.RefCount > 0); Debug.Assert(OuterInstance.Reader.RefCount > 0); if (newReader.Version > OuterInstance.Reader.Version) { if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": install new reader=" + newReader); } OuterInstance.Reader.DecRef(); OuterInstance.Reader = newReader; // Silly: forces fieldInfos to be // loaded so we don't hit IOE on later // reader.toString newReader.ToString(); // install this snapshot only if it's newer than the current one if (version >= OuterInstance.CommittedModelClock) { if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": install new model version=" + version); } OuterInstance.CommittedModel = newCommittedModel; OuterInstance.CommittedModelClock = version; } else { if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": skip install new model version=" + version); } } } else { // if the same reader, don't decRef. if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": skip install new reader=" + newReader); } newReader.DecRef(); } } } NumCommitting.DecrementAndGet(); } else { int id = rand.Next(Ndocs); object sync = OuterInstance.SyncArr[id]; // set the lastId before we actually change it sometimes to try and // uncover more race conditions between writing and reading bool before = Random().NextBoolean(); if (before) { OuterInstance.LastId = id; } // We can't concurrently update the same document and retain our invariants of increasing values // since we can't guarantee what order the updates will be executed. lock (sync) { long val = OuterInstance.Model[id]; long nextVal = Math.Abs(val) + 1; if (oper < CommitPercent + DeletePercent) { // assertU("<delete><id>" + id + "</id></delete>"); // add tombstone first if (Tombstones) { Document d = new Document(); d.Add(OuterInstance.NewStringField("id", "-" + Convert.ToString(id), Documents.Field.Store.YES)); d.Add(OuterInstance.NewField(OuterInstance.Field, Convert.ToString(nextVal), StoredOnlyType)); Writer.UpdateDocument(new Term("id", "-" + Convert.ToString(id)), d); } if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": term delDocs id:" + id + " nextVal=" + nextVal); } Writer.DeleteDocuments(new Term("id", Convert.ToString(id))); OuterInstance.Model[id] = -nextVal; } else if (oper < CommitPercent + DeletePercent + DeleteByQueryPercent) { //assertU("<delete><query>id:" + id + "</query></delete>"); // add tombstone first if (Tombstones) { Document d = new Document(); d.Add(OuterInstance.NewStringField("id", "-" + Convert.ToString(id), Documents.Field.Store.YES)); d.Add(OuterInstance.NewField(OuterInstance.Field, Convert.ToString(nextVal), StoredOnlyType)); Writer.UpdateDocument(new Term("id", "-" + Convert.ToString(id)), d); } if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": query delDocs id:" + id + " nextVal=" + nextVal); } Writer.DeleteDocuments(new TermQuery(new Term("id", Convert.ToString(id)))); OuterInstance.Model[id] = -nextVal; } else { // assertU(adoc("id",Integer.toString(id), field, Long.toString(nextVal))); Document d = new Document(); d.Add(OuterInstance.NewStringField("id", Convert.ToString(id), Documents.Field.Store.YES)); d.Add(OuterInstance.NewField(OuterInstance.Field, Convert.ToString(nextVal), StoredOnlyType)); if (VERBOSE) { Console.WriteLine("TEST: " + Thread.CurrentThread.Name + ": u id:" + id + " val=" + nextVal); } Writer.UpdateDocument(new Term("id", Convert.ToString(id)), d); if (Tombstones) { // remove tombstone after new addition (this should be optional?) Writer.DeleteDocuments(new Term("id", "-" + Convert.ToString(id))); } OuterInstance.Model[id] = nextVal; } } if (!before) { OuterInstance.LastId = id; } } } } catch (Exception e) { Console.WriteLine(Thread.CurrentThread.Name + ": FAILED: unexpected exception"); Console.WriteLine(e.StackTrace); throw new Exception(e.Message, e); } }
public virtual void TestIndexing() { DirectoryInfo tmpDir = CreateTempDir("TestNeverDelete"); BaseDirectoryWrapper d = NewFSDirectory(tmpDir); // We want to "see" files removed if Lucene removed // them. this is still worth running on Windows since // some files the IR opens and closes. if (d is MockDirectoryWrapper) { ((MockDirectoryWrapper)d).NoDeleteOpenFile = false; } RandomIndexWriter w = new RandomIndexWriter(Random(), d, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetIndexDeletionPolicy(NoDeletionPolicy.INSTANCE)); w.w.Config.SetMaxBufferedDocs(TestUtil.NextInt(Random(), 5, 30)); w.Commit(); ThreadClass[] indexThreads = new ThreadClass[Random().Next(4)]; long stopTime = Environment.TickCount + AtLeast(1000); for (int x = 0; x < indexThreads.Length; x++) { indexThreads[x] = new ThreadAnonymousInnerClassHelper(w, stopTime, NewStringField, NewTextField); indexThreads[x].Name = "Thread " + x; indexThreads[x].Start(); } HashSet <string> allFiles = new HashSet <string>(); DirectoryReader r = DirectoryReader.Open(d); while (Environment.TickCount < stopTime) { IndexCommit ic = r.IndexCommit; if (VERBOSE) { Console.WriteLine("TEST: check files: " + ic.FileNames); } allFiles.AddAll(ic.FileNames); // Make sure no old files were removed foreach (string fileName in allFiles) { Assert.IsTrue(SlowFileExists(d, fileName), "file " + fileName + " does not exist"); } DirectoryReader r2 = DirectoryReader.OpenIfChanged(r); if (r2 != null) { r.Dispose(); r = r2; } Thread.Sleep(1); } r.Dispose(); foreach (ThreadClass t in indexThreads) { t.Join(); } w.Dispose(); d.Dispose(); System.IO.Directory.Delete(tmpDir.FullName, true); }
public virtual void TestReopenOnCommit() { Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetIndexDeletionPolicy(new KeepAllCommits <IndexCommit>()).SetMaxBufferedDocs(-1).SetMergePolicy(NewLogMergePolicy(10))); for (int i = 0; i < 4; i++) { Document doc = new Document(); doc.Add(NewStringField("id", "" + i, Field.Store.NO)); writer.AddDocument(doc); IDictionary <string, string> data = new Dictionary <string, string>(); data["index"] = i + ""; writer.CommitData = data; writer.Commit(); } for (int i = 0; i < 4; i++) { writer.DeleteDocuments(new Term("id", "" + i)); IDictionary <string, string> data = new Dictionary <string, string>(); data["index"] = (4 + i) + ""; writer.CommitData = data; writer.Commit(); } writer.Dispose(); DirectoryReader r = DirectoryReader.Open(dir); Assert.AreEqual(0, r.NumDocs); ICollection <IndexCommit> commits = DirectoryReader.ListCommits(dir); foreach (IndexCommit commit in commits) { DirectoryReader r2 = DirectoryReader.OpenIfChanged(r, commit); Assert.IsNotNull(r2); Assert.IsTrue(r2 != r); IDictionary <string, string> s = commit.UserData; int v; if (s.Count == 0) { // First commit created by IW v = -1; } else { v = Convert.ToInt32(s["index"]); } if (v < 4) { Assert.AreEqual(1 + v, r2.NumDocs); } else { Assert.AreEqual(7 - v, r2.NumDocs); } r.Dispose(); r = r2; } r.Dispose(); dir.Dispose(); }
protected override void DoSearching(TaskScheduler es, DateTime stopTime) { bool anyOpenDelFiles = false; DirectoryReader r = DirectoryReader.Open(Writer, true); while (DateTime.UtcNow < stopTime && !Failed.Get()) { if (Random().NextBoolean()) { if (VERBOSE) { Console.WriteLine("TEST: now reopen r=" + r); } DirectoryReader r2 = DirectoryReader.OpenIfChanged(r); if (r2 != null) { r.Dispose(); r = r2; } } else { if (VERBOSE) { Console.WriteLine("TEST: now close reader=" + r); } r.Dispose(); Writer.Commit(); ISet <string> openDeletedFiles = ((MockDirectoryWrapper)Dir).OpenDeletedFiles; if (openDeletedFiles.Count > 0) { Console.WriteLine("OBD files: " + openDeletedFiles); } anyOpenDelFiles |= openDeletedFiles.Count > 0; //Assert.AreEqual("open but deleted: " + openDeletedFiles, 0, openDeletedFiles.Size()); if (VERBOSE) { Console.WriteLine("TEST: now open"); } r = DirectoryReader.Open(Writer, true); } if (VERBOSE) { Console.WriteLine("TEST: got new reader=" + r); } //System.out.println("numDocs=" + r.NumDocs + " //openDelFileCount=" + dir.openDeleteFileCount()); if (r.NumDocs > 0) { FixedSearcher = new IndexSearcher(r, es); SmokeTestSearcher(FixedSearcher); RunSearchThreads(DateTime.UtcNow.AddMilliseconds(500)); } } r.Dispose(); //System.out.println("numDocs=" + r.NumDocs + " openDelFileCount=" + dir.openDeleteFileCount()); ISet <string> openDeletedFiles_ = ((MockDirectoryWrapper)Dir).OpenDeletedFiles; if (openDeletedFiles_.Count > 0) { Console.WriteLine("OBD files: " + openDeletedFiles_); } anyOpenDelFiles |= openDeletedFiles_.Count > 0; Assert.IsFalse(anyOpenDelFiles, "saw non-zero open-but-deleted count"); }
public virtual void TestThreadSafety() { Directory dir = NewDirectory(); // NOTE: this also controls the number of threads! int n = TestUtil.NextInt(Random(), 20, 40); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); for (int i = 0; i < n; i++) { writer.AddDocument(CreateDocument(i, 3)); } writer.ForceMerge(1); writer.Dispose(); TestReopen test = new TestReopenAnonymousInnerClassHelper3(this, dir, n); IList <ReaderCouple> readers = new SynchronizedCollection <ReaderCouple>(); DirectoryReader firstReader = DirectoryReader.Open(dir); DirectoryReader reader = firstReader; ReaderThread[] threads = new ReaderThread[n]; ISet <DirectoryReader> readersToClose = new ConcurrentHashSet <DirectoryReader>(new HashSet <DirectoryReader>()); for (int i = 0; i < n; i++) { if (i % 2 == 0) { DirectoryReader refreshed = DirectoryReader.OpenIfChanged(reader); if (refreshed != null) { readersToClose.Add(reader); reader = refreshed; } } DirectoryReader r = reader; int index = i; ReaderThreadTask task; if (i < 4 || (i >= 10 && i < 14) || i > 18) { task = new ReaderThreadTaskAnonymousInnerClassHelper(this, test, readers, readersToClose, r, index); } else { task = new ReaderThreadTaskAnonymousInnerClassHelper2(this, readers); } threads[i] = new ReaderThread(task); threads[i].Start(); } lock (this) { Monitor.Wait(this, TimeSpan.FromMilliseconds(1000)); } for (int i = 0; i < n; i++) { if (threads[i] != null) { threads[i].StopThread(); } } for (int i = 0; i < n; i++) { if (threads[i] != null) { threads[i].Join(); if (threads[i].Error != null) { string msg = "Error occurred in thread " + threads[i].Name + ":\n" + threads[i].Error.Message; Assert.Fail(msg); } } } foreach (DirectoryReader readerToClose in readersToClose) { readerToClose.Dispose(); } firstReader.Dispose(); reader.Dispose(); foreach (DirectoryReader readerToClose in readersToClose) { AssertReaderClosed(readerToClose, true); } AssertReaderClosed(reader, true); AssertReaderClosed(firstReader, true); dir.Dispose(); }
private void DoTestReopenWithCommit(Random random, Directory dir, bool withReopen) { IndexWriter iwriter = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).SetOpenMode(OpenMode_e.CREATE).SetMergeScheduler(new SerialMergeScheduler()).SetMergePolicy(NewLogMergePolicy())); iwriter.Commit(); DirectoryReader reader = DirectoryReader.Open(dir); try { int M = 3; FieldType customType = new FieldType(TextField.TYPE_STORED); customType.Tokenized = false; FieldType customType2 = new FieldType(TextField.TYPE_STORED); customType2.Tokenized = false; customType2.OmitNorms = true; FieldType customType3 = new FieldType(); customType3.Stored = true; for (int i = 0; i < 4; i++) { for (int j = 0; j < M; j++) { Document doc = new Document(); doc.Add(NewField("id", i + "_" + j, customType)); doc.Add(NewField("id2", i + "_" + j, customType2)); doc.Add(NewField("id3", i + "_" + j, customType3)); iwriter.AddDocument(doc); if (i > 0) { int k = i - 1; int n = j + k * M; Document prevItereationDoc = reader.Document(n); Assert.IsNotNull(prevItereationDoc); string id = prevItereationDoc.Get("id"); Assert.AreEqual(k + "_" + j, id); } } iwriter.Commit(); if (withReopen) { // reopen DirectoryReader r2 = DirectoryReader.OpenIfChanged(reader); if (r2 != null) { reader.Dispose(); reader = r2; } } else { // recreate reader.Dispose(); reader = DirectoryReader.Open(dir); } } } finally { iwriter.Dispose(); reader.Dispose(); } }
public override void Run() { DirectoryReader currentReader = null; Random random = LuceneTestCase.Random(); try { Document doc = new Document(); doc.Add(new TextField("id", "1", Field.Store.NO)); Writer.AddDocument(doc); Holder.Reader = currentReader = Writer.GetReader(true); Term term = new Term("id"); for (int i = 0; i < NumOps && !Holder.Stop; i++) { float nextOp = (float)random.NextDouble(); if (nextOp < 0.3) { term.Set("id", new BytesRef("1")); Writer.UpdateDocument(term, doc); } else if (nextOp < 0.5) { Writer.AddDocument(doc); } else { term.Set("id", new BytesRef("1")); Writer.DeleteDocuments(term); } if (Holder.Reader != currentReader) { Holder.Reader = currentReader; if (Countdown) { Countdown = false; Latch.Signal(); } } if (random.NextBoolean()) { Writer.Commit(); DirectoryReader newReader = DirectoryReader.OpenIfChanged(currentReader); if (newReader != null) { currentReader.DecRef(); currentReader = newReader; } if (currentReader.NumDocs == 0) { Writer.AddDocument(doc); } } } } catch (Exception e) { Failed = e; } finally { Holder.Reader = null; if (Countdown) { Latch.Signal(); } if (currentReader != null) { try { currentReader.DecRef(); } catch (IOException e) { } } } if (VERBOSE) { Console.WriteLine("writer stopped - forced by reader: " + Holder.Stop); } }
public override void Run() { DirectoryReader reader = null; bool success = false; try { Random random = Random; while (numUpdates.GetAndDecrement() > 0) { double group = random.NextDouble(); Term t; if (group < 0.1) { t = new Term("updKey", "g0"); } else if (group < 0.5) { t = new Term("updKey", "g1"); } else if (group < 0.8) { t = new Term("updKey", "g2"); } else { t = new Term("updKey", "g3"); } // System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t); if (random.NextBoolean()) // sometimes unset a value { // System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=UNSET"); writer.UpdateBinaryDocValue(t, f, null); writer.UpdateNumericDocValue(t, cf, null); } else { long updValue = random.Next(); // System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=" + updValue); writer.UpdateBinaryDocValue(t, f, TestBinaryDocValuesUpdates.ToBytes(updValue)); writer.UpdateNumericDocValue(t, cf, updValue * 2); } if (random.NextDouble() < 0.2) { // delete a random document int doc = random.Next(numDocs); // System.out.println("[" + Thread.currentThread().getName() + "] deleteDoc=doc" + doc); writer.DeleteDocuments(new Term("id", "doc" + doc)); } if (random.NextDouble() < 0.05) // commit every 20 updates on average { // System.out.println("[" + Thread.currentThread().getName() + "] commit"); writer.Commit(); } if (random.NextDouble() < 0.1) // reopen NRT reader (apply updates), on average once every 10 updates { if (reader is null) { // System.out.println("[" + Thread.currentThread().getName() + "] open NRT"); reader = DirectoryReader.Open(writer, true); } else { // System.out.println("[" + Thread.currentThread().getName() + "] reopen NRT"); DirectoryReader r2 = DirectoryReader.OpenIfChanged(reader, writer, true); if (r2 != null) { reader.Dispose(); reader = r2; } } } } // System.out.println("[" + Thread.currentThread().getName() + "] DONE"); success = true; } catch (Exception e) when(e.IsIOException()) { throw RuntimeException.Create(e); } finally { if (reader != null) { try { reader.Dispose(); } catch (Exception e) when(e.IsIOException()) { if (success) // suppress this exception only if there was another exception { throw RuntimeException.Create(e); } } } done.Signal(); } }
public override void Run() { DirectoryReader currentReader = null; Random random = LuceneTestCase.Random; try { Document doc = new Document(); doc.Add(new TextField("id", "1", Field.Store.NO)); writer.AddDocument(doc); holder.reader = currentReader = writer.GetReader(true); Term term = new Term("id"); for (int i = 0; i < numOps && !holder.stop; i++) { float nextOp = (float)random.NextDouble(); if (nextOp < 0.3) { term.Set("id", new BytesRef("1")); writer.UpdateDocument(term, doc); } else if (nextOp < 0.5) { writer.AddDocument(doc); } else { term.Set("id", new BytesRef("1")); writer.DeleteDocuments(term); } if (holder.reader != currentReader) { holder.reader = currentReader; if (countdown) { countdown = false; latch.Signal(); } } if (random.NextBoolean()) { writer.Commit(); DirectoryReader newReader = DirectoryReader.OpenIfChanged(currentReader); if (newReader != null) { currentReader.DecRef(); currentReader = newReader; } if (currentReader.NumDocs == 0) { writer.AddDocument(doc); } } } } catch (Exception e) { failed = e; } finally { holder.reader = null; if (countdown) { latch.Signal(); } if (currentReader != null) { try { currentReader.DecRef(); } #pragma warning disable 168 catch (IOException e) #pragma warning restore 168 { } } } if (Verbose) { Console.WriteLine("writer stopped - forced by reader: " + holder.stop); } }
public virtual void TestIndexing() { DirectoryInfo tmpDir = CreateTempDir("TestNeverDelete"); BaseDirectoryWrapper d = NewFSDirectory(tmpDir); // We want to "see" files removed if Lucene removed // them. this is still worth running on Windows since // some files the IR opens and closes. if (d is MockDirectoryWrapper) { ((MockDirectoryWrapper)d).NoDeleteOpenFile = false; } RandomIndexWriter w = new RandomIndexWriter(Random, d, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetIndexDeletionPolicy(NoDeletionPolicy.INSTANCE)); w.IndexWriter.Config.SetMaxBufferedDocs(TestUtil.NextInt32(Random, 5, 30)); w.Commit(); ThreadJob[] indexThreads = new ThreadJob[Random.Next(4)]; long stopTime = (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) + AtLeast(1000); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results for (int x = 0; x < indexThreads.Length; x++) { indexThreads[x] = new ThreadAnonymousClass(w, stopTime, NewStringField, NewTextField); indexThreads[x].Name = "Thread " + x; indexThreads[x].Start(); } ISet <string> allFiles = new JCG.HashSet <string>(); DirectoryReader r = DirectoryReader.Open(d); while (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond < stopTime) // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results { IndexCommit ic = r.IndexCommit; if (Verbose) { Console.WriteLine("TEST: check files: " + ic.FileNames); } allFiles.UnionWith(ic.FileNames); // Make sure no old files were removed foreach (string fileName in allFiles) { Assert.IsTrue(SlowFileExists(d, fileName), "file " + fileName + " does not exist"); } DirectoryReader r2 = DirectoryReader.OpenIfChanged(r); if (r2 != null) { r.Dispose(); r = r2; } Thread.Sleep(1); } r.Dispose(); foreach (ThreadJob t in indexThreads) { t.Join(); } w.Dispose(); d.Dispose(); System.IO.Directory.Delete(tmpDir.FullName, true); }
public virtual void TestManyReopensAndFields() { Directory dir = NewDirectory(); Random random = Random; IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); LogMergePolicy lmp = NewLogMergePolicy(); lmp.MergeFactor = 3; // merge often conf.SetMergePolicy(lmp); IndexWriter writer = new IndexWriter(dir, conf); bool isNRT = random.NextBoolean(); DirectoryReader reader; if (isNRT) { reader = DirectoryReader.Open(writer, true); } else { writer.Commit(); reader = DirectoryReader.Open(dir); } int numFields = random.Next(4) + 3; // 3-7 int numNDVFields = random.Next(numFields / 2) + 1; // 1-3 long[] fieldValues = new long[numFields]; bool[] fieldHasValue = new bool[numFields]; Arrays.Fill(fieldHasValue, true); for (int i = 0; i < fieldValues.Length; i++) { fieldValues[i] = 1; } int numRounds = AtLeast(15); int docID = 0; for (int i = 0; i < numRounds; i++) { int numDocs = AtLeast(5); // System.out.println("[" + Thread.currentThread().getName() + "]: round=" + i + ", numDocs=" + numDocs); for (int j = 0; j < numDocs; j++) { Document doc = new Document(); doc.Add(new StringField("id", "doc-" + docID, Store.NO)); doc.Add(new StringField("key", "all", Store.NO)); // update key // add all fields with their current value for (int f = 0; f < fieldValues.Length; f++) { if (f < numNDVFields) { doc.Add(new NumericDocValuesField("f" + f, fieldValues[f])); } else { doc.Add(new BinaryDocValuesField("f" + f, TestBinaryDocValuesUpdates.ToBytes(fieldValues[f]))); } } writer.AddDocument(doc); ++docID; } // if field's value was unset before, unset it from all new added documents too for (int field = 0; field < fieldHasValue.Length; field++) { if (!fieldHasValue[field]) { if (field < numNDVFields) { writer.UpdateNumericDocValue(new Term("key", "all"), "f" + field, null); } else { writer.UpdateBinaryDocValue(new Term("key", "all"), "f" + field, null); } } } int fieldIdx = random.Next(fieldValues.Length); string updateField = "f" + fieldIdx; if (random.NextBoolean()) { // System.out.println("[" + Thread.currentThread().getName() + "]: unset field '" + updateField + "'"); fieldHasValue[fieldIdx] = false; if (fieldIdx < numNDVFields) { writer.UpdateNumericDocValue(new Term("key", "all"), updateField, null); } else { writer.UpdateBinaryDocValue(new Term("key", "all"), updateField, null); } } else { fieldHasValue[fieldIdx] = true; if (fieldIdx < numNDVFields) { writer.UpdateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]); } else { writer.UpdateBinaryDocValue(new Term("key", "all"), updateField, TestBinaryDocValuesUpdates.ToBytes(++fieldValues[fieldIdx])); } // System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]); } if (random.NextDouble() < 0.2) { int deleteDoc = random.Next(docID); // might also delete an already deleted document, ok! writer.DeleteDocuments(new Term("id", "doc-" + deleteDoc)); // System.out.println("[" + Thread.currentThread().getName() + "]: deleted document: doc-" + deleteDoc); } // verify reader if (!isNRT) { writer.Commit(); } // System.out.println("[" + Thread.currentThread().getName() + "]: reopen reader: " + reader); DirectoryReader newReader = DirectoryReader.OpenIfChanged(reader); Assert.IsNotNull(newReader); reader.Dispose(); reader = newReader; // System.out.println("[" + Thread.currentThread().getName() + "]: reopened reader: " + reader); Assert.IsTrue(reader.NumDocs > 0); // we delete at most one document per round BytesRef scratch = new BytesRef(); foreach (AtomicReaderContext context in reader.Leaves) { AtomicReader r = context.AtomicReader; // System.out.println(((SegmentReader) r).getSegmentName()); IBits liveDocs = r.LiveDocs; for (int field = 0; field < fieldValues.Length; field++) { string f = "f" + field; BinaryDocValues bdv = r.GetBinaryDocValues(f); NumericDocValues ndv = r.GetNumericDocValues(f); IBits docsWithField = r.GetDocsWithField(f); if (field < numNDVFields) { Assert.IsNotNull(ndv); Assert.IsNull(bdv); } else { Assert.IsNull(ndv); Assert.IsNotNull(bdv); } int maxDoc = r.MaxDoc; for (int doc = 0; doc < maxDoc; doc++) { if (liveDocs is null || liveDocs.Get(doc)) { // System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + getValue(bdv, doc, scratch)); if (fieldHasValue[field]) { Assert.IsTrue(docsWithField.Get(doc)); if (field < numNDVFields) { Assert.AreEqual(fieldValues[field], ndv.Get(doc), "invalid value for doc=" + doc + ", field=" + f + ", reader=" + r); } else { Assert.AreEqual(fieldValues[field], TestBinaryDocValuesUpdates.GetValue(bdv, doc, scratch), "invalid value for doc=" + doc + ", field=" + f + ", reader=" + r); } } else { Assert.IsFalse(docsWithField.Get(doc)); } } } } } // System.out.println(); } IOUtils.Dispose(writer, reader, dir); }
public virtual void TestPrepareCommitRollback() { Directory dir = NewDirectory(); if (dir is MockDirectoryWrapper) { ((MockDirectoryWrapper)dir).PreventDoubleWrite = false; } IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy(5))); writer.Commit(); for (int i = 0; i < 23; i++) { AddDoc(writer); } DirectoryReader reader = DirectoryReader.Open(dir); Assert.AreEqual(0, reader.NumDocs); writer.PrepareCommit(); IndexReader reader2 = DirectoryReader.Open(dir); Assert.AreEqual(0, reader2.NumDocs); writer.Rollback(); IndexReader reader3 = DirectoryReader.OpenIfChanged(reader); Assert.IsNull(reader3); Assert.AreEqual(0, reader.NumDocs); Assert.AreEqual(0, reader2.NumDocs); reader.Dispose(); reader2.Dispose(); writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); for (int i = 0; i < 17; i++) { AddDoc(writer); } reader = DirectoryReader.Open(dir); Assert.AreEqual(0, reader.NumDocs); reader.Dispose(); writer.PrepareCommit(); reader = DirectoryReader.Open(dir); Assert.AreEqual(0, reader.NumDocs); reader.Dispose(); writer.Commit(); reader = DirectoryReader.Open(dir); Assert.AreEqual(17, reader.NumDocs); reader.Dispose(); writer.Dispose(); dir.Dispose(); }