public virtual void TestReferenceDecrementIllegally([ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler) { Directory dir = NewDirectory(); var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())) .SetMergeScheduler(newScheduler()); IndexWriter writer = new IndexWriter(dir, config); SearcherManager sm = new SearcherManager(writer, false, new SearcherFactory()); writer.AddDocument(new Document()); writer.Commit(); sm.MaybeRefreshBlocking(); IndexSearcher acquire = sm.Acquire(); IndexSearcher acquire2 = sm.Acquire(); sm.Release(acquire); sm.Release(acquire2); acquire = sm.Acquire(); acquire.IndexReader.DecRef(); sm.Release(acquire); Assert.Throws <InvalidOperationException>(() => sm.Acquire(), "acquire should have thrown an InvalidOperationException since we modified the refCount outside of the manager"); // sm.Dispose(); -- already closed writer.Dispose(); dir.Dispose(); }
public virtual void TestReferenceDecrementIllegally() { Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(new ConcurrentMergeScheduler())); SearcherManager sm = new SearcherManager(writer, false, new SearcherFactory()); writer.AddDocument(new Document()); writer.Commit(); sm.MaybeRefreshBlocking(); IndexSearcher acquire = sm.Acquire(); IndexSearcher acquire2 = sm.Acquire(); sm.Release(acquire); sm.Release(acquire2); acquire = sm.Acquire(); acquire.IndexReader.DecRef(); sm.Release(acquire); try { sm.Acquire(); Assert.Fail("acquire should have thrown an InvalidOperationException since we modified the refCount outside of the manager"); } catch (InvalidOperationException ex) { // } // sm.Dispose(); -- already closed writer.Dispose(); dir.Dispose(); }
public virtual void TestEnsureOpen() { Directory dir = NewDirectory(); (new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose(); SearcherManager sm = new SearcherManager(dir, null); IndexSearcher s = sm.Acquire(); sm.Dispose(); // this should succeed; sm.Release(s); try { // this should fail sm.Acquire(); } catch (Exception e) when(e.IsAlreadyClosedException()) { // ok } try { // this should fail sm.MaybeRefresh(); } catch (Exception e) when(e.IsAlreadyClosedException()) { // ok } dir.Dispose(); }
public virtual void TestDeleteDocuments() { Directory directory = CreateIndex(); IndexWriter writer = GetWriter(directory); ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory()); IndexSearcher searcher = mgr.Acquire(); TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100); Assert.AreEqual(1, topDocs.TotalHits); TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer); long result = mgrWriter.DeleteDocuments(new TermQuery(new Term("foo", "0"))); Assert.AreEqual(1, result); // writer.Commit(); Assert.IsTrue(writer.HasDeletions()); mgr.MaybeRefresh(); searcher = mgr.Acquire(); topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100); Assert.AreEqual(0, topDocs.TotalHits); }
public IEnumerable <LuceneSearchEntry> ExecuteQuery(string indexFolder, LSearch.Query query) { var searchIndex = LStore.FSDirectory.Open(indexFolder); var searchManager = new LSearch.SearcherManager(searchIndex, null); // try to release locs searchManager.MaybeRefreshBlocking(); //execute search var searcher = searchManager.Acquire(); var rawResults = searcher.Search(query, 20); //convert raw results to model var outResults = new List <LuceneSearchEntry>(); for (var i = 0; i < rawResults.ScoreDocs.Length; i++) { var result = rawResults.ScoreDocs[i]; var doc = searcher.Doc(result.Doc); outResults.Add(new LuceneSearchEntry { Rank = i, Id = doc.Get("id"), Uri = doc.Get("uri"), Title = doc.Get("title"), Snippet = doc.Get("snippet"), Score = result.Score }); } return(outResults); }
/* * LUCENE-3528 - NRTManager hangs in certain situations */ public virtual void TestThreadStarvationNoDeleteNRTReader() { IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); conf.SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES); Directory d = NewDirectory(); CountDownLatch latch = new CountDownLatch(1); CountDownLatch signal = new CountDownLatch(1); LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal); TrackingIndexWriter writer = new TrackingIndexWriter(_writer); SearcherManager manager = new SearcherManager(_writer, false, null); Document doc = new Document(); doc.Add(NewTextField("test", "test", Field.Store.YES)); writer.AddDocument(doc); manager.MaybeRefresh(); ThreadClass t = new ThreadAnonymousInnerClassHelper(this, latch, signal, writer, manager); t.Start(); _writer.WaitAfterUpdate = true; // wait in addDocument to let some reopens go through long lastGen = writer.UpdateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen Assert.IsFalse(manager.SearcherCurrent); // false since there is a delete in the queue IndexSearcher searcher = manager.Acquire(); try { Assert.AreEqual(2, searcher.IndexReader.NumDocs()); } finally { manager.Release(searcher); } ControlledRealTimeReopenThread <IndexSearcher> thread = new ControlledRealTimeReopenThread <IndexSearcher>(writer, manager, 0.01, 0.01); thread.Start(); // start reopening if (VERBOSE) { Console.WriteLine("waiting now for generation " + lastGen); } AtomicBoolean finished = new AtomicBoolean(false); ThreadClass waiter = new ThreadAnonymousInnerClassHelper2(this, lastGen, thread, finished); waiter.Start(); manager.MaybeRefresh(); waiter.Join(1000); if (!finished.Get()) { waiter.Interrupt(); Assert.Fail("thread deadlocked on waitForGeneration"); } thread.Dispose(); thread.Join(); IOUtils.Close(manager, _writer, d); }
public virtual void TestEnsureOpen() { Directory dir = NewDirectory(); (new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose(); SearcherManager sm = new SearcherManager(dir, null); IndexSearcher s = sm.Acquire(); sm.Dispose(); // this should succeed; sm.Release(s); try { // this should fail sm.Acquire(); } #pragma warning disable 168 catch (ObjectDisposedException e) #pragma warning restore 168 { // ok } try { // this should fail sm.MaybeRefresh(); } #pragma warning disable 168 catch (ObjectDisposedException e) #pragma warning restore 168 { // ok } dir.Dispose(); }
public override void Run() { try { IDictionary <string, int?> values = new Dictionary <string, int?>(); IList <string> allIDs = new SynchronizedList <string>(); StartingGun.Wait(); for (int iter = 0; iter < Iters; iter++) { // Add/update a document Document doc = new Document(); // Threads must not update the same id at the // same time: if (ThreadRandom.NextDouble() <= AddChance) { string id = string.Format(CultureInfo.InvariantCulture, "{0}_{1:X4}", ThreadID, ThreadRandom.Next(IdCount)); int field = ThreadRandom.Next(int.MaxValue); doc.Add(new StringField("id", id, Field.Store.YES)); doc.Add(new Int32Field("field", (int)field, Field.Store.YES)); w.UpdateDocument(new Term("id", id), doc); Rt.Add(id, field); if (!values.ContainsKey(id))//Key didn't exist before { allIDs.Add(id); } values[id] = field; } if (allIDs.Count > 0 && ThreadRandom.NextDouble() <= DeleteChance) { string randomID = allIDs[ThreadRandom.Next(allIDs.Count)]; w.DeleteDocuments(new Term("id", randomID)); Rt.Delete(randomID); values[randomID] = Missing; } if (ThreadRandom.NextDouble() <= ReopenChance || Rt.Count > 10000) { //System.out.println("refresh @ " + rt.Size()); Mgr.MaybeRefresh(); if (VERBOSE) { IndexSearcher s = Mgr.Acquire(); try { Console.WriteLine("TEST: reopen " + s); } finally { Mgr.Release(s); } Console.WriteLine("TEST: " + values.Count + " values"); } } if (ThreadRandom.Next(10) == 7) { Assert.AreEqual(null, Rt.Get("foo")); } if (allIDs.Count > 0) { string randomID = allIDs[ThreadRandom.Next(allIDs.Count)]; int? expected = values[randomID]; if (expected == Missing) { expected = null; } Assert.AreEqual(expected, Rt.Get(randomID), "id=" + randomID); } } } catch (Exception t) { throw new Exception(t.Message, t); } }
// LUCENE-5461 public virtual void TestCRTReopen() { //test behaving badly //should be high enough int maxStaleSecs = 20; //build crap data just to store it. string s = " abcdefghijklmnopqrstuvwxyz "; char[] chars = s.ToCharArray(); StringBuilder builder = new StringBuilder(2048); for (int i = 0; i < 2048; i++) { builder.Append(chars[Random().Next(chars.Length)]); } string content = builder.ToString(); SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, new MockAnalyzer(Random())); config.SetIndexDeletionPolicy(sdp); config.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE_OR_APPEND); IndexWriter iw = new IndexWriter(dir, config); SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory()); TrackingIndexWriter tiw = new TrackingIndexWriter(iw); ControlledRealTimeReopenThread <IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(tiw, sm, maxStaleSecs, 0); controlledRealTimeReopenThread.SetDaemon(true); controlledRealTimeReopenThread.Start(); IList <Thread> commitThreads = new List <Thread>(); for (int i = 0; i < 500; i++) { if (i > 0 && i % 50 == 0) { Thread commitThread = new Thread(new RunnableAnonymousInnerClassHelper(this, sdp, dir, iw)); commitThread.Start(); commitThreads.Add(commitThread); } Document d = new Document(); d.Add(new TextField("count", i + "", Field.Store.NO)); d.Add(new TextField("content", content, Field.Store.YES)); long start = DateTime.Now.Millisecond; long l = tiw.AddDocument(d); controlledRealTimeReopenThread.WaitForGeneration(l); long wait = DateTime.Now.Millisecond - start; Assert.IsTrue(wait < (maxStaleSecs * 1000), "waited too long for generation " + wait); IndexSearcher searcher = sm.Acquire(); TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10); sm.Release(searcher); Assert.AreEqual(1, td.TotalHits); } foreach (Thread commitThread in commitThreads) { commitThread.Join(); } controlledRealTimeReopenThread.Dispose(); sm.Dispose(); iw.Dispose(); dir.Dispose(); }
public virtual void TestIntermediateClose() { Directory dir = NewDirectory(); // Test can deadlock if we use SMS: IConcurrentMergeScheduler scheduler; #if !FEATURE_CONCURRENTMERGESCHEDULER scheduler = new TaskMergeScheduler(); #else scheduler = new ConcurrentMergeScheduler(); #endif IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(scheduler)); writer.AddDocument(new Document()); writer.Commit(); CountdownEvent awaitEnterWarm = new CountdownEvent(1); CountdownEvent awaitClose = new CountdownEvent(1); AtomicBoolean triedReopen = new AtomicBoolean(false); //TaskScheduler es = Random().NextBoolean() ? null : Executors.newCachedThreadPool(new NamedThreadFactory("testIntermediateClose")); TaskScheduler es = Random().NextBoolean() ? null : TaskScheduler.Default; SearcherFactory factory = new SearcherFactoryAnonymousInnerClassHelper2(this, awaitEnterWarm, awaitClose, triedReopen, es); SearcherManager searcherManager = Random().NextBoolean() ? new SearcherManager(dir, factory) : new SearcherManager(writer, Random().NextBoolean(), factory); if (VERBOSE) { Console.WriteLine("sm created"); } IndexSearcher searcher = searcherManager.Acquire(); try { assertEquals(1, searcher.IndexReader.NumDocs); } finally { searcherManager.Release(searcher); } writer.AddDocument(new Document()); writer.Commit(); AtomicBoolean success = new AtomicBoolean(false); Exception[] exc = new Exception[1]; ThreadClass thread = new ThreadClass(() => new RunnableAnonymousInnerClassHelper(this, triedReopen, searcherManager, success, exc).Run()); thread.Start(); if (VERBOSE) { Console.WriteLine("THREAD started"); } awaitEnterWarm.Wait(); if (VERBOSE) { Console.WriteLine("NOW call close"); } searcherManager.Dispose(); awaitClose.Signal(); thread.Join(); try { searcherManager.Acquire(); fail("already closed"); } #pragma warning disable 168 catch (ObjectDisposedException ex) #pragma warning restore 168 { // expected } assertFalse(success.Get()); assertTrue(triedReopen.Get()); assertNull("" + exc[0], exc[0]); writer.Dispose(); dir.Dispose(); //if (es != null) //{ // es.shutdown(); // es.awaitTermination(1, TimeUnit.SECONDS); //} }
public virtual void TestCRTReopen() { //test behaving badly //should be high enough int maxStaleSecs = 20; //build crap data just to store it. string s = " abcdefghijklmnopqrstuvwxyz "; char[] chars = s.ToCharArray(); StringBuilder builder = new StringBuilder(2048); for (int i = 0; i < 2048; i++) { builder.Append(chars[Random.Next(chars.Length)]); } string content = builder.ToString(); SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128); IndexWriterConfig config = new IndexWriterConfig( #pragma warning disable 612, 618 Version.LUCENE_46, #pragma warning restore 612, 618 new MockAnalyzer(Random)); config.SetIndexDeletionPolicy(sdp); config.SetOpenMode(OpenMode.CREATE_OR_APPEND); IndexWriter iw = new IndexWriter(dir, config); SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory()); TrackingIndexWriter tiw = new TrackingIndexWriter(iw); ControlledRealTimeReopenThread <IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(tiw, sm, maxStaleSecs, 0); controlledRealTimeReopenThread.IsBackground = (true); controlledRealTimeReopenThread.Start(); IList <ThreadJob> commitThreads = new JCG.List <ThreadJob>(); for (int i = 0; i < 500; i++) { if (i > 0 && i % 50 == 0) { ThreadJob commitThread = new RunnableAnonymousClass(this, sdp, dir, iw); commitThread.Start(); commitThreads.Add(commitThread); } Document d = new Document(); d.Add(new TextField("count", i + "", Field.Store.NO)); d.Add(new TextField("content", content, Field.Store.YES)); long start = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results long l = tiw.AddDocument(d); controlledRealTimeReopenThread.WaitForGeneration(l); long wait = (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - start; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results assertTrue("waited too long for generation " + wait, wait < (maxStaleSecs * 1000)); IndexSearcher searcher = sm.Acquire(); TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10); sm.Release(searcher); assertEquals(1, td.TotalHits); } foreach (ThreadJob commitThread in commitThreads) { commitThread.Join(); } controlledRealTimeReopenThread.Dispose(); sm.Dispose(); iw.Dispose(); dir.Dispose(); }
public void InitSearcher(long[] nodeVersions) { Debug.Assert(CurrentShardSearcher == null); Array.Copy(nodeVersions, 0, CurrentNodeVersions, 0, CurrentNodeVersions.Length); CurrentShardSearcher = new ShardIndexSearcher(this, (long[])CurrentNodeVersions.Clone(), Mgr.Acquire().IndexReader, MyNodeID); }
public virtual void TestTryDeleteDocument() { Directory directory = CreateIndex(); IndexWriter writer = GetWriter(directory); ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory()); TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer); IndexSearcher searcher = mgr.Acquire(); TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100); Assert.AreEqual(1, topDocs.TotalHits); long result; if (Random().NextBoolean()) { IndexReader r = DirectoryReader.Open(writer, true); result = mgrWriter.TryDeleteDocument(r, 0); r.Dispose(); } else { result = mgrWriter.TryDeleteDocument(searcher.IndexReader, 0); } // The tryDeleteDocument should have succeeded: Assert.IsTrue(result != -1); Assert.IsTrue(writer.HasDeletions()); if (Random().NextBoolean()) { writer.Commit(); } Assert.IsTrue(writer.HasDeletions()); mgr.MaybeRefresh(); searcher = mgr.Acquire(); topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100); Assert.AreEqual(0, topDocs.TotalHits); }
public void TestStraightForwardDemonstration() { RAMDirectory indexDir = new RAMDirectory(); Analyzer standardAnalyzer = new StandardAnalyzer(TEST_VERSION_CURRENT); IndexWriterConfig indexConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, standardAnalyzer); IndexWriter indexWriter = new IndexWriter(indexDir, indexConfig); TrackingIndexWriter trackingWriter = new TrackingIndexWriter(indexWriter); Document doc = new Document(); doc.Add(new Int32Field("id", 1, Field.Store.YES)); doc.Add(new StringField("name", "Doc1", Field.Store.YES)); trackingWriter.AddDocument(doc); SearcherManager searcherManager = new SearcherManager(indexWriter, applyAllDeletes: true, null); //Reopen SearcherManager every 1 secs via background thread if no thread waiting for newer generation. //Reopen SearcherManager after .2 secs if another thread IS waiting on a newer generation. var controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(trackingWriter, searcherManager, 1, 0.2); //Start() will start a seperate thread that will invoke the object's Run(). However, //calling Run() directly would execute that code on the current thread rather then a new thread //which would defeat the purpose of using controlledRealTimeReopenThread. This aspect of the API //is not as intuitive as it could be. ie. Call Start() not Run(). controlledRealTimeReopenThread.IsBackground = true; //Set as a background thread controlledRealTimeReopenThread.Name = "Controlled Real Time Reopen Thread"; controlledRealTimeReopenThread.Priority = (ThreadPriority)Math.Min((int)Thread.CurrentThread.Priority + 2, (int)ThreadPriority.Highest); controlledRealTimeReopenThread.Start(); //An indexSearcher only sees Doc1 IndexSearcher indexSearcher = searcherManager.Acquire(); try { TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1); assertEquals(1, topDocs.TotalHits); //There is only one doc } finally { searcherManager.Release(indexSearcher); } //Add a 2nd document doc = new Document(); doc.Add(new Int32Field("id", 2, Field.Store.YES)); doc.Add(new StringField("name", "Doc2", Field.Store.YES)); trackingWriter.AddDocument(doc); //Demonstrate that we can only see the first doc because we haven't //waited 1 sec or called WaitForGeneration indexSearcher = searcherManager.Acquire(); try { TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1); assertEquals(1, topDocs.TotalHits); //Can see both docs due to auto refresh after 1.1 secs } finally { searcherManager.Release(indexSearcher); } //Demonstrate that we can see both docs after we wait a little more //then 1 sec so that controlledRealTimeReopenThread max interval is exceeded //and it calls MaybeRefresh Thread.Sleep(1100); //wait 1.1 secs as ms indexSearcher = searcherManager.Acquire(); try { TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1); assertEquals(2, topDocs.TotalHits); //Can see both docs due to auto refresh after 1.1 secs } finally { searcherManager.Release(indexSearcher); } //Add a 3rd document doc = new Document(); doc.Add(new Int32Field("id", 3, Field.Store.YES)); doc.Add(new StringField("name", "Doc3", Field.Store.YES)); long generation = trackingWriter.AddDocument(doc); //Demonstrate that if we call WaitForGeneration our wait will be // .2 secs or less (the min interval we set earlier) and then we will //see all 3 documents. Stopwatch stopwatch = Stopwatch.StartNew(); controlledRealTimeReopenThread.WaitForGeneration(generation); stopwatch.Stop(); assertTrue(stopwatch.Elapsed.TotalMilliseconds <= 200 + 30); //30ms is fudged factor to account for call overhead. indexSearcher = searcherManager.Acquire(); try { TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1); assertEquals(3, topDocs.TotalHits); //Can see both docs due to auto refresh after 1.1 secs } finally { searcherManager.Release(indexSearcher); } controlledRealTimeReopenThread.Dispose(); searcherManager.Dispose(); indexWriter.Dispose(); indexDir.Dispose(); }