public SearcherContextInternal(Directory dir, Analyzer defaultAnalyzer, TimeSpan targetMinStale, TimeSpan targetMaxStale) { Analyzer = new PerFieldAnalyzerWrapper(defaultAnalyzer); _writer = new IndexWriter(dir, new IndexWriterConfig(LeoLuceneVersion.Version, Analyzer) { OpenMode = OpenMode.CREATE_OR_APPEND }); _trackingWriter = new TrackingIndexWriter(_writer); _searcherManager = new SearcherManager(_writer, true, null); _nrtReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(_trackingWriter, _searcherManager, targetMaxStale.TotalSeconds, targetMinStale.TotalSeconds); _nrtReopenThread.SetDaemon(true); _nrtReopenThread.Start(); }
static LuceneIndex() { System.IO.Directory.CreateDirectory(IndexDirectory); directory = new NRTCachingDirectory(new MMapDirectory(IndexDirectory), 256, 256); analyzer = new WordAnalyzer(LuceneVersion.LUCENE_48); writer = new IndexWriter(directory, new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer) { IndexDeletionPolicy = new KeepOnlyLastCommitDeletionPolicy(), MergedSegmentWarmer = new SimpleMergedSegmentWarmer(InfoStream.NO_OUTPUT), //UseCompoundFile = false, OpenMode = OpenMode.CREATE_OR_APPEND, UseReaderPooling = true, MergeScheduler = new TaskMergeScheduler(), MergePolicy = new TieredMergePolicy(), RAMBufferSizeMB = 1024, MaxBufferedDocs = 1024 }.SetInfoStream(InfoStream.NO_OUTPUT)); trackingWriter = new TrackingIndexWriter(writer); searcherKeeper = new SearcherManager(writer, false, null); reopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(trackingWriter, searcherKeeper, 3.0, 0.0) { Priority = ThreadPriority.BelowNormal, IsBackground = true }; reopenThread.Start(); new Thread(() => { while (true) { Thread.Sleep(30000); try { writer.Commit(); } catch {} } }) { IsBackground = true, Priority = ThreadPriority.BelowNormal }.Start(); }
public ThreadAnonymousInnerClassHelper2(TestControlledRealTimeReopenThread outerInstance, long lastGen, ControlledRealTimeReopenThread<IndexSearcher> thread, AtomicBoolean finished) { this.OuterInstance = outerInstance; this.LastGen = lastGen; this.thread = thread; this.Finished = finished; }
protected internal override void DoAfterWriter(TaskScheduler es) { double minReopenSec = 0.01 + 0.05 * Random().NextDouble(); double maxReopenSec = minReopenSec * (1.0 + 10 * Random().NextDouble()); if (VERBOSE) { Console.WriteLine("TEST: make SearcherManager maxReopenSec=" + maxReopenSec + " minReopenSec=" + minReopenSec); } GenWriter = new TrackingIndexWriter(Writer); SearcherFactory sf = new SearcherFactoryAnonymousInnerClassHelper(this, es); NrtNoDeletes = new SearcherManager(Writer, false, sf); NrtDeletes = new SearcherManager(Writer, true, sf); NrtDeletesThread = new ControlledRealTimeReopenThread<>(GenWriter, NrtDeletes, maxReopenSec, minReopenSec); NrtDeletesThread.Name = "NRTDeletes Reopen Thread"; NrtDeletesThread.Priority = Math.Min(Thread.CurrentThread.Priority + 2, Thread.MAX_PRIORITY); NrtDeletesThread.SetDaemon(true); NrtDeletesThread.Start(); NrtNoDeletesThread = new ControlledRealTimeReopenThread<>(GenWriter, NrtNoDeletes, maxReopenSec, minReopenSec); NrtNoDeletesThread.Name = "NRTNoDeletes Reopen Thread"; NrtNoDeletesThread.Priority = Math.Min(Thread.CurrentThread.Priority + 2, Thread.MAX_PRIORITY); NrtNoDeletesThread.SetDaemon(true); NrtNoDeletesThread.Start(); }
/* * LUCENE-3528 - NRTManager hangs in certain situations */ public virtual void TestThreadStarvationNoDeleteNRTReader() { IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); conf.SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES); Directory d = NewDirectory(); CountDownLatch latch = new CountDownLatch(1); CountDownLatch signal = new CountDownLatch(1); LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal); TrackingIndexWriter writer = new TrackingIndexWriter(_writer); SearcherManager manager = new SearcherManager(_writer, false, null); Document doc = new Document(); doc.Add(NewTextField("test", "test", Field.Store.YES)); writer.AddDocument(doc); manager.MaybeRefresh(); ThreadClass t = new ThreadAnonymousInnerClassHelper(this, latch, signal, writer, manager); t.Start(); _writer.WaitAfterUpdate = true; // wait in addDocument to let some reopens go through long lastGen = writer.UpdateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen Assert.IsFalse(manager.SearcherCurrent); // false since there is a delete in the queue IndexSearcher searcher = manager.Acquire(); try { Assert.AreEqual(2, searcher.IndexReader.NumDocs()); } finally { manager.Release(searcher); } ControlledRealTimeReopenThread<IndexSearcher> thread = new ControlledRealTimeReopenThread<IndexSearcher>(writer, manager, 0.01, 0.01); thread.Start(); // start reopening if (VERBOSE) { Console.WriteLine("waiting now for generation " + lastGen); } AtomicBoolean finished = new AtomicBoolean(false); ThreadClass waiter = new ThreadAnonymousInnerClassHelper2(this, lastGen, thread, finished); waiter.Start(); manager.MaybeRefresh(); waiter.Join(1000); if (!finished.Get()) { waiter.Interrupt(); Assert.Fail("thread deadlocked on waitForGeneration"); } thread.Dispose(); thread.Join(); IOUtils.Close(manager, _writer, d); }
// LUCENE-5461 public virtual void TestCRTReopen() { //test behaving badly //should be high enough int maxStaleSecs = 20; //build crap data just to store it. string s = " abcdefghijklmnopqrstuvwxyz "; char[] chars = s.ToCharArray(); StringBuilder builder = new StringBuilder(2048); for (int i = 0; i < 2048; i++) { builder.Append(chars[Random().Next(chars.Length)]); } string content = builder.ToString(); SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, new MockAnalyzer(Random())); config.SetIndexDeletionPolicy(sdp); config.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE_OR_APPEND); IndexWriter iw = new IndexWriter(dir, config); SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory()); TrackingIndexWriter tiw = new TrackingIndexWriter(iw); ControlledRealTimeReopenThread<IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread<IndexSearcher>(tiw, sm, maxStaleSecs, 0); controlledRealTimeReopenThread.SetDaemon(true); controlledRealTimeReopenThread.Start(); IList<Thread> commitThreads = new List<Thread>(); for (int i = 0; i < 500; i++) { if (i > 0 && i % 50 == 0) { Thread commitThread = new Thread(new RunnableAnonymousInnerClassHelper(this, sdp, dir, iw)); commitThread.Start(); commitThreads.Add(commitThread); } Document d = new Document(); d.Add(new TextField("count", i + "", Field.Store.NO)); d.Add(new TextField("content", content, Field.Store.YES)); long start = DateTime.Now.Millisecond; long l = tiw.AddDocument(d); controlledRealTimeReopenThread.WaitForGeneration(l); long wait = DateTime.Now.Millisecond - start; Assert.IsTrue(wait < (maxStaleSecs * 1000), "waited too long for generation " + wait); IndexSearcher searcher = sm.Acquire(); TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10); sm.Release(searcher); Assert.AreEqual(1, td.TotalHits); } foreach (Thread commitThread in commitThreads) { commitThread.Join(); } controlledRealTimeReopenThread.Dispose(); sm.Dispose(); iw.Dispose(); dir.Dispose(); }