public override int DoLogic() { PerfRunData runData = RunData; // Get initial reader IndexWriter w = runData.IndexWriter; if (w == null) { throw new Exception("please open the writer before invoking NearRealtimeReader"); } if (runData.GetIndexReader() != null) { throw new Exception("please close the existing reader before invoking NearRealtimeReader"); } long t = J2N.Time.CurrentTimeMilliseconds(); DirectoryReader r = DirectoryReader.Open(w, true); runData.SetIndexReader(r); // Transfer our reference to runData r.DecRef(); // TODO: gather basic metrics for reporting -- eg mean, // stddev, min/max reopen latencies // Parent sequence sets stopNow reopenCount = 0; while (!Stop) { long waitForMsec = (pauseMSec - (J2N.Time.CurrentTimeMilliseconds() - t)); if (waitForMsec > 0) { Thread.Sleep((int)waitForMsec); //System.out.println("NRT wait: " + waitForMsec + " msec"); } t = J2N.Time.CurrentTimeMilliseconds(); DirectoryReader newReader = DirectoryReader.OpenIfChanged(r); if (newReader != null) { int delay = (int)(J2N.Time.CurrentTimeMilliseconds() - t); if (reopenTimes.Length == reopenCount) { reopenTimes = ArrayUtil.Grow(reopenTimes, 1 + reopenCount); } reopenTimes[reopenCount++] = delay; // TODO: somehow we need to enable warming, here runData.SetIndexReader(newReader); // Transfer our reference to runData newReader.DecRef(); r = newReader; } } Stop = false; return(reopenCount); }
internal static void SetIndexReaderSearcher() { lock (typeof(PatternsForEachTokenLucene)) { try { FSDirectory index = NIOFSDirectory.Open(indexDir); if (reader == null) { reader = DirectoryReader.Open(index); searcher = new IndexSearcher(reader); } else { DirectoryReader newreader = DirectoryReader.OpenIfChanged(reader); if (newreader != null) { reader.Close(); reader = newreader; searcher = new IndexSearcher(reader); } } } catch (IOException e) { throw new Exception(e); } } }
public IEnumerable <(string Term, T Data)> Search(string[] searchTerms, int maxHits = 100) { if (!_reader.IsCurrent()) { using (_reader) { _reader = DirectoryReader.OpenIfChanged(_reader); _searcher = new IndexSearcher(_reader); } } MultiPhraseQuery query = new MultiPhraseQuery(); foreach (string searchTerm in searchTerms) { query.Add(new Term(TermsFieldName, searchTerm)); } TopDocs hits = _searcher.Search(query, maxHits); foreach (ScoreDoc topDoc in hits.ScoreDocs) { Document document = _reader.Document(topDoc.Doc); string serializedData = document.GetField(DataFieldName).GetStringValue(); T t = _serializer.Deserialize <T>(serializedData); string term = document.GetField(TermsFieldName).GetStringValue(); yield return(term, t); } }
protected override SearcherAndTaxonomy RefreshIfNeeded(SearcherAndTaxonomy @ref) { // Must re-open searcher first, otherwise we may get a // new reader that references ords not yet known to the // taxonomy reader: IndexReader r = @ref.searcher.IndexReader; IndexReader newReader = DirectoryReader.OpenIfChanged((DirectoryReader)r); if (newReader == null) { return(null); } else { var tr = TaxonomyReader.OpenIfChanged(@ref.taxonomyReader); if (tr == null) { @ref.taxonomyReader.IncRef(); tr = @ref.taxonomyReader; } else if (taxoWriter != null && taxoWriter.TaxonomyEpoch != taxoEpoch) { IOUtils.Close(newReader, tr); throw new ThreadStateException("DirectoryTaxonomyWriter.replaceTaxonomy was called, which is not allowed when using SearcherTaxonomyManager"); } return(new SearcherAndTaxonomy(SearcherManager.GetSearcher(searcherFactory, newReader), tr)); } }
private void ReopenReader() { DirectoryReader newReader = DirectoryReader.OpenIfChanged(reader); assertNotNull(newReader); reader.Dispose(); reader = newReader; }
public override int DoLogic() { DirectoryReader r = RunData.GetIndexReader(); DirectoryReader nr = DirectoryReader.OpenIfChanged(r); if (nr != null) { RunData.SetIndexReader(nr); nr.DecRef(); } r.DecRef(); return(1); }
private static DirectoryReader RefreshReader(DirectoryReader reader) { DirectoryReader oldReader = reader; reader = DirectoryReader.OpenIfChanged(reader); if (reader != null) { oldReader.Dispose(); return(reader); } else { return(oldReader); } }
protected override IndexSearcher RefreshIfNeeded(IndexSearcher referenceToRefresh) { IndexReader r = referenceToRefresh.IndexReader; Debug.Assert(r is DirectoryReader, "searcher's IndexReader should be a DirectoryReader, but got " + r); IndexReader newReader = DirectoryReader.OpenIfChanged((DirectoryReader)r); if (newReader == null) { return(null); } else { return(GetSearcher(SearcherFactory, newReader)); } }
public bool? Call() { if (indexReader == null) { indexReader = DirectoryReader.Open(indexDir); lastIndexGeneration = indexReader.IndexCommit.Generation; taxoReader = new DirectoryTaxonomyReader(taxoDir); } else { // verify search index DirectoryReader newReader = DirectoryReader.OpenIfChanged(indexReader); assertNotNull("should not have reached here if no changes were made to the index", newReader); long newGeneration = newReader.IndexCommit.Generation; assertTrue("expected newer generation; current=" + lastIndexGeneration + " new=" + newGeneration, newGeneration > lastIndexGeneration); indexReader.Dispose(); indexReader = newReader; lastIndexGeneration = newGeneration; TestUtil.CheckIndex(indexDir); // verify taxonomy index DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.OpenIfChanged(taxoReader); if (newTaxoReader != null) { taxoReader.Dispose(); taxoReader = newTaxoReader; } TestUtil.CheckIndex(taxoDir); // verify faceted search int id = int.Parse(indexReader.IndexCommit.UserData[VERSION_ID], NumberStyles.HexNumber); IndexSearcher searcher = new IndexSearcher(indexReader); FacetsCollector fc = new FacetsCollector(); searcher.Search(new MatchAllDocsQuery(), fc); Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc); assertEquals(1, (int)facets.GetSpecificValue("A", id.ToString("X"))); DrillDownQuery drillDown = new DrillDownQuery(config); drillDown.Add("A", id.ToString("X")); TopDocs docs = searcher.Search(drillDown, 10); assertEquals(1, docs.TotalHits); } return null; }
/// <summary> /// Opens the index reader. /// </summary> private void OpenReader() { if (_reader == null) { _reader = DirectoryReader.Open(Directory); _indexSearcher = new IndexSearcher(_reader); } else { DirectoryReader newReader = DirectoryReader.OpenIfChanged(_reader); if (newReader != null) { _reader.Dispose(); _reader = newReader; _indexSearcher = new IndexSearcher(_reader); } } }
public void Call() { if (reader is null) { reader = DirectoryReader.Open(indexDir); lastGeneration = reader.IndexCommit.Generation; } else { DirectoryReader newReader = DirectoryReader.OpenIfChanged(reader); assertNotNull("should not have reached here if no changes were made to the index", newReader); long newGeneration = newReader.IndexCommit.Generation; assertTrue("expected newer generation; current=" + lastGeneration + " new=" + newGeneration, newGeneration > lastGeneration); reader.Dispose(); reader = newReader; lastGeneration = newGeneration; TestUtil.CheckIndex(indexDir); } }
private async Task <IndexReader> GetReaderAsync() { try { await _semaphoreSlim.WaitAsync(); var newReader = DirectoryReader.OpenIfChanged(_indexReader); if (newReader != null) { _indexReader.Dispose(); _indexReader = newReader; } return(_indexReader); } finally { _semaphoreSlim.Release(); } }
/// <exception cref="System.IO.IOException"/> internal virtual void SetIndexReaderSearcher() { FSDirectory index = FSDirectory.Open(indexDir); if (reader == null) { reader = DirectoryReader.Open(index); searcher = new IndexSearcher(reader); } else { DirectoryReader newreader = DirectoryReader.OpenIfChanged(reader); if (newreader != null) { reader.Close(); reader = newreader; searcher = new IndexSearcher(reader); } } }
public virtual void TestNRTAndCommit() { Directory dir = NewDirectory(); NRTCachingDirectory cachedDir = new NRTCachingDirectory(dir, 2.0, 25.0); MockAnalyzer analyzer = new MockAnalyzer(Random); analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH); IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); RandomIndexWriter w = new RandomIndexWriter(Random, cachedDir, conf); LineFileDocs docs = new LineFileDocs(Random, DefaultCodecSupportsDocValues); int numDocs = TestUtil.NextInt32(Random, 100, 400); if (Verbose) { Console.WriteLine("TEST: numDocs=" + numDocs); } IList <BytesRef> ids = new List <BytesRef>(); DirectoryReader r = null; for (int docCount = 0; docCount < numDocs; docCount++) { Document doc = docs.NextDoc(); ids.Add(new BytesRef(doc.Get("docid"))); w.AddDocument(doc); if (Random.Next(20) == 17) { if (r == null) { r = DirectoryReader.Open(w.IndexWriter, false); } else { DirectoryReader r2 = DirectoryReader.OpenIfChanged(r); if (r2 != null) { r.Dispose(); r = r2; } } Assert.AreEqual(1 + docCount, r.NumDocs); IndexSearcher s = NewSearcher(r); // Just make sure search can run; we can't assert // totHits since it could be 0 TopDocs hits = s.Search(new TermQuery(new Term("body", "the")), 10); // System.out.println("tot hits " + hits.totalHits); } } if (r != null) { r.Dispose(); } // Close should force cache to clear since all files are sync'd w.Dispose(); string[] cachedFiles = cachedDir.ListCachedFiles(); foreach (string file in cachedFiles) { Console.WriteLine("FAIL: cached file " + file + " remains after sync"); } Assert.AreEqual(0, cachedFiles.Length); r = DirectoryReader.Open(dir); foreach (BytesRef id in ids) { Assert.AreEqual(1, r.DocFreq(new Term("docid", id))); } r.Dispose(); cachedDir.Dispose(); docs.Dispose(); }
/// <summary> /// An <see cref="IndexReader"/> is an instance of the index at a given point in time /// We need to update this Reader by reopen the <see cref="IndexReader"/> /// Maybe change this method later ? /// </summary> /// <param name="appName"></param> private void UpdateReader(string appName) { _readers[appName] = DirectoryReader.OpenIfChanged(_readers[appName] as DirectoryReader) ?? _readers[appName]; }
//////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// Returns an instance of the index reader based on the writer supporting NRTS for the main /// index. Please note that this method will also open a taxonomy reader at the same time, if it /// is not yet open. /// </summary> /// /// <remarks> Semantika d.o.o.,. </remarks> /// /// <param name="indexType"> Type of the index. </param> /// /// <returns> /// Main index reader instance containing the latest NRTS changes and a coresponding taxonomy /// reader. /// </returns> //////////////////////////////////////////////////////////////////////////////////////////////////// public IndexReader GetIndexReader(IndexType indexType) { var tmpIndexType = ApplyIndexingMode(indexType); var currentManagementObjects = m_managementObjects[tmpIndexType]; //Make sure that only one thread at a time is manipulating reader instances //lock (m_indexIOLock) { //Check if we have an instance of the main index reader if (currentManagementObjects.Reader == null) { if (IsReadOnly) { currentManagementObjects.Reader = DirectoryReader.Open(currentManagementObjects.Directory); } else { //If we do not yet have an instance of the reader, we need to make sure we have a functional writer first if (currentManagementObjects.Writer == null || currentManagementObjects.Writer.IsClosed) { //If we did not yet have an open writer, we need to open one. EnsureOpenWriter(); } //Since we had no open reader, we can now open a new one which will include all of the NRTS changes currentManagementObjects.Reader = DirectoryReader.Open(currentManagementObjects.Writer, true); } } else { if (!IsReadOnly) { //We already have a previous instance of the reader - make sure that includes all of the changes done to the index in the meantime DirectoryReader newReader; try { newReader = DirectoryReader.OpenIfChanged((DirectoryReader)currentManagementObjects.Reader, currentManagementObjects.Writer, true); } catch (IOException) { if (currentManagementObjects.Writer == null || currentManagementObjects.Writer.IsClosed) { //If we did not yet have an open writer, we need to open one. EnsureOpenWriter(); } newReader = DirectoryReader.Open(currentManagementObjects.Writer, true); } if (newReader != null) { //There was a change in the index since the last time reader was accessed - create a new instance and close the old one if (currentManagementObjects.CurrentReaderUsers == 0) { //No one is using the reader at the moment - close it currentManagementObjects.Reader.Dispose(); } else { //We have searchers using the reader - add it to cleanup queue currentManagementObjects.OldReaders[currentManagementObjects.Reader] = currentManagementObjects.CurrentReaderUsers; } currentManagementObjects.CurrentReaderUsers = 0; currentManagementObjects.Reader = newReader; //Inform listeners that the index was refreshed IndexRefreshed?.Invoke(this, new EventArgs()); } } } currentManagementObjects.CurrentReaderUsers++; //Return the created / refreshed instance of the reader to the caller and exit critical section return(currentManagementObjects.Reader); } }
/// <summary> /// Implements the opening of a new <seealso cref="DirectoryTaxonomyReader"/> instance if /// the taxonomy has changed. /// /// <para> /// <b>NOTE:</b> the returned <seealso cref="DirectoryTaxonomyReader"/> shares the /// ordinal and category caches with this reader. This is not expected to cause /// any issues, unless the two instances continue to live. The reader /// guarantees that the two instances cannot affect each other in terms of /// correctness of the caches, however if the size of the cache is changed /// through <seealso cref="#setCacheSize(int)"/>, it will affect both reader instances. /// </para> /// </summary> protected override TaxonomyReader DoOpenIfChanged() { EnsureOpen(); // This works for both NRT and non-NRT readers (i.e. an NRT reader remains NRT). var r2 = DirectoryReader.OpenIfChanged(indexReader); if (r2 == null) { return(null); // no changes, nothing to do } // check if the taxonomy was recreated bool success = false; try { bool recreated = false; if (taxoWriter == null) { // not NRT, check epoch from commit data string t1 = indexReader.IndexCommit.UserData[DirectoryTaxonomyWriter.INDEX_EPOCH]; string t2 = r2.IndexCommit.UserData[DirectoryTaxonomyWriter.INDEX_EPOCH]; if (t1 == null) { if (t2 != null) { recreated = true; } } else if (!t1.Equals(t2)) { // t1 != null and t2 cannot be null b/c DirTaxoWriter always puts the commit data. // it's ok to use String.equals because we require the two epoch values to be the same. recreated = true; } } else { // NRT, compare current taxoWriter.epoch() vs the one that was given at construction if (taxoEpoch != taxoWriter.TaxonomyEpoch) { recreated = true; } } DirectoryTaxonomyReader newtr; if (recreated) { // if recreated, do not reuse anything from this instace. the information // will be lazily computed by the new instance when needed. newtr = new DirectoryTaxonomyReader(r2, taxoWriter, null, null, null); } else { newtr = new DirectoryTaxonomyReader(r2, taxoWriter, ordinalCache, categoryCache, taxoArrays); } success = true; return(newtr); } finally { if (!success) { IOUtils.CloseWhileHandlingException(r2); } } }