// See PersonIndexer.cs for details on how fields were originally indexed. public IList <LuceneSearchResult> Search(string term, int numResults, bool includeRestrictedProfiles) { using (SearcherManager manager = new SearcherManager(PersonIndexWriterSingleton.Instance)) { this.searcher = manager.Acquire().Searcher; try { Query query = CreateQueryFromTerm(term); if (query == null) { return(new List <LuceneSearchResult>()); } // filter on Person.IsRestrictedProfile since restricted profiles are not viewable in screening if (!includeRestrictedProfiles) { BooleanQuery booleanQuery = new BooleanQuery(); booleanQuery.Add(query, Occur.MUST); booleanQuery.Add(new TermQuery(new Term("IsRestrictedProfile", "0")), Occur.MUST); query = booleanQuery; } this.topDocs = this.searcher.Search(query, numResults); return(TransformTopDocs()); } catch (ParseException e) { log.Error("Encountered problem parsing the search term: " + term, e); return(new List <LuceneSearchResult>()); } } }
public virtual void TestDeleteDocuments() { Directory directory = CreateIndex(); IndexWriter writer = GetWriter(directory); ReferenceManager <IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory()); IndexSearcher searcher = mgr.Acquire(); TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100); Assert.AreEqual(1, topDocs.TotalHits); TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer); long result = mgrWriter.DeleteDocuments(new TermQuery(new Term("foo", "0"))); Assert.AreEqual(1, result); // writer.Commit(); Assert.IsTrue(writer.HasDeletions()); mgr.MaybeRefresh(); searcher = mgr.Acquire(); topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100); Assert.AreEqual(0, topDocs.TotalHits); }
protected BaseSearchService(string indexDir) { this.log = LogManager.GetLogger(GetType()); this.IndexDir = indexDir; this._writer = IndexWriterDic.GetOrAdd(indexDir, dic => { lock (IndexWriterDic) { if (IndexWriterDic.TryGetValue(indexDir, out var writer)) { return(writer); } InfoStream.Default = new Log4NetInfoStream(); var merge = new ConcurrentMergeScheduler(); merge.SetMaxMergesAndThreads(6, 2); return(new IndexWriter(IndexDirectory, new IndexWriterConfig(LuceneVersion.LUCENE_48, KeyWordAnalyzer) { OpenMode = OpenMode.CREATE_OR_APPEND, MergeScheduler = merge })); } }); this._searcherManager = new SearcherManager(_writer, true, null); }
public IList <LuceneSearchResult> AllSourcesWithCaseNumbers(string term, int numResults, bool includeRestrictedSources, string sortField, bool descending) { using (SearcherManager manager = new SearcherManager(SourceIndexWriterSingleton.Instance)) { this.searcher = manager.Acquire().Searcher; BooleanQuery booleanQuery = new BooleanQuery(); if (string.IsNullOrEmpty(term)) { booleanQuery.Add(new MatchAllDocsQuery(), Occur.MUST); } else { booleanQuery.Add(new WildcardQuery(new Term("JhroCaseNumber", "*" + term + "*")), Occur.MUST); } // value '0' set by SourceIndexer booleanQuery.Add(new TermQuery(new Term("JhroCaseNumber", "0")), Occur.MUST_NOT); if (!includeRestrictedSources) { booleanQuery.Add(new TermQuery(new Term("IsRestricted", "0")), Occur.MUST); } log.Debug("Search query: " + booleanQuery.ToString()); this.PerformSearch(booleanQuery, numResults, sortField, descending); return(this.TransformTopDocs()); } }
public IList <LuceneSearchResult> Search(string term, string screeningEntityName, int numResults) { using (SearcherManager manager = new SearcherManager(ScreeningResponseIndexWriterSingleton.Instance)) { this.searcher = manager.Acquire().Searcher; QueryParser parser = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new string[] { "Reason", "Commentary" }, new SimpleAnalyzer()); parser.DefaultOperator = QueryParser.Operator.AND; parser.AllowLeadingWildcard = true; try { Query query = parser.Parse(term); BooleanQuery bq = new BooleanQuery(); bq.Add(query, Occur.MUST); bq.Add(new TermQuery(new Term("ScreeningEntityName", screeningEntityName)), Occur.MUST); log.Debug("Search query: " + bq.ToString()); this.topDocs = this.searcher.Search(bq, numResults); return(TransformTopDocs()); } catch (ParseException e) { log.Error("Encountered problem parsing the search term: " + term, e); return(new List <LuceneSearchResult>()); } } }
public int GetMaxSourceID() { using (SearcherManager manager = new SearcherManager(SourceIndexWriterSingleton.Instance)) { this.searcher = manager.Acquire().Searcher; Query query = NumericRangeQuery.NewIntRange("Id", 1, int.MaxValue, true, true); this.topDocs = this.searcher.Search(query, null, 1, new Sort(new SortField("Id", SortField.INT, true))); if (this.topDocs != null && this.topDocs.ScoreDocs != null && this.topDocs.ScoreDocs.Length > 0) { ScoreDoc scoreDoc = this.topDocs.ScoreDocs[0]; LuceneSearchResult result = new LuceneSearchResult(this.searcher.Doc(scoreDoc.Doc), scoreDoc.Score, this.topDocs.TotalHits); int id = 0; if (result.FieldValues != null && result.FieldValues["Id"] != null && result.FieldValues["Id"].Count > 0 && int.TryParse(result.FieldValues["Id"][0], out id)) { return(id); } } } return(0); }
public MovieIndex(string indexPath) { analyzer = SetupAnalyzer(); queryParser = SetupQueryParser(analyzer); writer = new IndexWriter(FSDirectory.Open(indexPath), new IndexWriterConfig(MATCH_LUCENE_VERSION, analyzer)); searchManager = new SearcherManager(writer, true, null); }
public override void Build(IInputEnumerator enumerator) { if (m_searcherMgr != null) { m_searcherMgr.Dispose(); m_searcherMgr = null; } if (writer != null) { writer.Dispose(); writer = null; } AtomicReader r = null; bool success = false; try { // First pass: build a temporary normal Lucene index, // just indexing the suggestions as they iterate: writer = new IndexWriter(dir, GetIndexWriterConfig(matchVersion, GetGramAnalyzer(), OpenMode.CREATE)); //long t0 = System.nanoTime(); // TODO: use threads? BytesRef text; while (enumerator.MoveNext()) { text = enumerator.Current; BytesRef payload; if (enumerator.HasPayloads) { payload = enumerator.Payload; } else { payload = null; } Add(text, enumerator.Contexts, enumerator.Weight, payload); } //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec"); m_searcherMgr = new SearcherManager(writer, true, null); success = true; } finally { if (success) { IOUtils.Dispose(r); } else { IOUtils.DisposeWhileHandlingException(writer, r); writer = null; } } }
protected override SearcherAndTaxonomy RefreshIfNeeded(SearcherAndTaxonomy @ref) { // Must re-open searcher first, otherwise we may get a // new reader that references ords not yet known to the // taxonomy reader: IndexReader r = @ref.searcher.IndexReader; IndexReader newReader = DirectoryReader.OpenIfChanged((DirectoryReader)r); if (newReader == null) { return(null); } else { var tr = TaxonomyReader.OpenIfChanged(@ref.taxonomyReader); if (tr == null) { @ref.taxonomyReader.IncRef(); tr = @ref.taxonomyReader; } else if (taxoWriter != null && taxoWriter.TaxonomyEpoch != taxoEpoch) { IOUtils.Close(newReader, tr); throw new ThreadStateException("DirectoryTaxonomyWriter.replaceTaxonomy was called, which is not allowed when using SearcherTaxonomyManager"); } return(new SearcherAndTaxonomy(SearcherManager.GetSearcher(searcherFactory, newReader), tr)); } }
//LUCENENET specific -Support for LUCENE - 5889. private void EnsureOpen() { if (writer != null) { return; } UninterruptableMonitor.Enter(syncLock); try { if (writer == null) { if (m_searcherMgr != null) { m_searcherMgr.Dispose(); m_searcherMgr = null; } writer = new IndexWriter(dir, GetIndexWriterConfig(matchVersion, GetGramAnalyzer(), OpenMode.CREATE)); m_searcherMgr = new SearcherManager(writer, true, null); } } finally { UninterruptableMonitor.Exit(syncLock); } }
protected BaseSearchService(string indexDir) { this.log = LogManager.GetLogger(GetType()); this.IndexDir = indexDir; this._writer = GetIndexWriter(); this._searcherManager = new SearcherManager(_writer, true, null); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @BeforeEach void initLuceneResources() throws Exception //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: internal virtual void InitLuceneResources() { _dirFactory = new Org.Neo4j.Kernel.Api.Impl.Index.storage.DirectoryFactory_InMemoryDirectoryFactory(); Directory dir = _dirFactory.open(_testDir.directory("test")); _writer = new IndexWriter(dir, IndexWriterConfigs.standard()); _searcherManager = new SearcherManager(_writer, true, new SearcherFactory()); }
public static IReadOnlyList <EntryWithScore> SearchFull(SearcherManager manager, QueryParser queryParser, string searchTerm) { IndexSearcher searcher = null; try { searcher = manager.Acquire(); var query = queryParser.Parse(searchTerm); var searchResults = searcher.Search(query, null, int.MaxValue, IdSort, true, true); var results = new EntryWithScore[searchResults.TotalHits]; for (var i = 0; i < searchResults.ScoreDocs.Length; i++) { var doc = searchResults.ScoreDocs[i]; var item = searcher.Doc(doc.Doc); var id = item.GetField(nameof(Entry.Id)).GetInt32Value(); if (!id.HasValue) { throw new InvalidOperationException($"Id did not have a value for document: {item}."); } var time = item.GetField(nameof(Entry.Time)).GetInt64Value(); if (!time.HasValue) { throw new InvalidOperationException($"Time did not have a value for document: {item}."); } var date = DateTimeOffset.FromUnixTimeSeconds(time.Value).UtcDateTime; var entry = new EntryWithScore { Id = id.Value, Date = date, Score = doc.Score, Time = time.Value, Url = item.Get(nameof(Entry.Url)), Title = item.Get(nameof(Entry.Title)) }; results[i] = entry; } return(results); } finally { manager.Release(searcher); } }
/// <summary> /// The get searcher. /// </summary> /// <returns> /// The <see cref="IndexSearcher"/>. /// </returns> private IndexSearcher GetSearcher() { this.searcherManager = this.indexWriter != null ? new SearcherManager(this.indexWriter, false, null) : new SearcherManager(FSDirectory.Open(SearchIndexFolder), null); this.searcherManager.MaybeRefreshBlocking(); return(this.searcherManager.Acquire()); }
public MenuSearchEngine() { analyzer = new StandardAnalyzer(LUCENE_VERSION, StandardAnalyzer.STOP_WORDS_SET); parser = new QueryParser(LUCENE_VERSION, DESCRIPTION, analyzer); var directory = System.IO.Directory.CreateDirectory(Settings.SearchIndexPath); writer = new IndexWriter(FSDirectory.Open(Settings.SearchIndexPath), new IndexWriterConfig(LUCENE_VERSION, analyzer)); manager = new SearcherManager(writer, true, null); }
internal LuceneBatchInserterIndex(File dbStoreDir, IndexIdentifier identifier, IDictionary <string, string> config, RelationshipLookup relationshipLookup) { File storeDir = GetStoreDir(dbStoreDir); this._createdNow = !LuceneDataSource.GetFileDirectory(storeDir, identifier).exists(); this._identifier = identifier; this._type = IndexType.GetIndexType(config); this._relationshipLookup = relationshipLookup; this._writer = InstantiateWriter(storeDir); this._searcherManager = InstantiateSearcherManager(_writer); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: public static java.util.List<long> getAllNodes(org.apache.lucene.store.Directory directory, org.neo4j.values.storable.Value propertyValue) throws java.io.IOException public static IList <long> GetAllNodes(Directory directory, Value propertyValue) { using (SearcherManager manager = new SearcherManager(directory, new SearcherFactory())) { IndexSearcher searcher = manager.acquire(); Query query = LuceneDocumentStructure.NewSeekQuery(propertyValue); AllNodesCollector collector = new AllNodesCollector(); searcher.search(query, collector); return(collector._nodeIds); } }
public IList <LuceneSearchResult> Search(string term, string prefix, bool usePrefixQuery, DateTime?start, DateTime?end, int numResults, bool canViewAndSearchAll, bool includeRestrictedSources, string uploadedByUserId, IList <string> owners, string sortField, bool descending) { using (SearcherManager manager = new SearcherManager(SourceIndexWriterSingleton.Instance)) { this.searcher = manager.Acquire().Searcher; Query query = this.BuildQuery(term, prefix, usePrefixQuery, start, end, canViewAndSearchAll, includeRestrictedSources, uploadedByUserId, owners); this.PerformSearch(query, numResults, sortField, descending); return(TransformTopDocs(query)); } }
public void Find_Single_Item() { var anders = "Anders"; _indexWriter.AddDocument(CreateDocument(anders)); _indexWriter.Commit(); var manager = new SearcherManager(_dir); var result = Search(manager, anders); CollectionAssert.AreEquivalent(result, new[] { anders }); }
/// <summary> /// Creates search and taxonomy readers over the corresponding directories. /// /// <para> /// <b>NOTE:</b> you should only use this constructor if you commit and call /// <seealso cref="#maybeRefresh()"/> in the same thread. Otherwise it could lead to an /// unsync'd <seealso cref="IndexSearcher"/> and <seealso cref="TaxonomyReader"/> pair. /// </para> /// </summary> public SearcherTaxonomyManager(Store.Directory indexDir, Store.Directory taxoDir, SearcherFactory searcherFactory) { if (searcherFactory == null) { searcherFactory = new SearcherFactory(); } this.searcherFactory = searcherFactory; var taxoReader = new DirectoryTaxonomyReader(taxoDir); Current = new SearcherAndTaxonomy(SearcherManager.GetSearcher(searcherFactory, DirectoryReader.Open(indexDir)), taxoReader); this.taxoWriter = null; taxoEpoch = -1; }
public void Check_IndexReader_Gets_Closed() { var dir = new RAMDirectory(); var writer = new IndexWriter(dir, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); var manager = new SearcherManager(dir); var searcher1 = manager.GetSearcher(); manager.ReleaseSearcher(searcher1); manager.ReleaseSearcher(searcher1); // The SearcherManager ctor creates an IndexReader. Assert.Throws <AlreadyClosedException>(() => manager.ReleaseSearcher(searcher1)); }
/// <summary> /// Creates near-real-time searcher and taxonomy reader /// from the corresponding writers. /// </summary> public SearcherTaxonomyManager(IndexWriter writer, bool applyAllDeletes, SearcherFactory searcherFactory, DirectoryTaxonomyWriter taxoWriter) { if (searcherFactory == null) { searcherFactory = new SearcherFactory(); } this.searcherFactory = searcherFactory; this.taxoWriter = taxoWriter; var taxoReader = new DirectoryTaxonomyReader(taxoWriter); Current = new SearcherAndTaxonomy(SearcherManager.GetSearcher(searcherFactory, DirectoryReader.Open(writer, applyAllDeletes)), taxoReader); this.taxoEpoch = taxoWriter.TaxonomyEpoch; }
public SearcherContextInternal(Directory dir, Analyzer defaultAnalyzer, TimeSpan targetMinStale, TimeSpan targetMaxStale) { Analyzer = new PerFieldAnalyzerWrapper(defaultAnalyzer); _writer = new IndexWriter(dir, new IndexWriterConfig(LeoLuceneVersion.Version, Analyzer) { OpenMode = OpenMode.CREATE_OR_APPEND }); _trackingWriter = new TrackingIndexWriter(_writer); _searcherManager = new SearcherManager(_writer, true, null); _nrtReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(_trackingWriter, _searcherManager, targetMaxStale.TotalSeconds, targetMinStale.TotalSeconds); _nrtReopenThread.SetDaemon(true); _nrtReopenThread.Start(); }
public static void AutoCompleteSample() { Directory dir = new RAMDirectory(); // Create n-edge grams for field suggestionText Analyzer analyzer = new EdgeNGramAnalyzer(Version); var indexWriter = new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.UNLIMITED); var docs = CreateAutoCompleteDocuments(GetDeals()); foreach (var doc in docs) { indexWriter.AddDocument(doc); } indexWriter.Commit(); indexWriter.Dispose(); var searcherManager = new SearcherManager(dir); // Default sort by dicount pct desc. var sort = new Sort(new SortField(DiscountPctFieldName, SortField.INT, true)); const int maxSuggestions = 5; for (;;) { Console.Write("Enter a text for auto completion and press enter: "); var input = Console.ReadLine(); Query query = new TermQuery(new Term(SuggestionTextFieldName, input)); var searcher = searcherManager.GetSearcher(); TopDocs hits; try { hits = searcher.Search(query, null, maxSuggestions, sort); } finally { searcherManager.ReleaseSearcher(searcher); } foreach (ScoreDoc match in hits.ScoreDocs) { Document doc = searcher.Doc(match.Doc); Console.WriteLine("Matched: '{0}' in '{1}' by '{2}'", doc.Get(SuggestionTextFieldName), doc.Get(LocationNameFieldName), doc.Get(SupplierFieldName)); } } }
// See UnitIndexer.cs for details on how fields were originally indexed. public IList <LuceneSearchResult> Search(string term, int numResults) { using (SearcherManager manager = new SearcherManager(UnitIndexWriterSingleton.Instance)) { this.searcher = manager.Acquire().Searcher; QueryParser parser; if (!string.IsNullOrEmpty(term) && term.Trim().StartsWith("Id")) { // Single and ranged numeric value search on Id field parser = new NumericQueryParser(Lucene.Net.Util.Version.LUCENE_30, "Id", new PersonAnalyzer()); } else { // General search across text fields parser = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new string[] { "Name", "ParentNameChange", "ChildNameChange", "BackgroundInformation", "Organization" }, new PersonAnalyzer()); // We maintain OR as default for maximum results parser.DefaultOperator = QueryParser.Operator.OR; if (!string.IsNullOrEmpty(term)) { if (!term.Contains(':')) { // Edit user's search string and add wildcards. term = string.Join(" ", term.Split(new string[] { " " }, System.StringSplitOptions.RemoveEmptyEntries) .Select(x => "*" + x + "*") .ToArray() ); } } } parser.AllowLeadingWildcard = true; try { Query query = parser.Parse(term); log.Debug("Search query: " + query.ToString()); this.topDocs = this.searcher.Search(query, numResults); return(TransformTopDocs()); } catch (ParseException e) { log.Error("Encountered problem parsing the search term: " + term, e); return(new List <LuceneSearchResult>()); } } }
public void Dispose() { if (searcherMgr != null) { searcherMgr.Dispose(); searcherMgr = null; } if (writer != null) { writer.Dispose(); dir.Dispose(); writer = null; } }
public virtual void Dispose() { if (m_searcherMgr != null) { m_searcherMgr.Dispose(); m_searcherMgr = null; } if (writer != null) { writer.Dispose(); dir.Dispose(); writer = null; } }
public virtual void TestTryDeleteDocument() { Directory directory = CreateIndex(); IndexWriter writer = GetWriter(directory); ReferenceManager <IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory()); TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer); IndexSearcher searcher = mgr.Acquire(); TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100); Assert.AreEqual(1, topDocs.TotalHits); long result; if (Random.NextBoolean()) { IndexReader r = DirectoryReader.Open(writer, true); result = mgrWriter.TryDeleteDocument(r, 0); r.Dispose(); } else { result = mgrWriter.TryDeleteDocument(searcher.IndexReader, 0); } // The tryDeleteDocument should have succeeded: Assert.IsTrue(result != -1); Assert.IsTrue(writer.HasDeletions()); if (Random.NextBoolean()) { writer.Commit(); } Assert.IsTrue(writer.HasDeletions()); mgr.MaybeRefresh(); searcher = mgr.Acquire(); topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100); Assert.AreEqual(0, topDocs.TotalHits); }
public static IReadOnlyList <LocatedEntry> Search(SearcherManager manager, QueryParser queryParser, string searchTerm) { IndexSearcher searcher = null; try { manager.MaybeRefresh(); searcher = manager.Acquire(); var query = queryParser.Parse(searchTerm); var searchResults = searcher.Search(query, int.MaxValue, IdSort); var results = new LocatedEntry[searchResults.TotalHits]; for (var i = 0; i < searchResults.ScoreDocs.Length; i++) { var doc = searchResults.ScoreDocs[i]; var item = searcher.Doc(doc.Doc); var id = item.GetField(nameof(Entry.Id)).GetInt32Value(); if (!id.HasValue) { throw new InvalidOperationException($"Id did not have a value for document: {item}."); } var time = item.GetField(nameof(Entry.Time)).GetInt64Value(); if (!time.HasValue) { throw new InvalidOperationException($"Time did not have a value for document: {item}."); } var date = DateTimeOffset.FromUnixTimeSeconds(time.Value).UtcDateTime; results[i] = new LocatedEntry(id.Value, date); } return(results); } finally { manager.Release(searcher); } }
public SearchResults Search(string queryString) { int resultsPerPage = 10; var analyzer = SetupAnalyzer(); var queryParser = SetupQueryParser(analyzer); IEnumerable <FieldDefinition> fields = new List <FieldDefinition> { new FieldDefinition { Name = "title", isDefault = true }, new FieldDefinition { Name = "description", isDefault = false } }; // Query query = BuildQuery(queryString,queryParser); // BuildQuery(queryString, fields); // Query query; if (queryString.EndsWith('~')) { query = BuildQuery(queryString, queryParser); } else { query = BuildQuery(queryString, fields); } using (var writer = new IndexWriter(_directory, new IndexWriterConfig(MATCH_LUCENE_VERSION, analyzer))) { var searchManager = new SearcherManager(writer, true, null); searchManager.MaybeRefreshBlocking(); IndexSearcher searcher = searchManager.Acquire(); try { TopDocs topDocs = searcher.Search(query, resultsPerPage); return(CompileResults(searcher, topDocs)); } finally { searchManager?.Release(searcher); searchManager?.Dispose(); searcher = null; analyzer?.Dispose(); ReleaseWriteLock(); } } }
protected internal override void DoAfterWriter(TaskScheduler es) { SearcherFactory factory = new SearcherFactoryAnonymousInnerClassHelper(this, es); if (Random().NextBoolean()) { // TODO: can we randomize the applyAllDeletes? But // somehow for final searcher we must apply // deletes... Mgr = new SearcherManager(Writer, true, factory); IsNRT = true; } else { // SearcherManager needs to see empty commit: Writer.Commit(); Mgr = new SearcherManager(Dir, factory); IsNRT = false; AssertMergedSegmentsWarmed = false; } LifetimeMGR = new SearcherLifetimeManager(); }
public virtual void Dispose() { if (searcherMgr != null) { searcherMgr.Dispose(); searcherMgr = null; } if (writer != null) { writer.Dispose(); dir.Dispose(); writer = null; } }
protected internal override void DoAfterWriter(TaskScheduler es) { double minReopenSec = 0.01 + 0.05 * Random().NextDouble(); double maxReopenSec = minReopenSec * (1.0 + 10 * Random().NextDouble()); if (VERBOSE) { Console.WriteLine("TEST: make SearcherManager maxReopenSec=" + maxReopenSec + " minReopenSec=" + minReopenSec); } GenWriter = new TrackingIndexWriter(Writer); SearcherFactory sf = new SearcherFactoryAnonymousInnerClassHelper(this, es); NrtNoDeletes = new SearcherManager(Writer, false, sf); NrtDeletes = new SearcherManager(Writer, true, sf); NrtDeletesThread = new ControlledRealTimeReopenThread<>(GenWriter, NrtDeletes, maxReopenSec, minReopenSec); NrtDeletesThread.Name = "NRTDeletes Reopen Thread"; NrtDeletesThread.Priority = Math.Min(Thread.CurrentThread.Priority + 2, Thread.MAX_PRIORITY); NrtDeletesThread.SetDaemon(true); NrtDeletesThread.Start(); NrtNoDeletesThread = new ControlledRealTimeReopenThread<>(GenWriter, NrtNoDeletes, maxReopenSec, minReopenSec); NrtNoDeletesThread.Name = "NRTNoDeletes Reopen Thread"; NrtNoDeletesThread.Priority = Math.Min(Thread.CurrentThread.Priority + 2, Thread.MAX_PRIORITY); NrtNoDeletesThread.SetDaemon(true); NrtNoDeletesThread.Start(); }
public ThreadAnonymousInnerClassHelper(TestControlledRealTimeReopenThread outerInstance, CountDownLatch latch, CountDownLatch signal, TrackingIndexWriter writer, SearcherManager manager) { this.OuterInstance = outerInstance; this.Latch = latch; this.Signal = signal; this.Writer = writer; this.Manager = manager; }
public virtual void TestListenerCalled() { Directory dir = NewDirectory(); IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)); AtomicBoolean afterRefreshCalled = new AtomicBoolean(false); SearcherManager sm = new SearcherManager(iw, false, new SearcherFactory()); sm.AddListener(new RefreshListenerAnonymousInnerClassHelper(this, afterRefreshCalled)); iw.AddDocument(new Document()); iw.Commit(); Assert.IsFalse(afterRefreshCalled.Get()); sm.MaybeRefreshBlocking(); Assert.IsTrue(afterRefreshCalled.Get()); sm.Dispose(); iw.Dispose(); dir.Dispose(); }
public SearchManager(Directory dir) { searcherManager = new SearcherManager(dir, new SearcherFactory()); Task.Run(Refresher); }
public NodeState(ShardSearchingTestBase outerInstance, Random random, int nodeID, int numNodes) { this.OuterInstance = outerInstance; MyNodeID = nodeID; Dir = NewFSDirectory(CreateTempDir("ShardSearchingTestBase")); // TODO: set warmer MockAnalyzer analyzer = new MockAnalyzer(Random()); analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH); IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer); iwc.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE); if (VERBOSE) { iwc.InfoStream = new PrintStreamInfoStream(Console.Out); } Writer = new IndexWriter(Dir, iwc); Mgr = new SearcherManager(Writer, true, null); Searchers = new SearcherLifetimeManager(); // Init w/ 0s... caller above will do initial // "broadcast" by calling initSearcher: CurrentNodeVersions = new long[numNodes]; }
public ThreadAnonymousInnerClassHelper2(TestSearcherManager outerInstance, SearcherManager sm) { this.OuterInstance = outerInstance; this.Sm = sm; }
public virtual void TestReferenceDecrementIllegally() { Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(new ConcurrentMergeScheduler())); SearcherManager sm = new SearcherManager(writer, false, new SearcherFactory()); writer.AddDocument(new Document()); writer.Commit(); sm.MaybeRefreshBlocking(); IndexSearcher acquire = sm.Acquire(); IndexSearcher acquire2 = sm.Acquire(); sm.Release(acquire); sm.Release(acquire2); acquire = sm.Acquire(); acquire.IndexReader.DecRef(); sm.Release(acquire); try { sm.Acquire(); Assert.Fail("acquire should have thrown an InvalidOperationException since we modified the refCount outside of the manager"); } catch (InvalidOperationException ex) { // } // sm.Dispose(); -- already closed writer.Dispose(); dir.Dispose(); }
public RunnableAnonymousInnerClassHelper(TestSearcherManager outerInstance, AtomicBoolean triedReopen, SearcherManager searcherManager, AtomicBoolean success, Exception[] exc) { this.OuterInstance = outerInstance; this.TriedReopen = triedReopen; this.SearcherManager = searcherManager; this.Success = success; this.Exc = exc; }
public virtual void TestReferenceDecrementIllegally([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler) { Directory dir = NewDirectory(); var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())) .SetMergeScheduler(scheduler); IndexWriter writer = new IndexWriter(dir, config); SearcherManager sm = new SearcherManager(writer, false, new SearcherFactory()); writer.AddDocument(new Document()); writer.Commit(); sm.MaybeRefreshBlocking(); IndexSearcher acquire = sm.Acquire(); IndexSearcher acquire2 = sm.Acquire(); sm.Release(acquire); sm.Release(acquire2); acquire = sm.Acquire(); acquire.IndexReader.DecRef(); sm.Release(acquire); Assert.Throws<InvalidOperationException>(() => sm.Acquire(), "acquire should have thrown an InvalidOperationException since we modified the refCount outside of the manager"); // sm.Dispose(); -- already closed writer.Dispose(); dir.Dispose(); }
public virtual void TestMaybeRefreshBlockingLock() { // make sure that maybeRefreshBlocking releases the lock, otherwise other // threads cannot obtain it. Directory dir = NewDirectory(); RandomIndexWriter w = new RandomIndexWriter(Random(), dir); w.Dispose(); SearcherManager sm = new SearcherManager(dir, null); ThreadClass t = new ThreadAnonymousInnerClassHelper2(this, sm); t.Start(); t.Join(); // if maybeRefreshBlocking didn't release the lock, this will fail. Assert.IsTrue(sm.MaybeRefresh(), "failde to obtain the refreshLock!"); sm.Dispose(); dir.Dispose(); }
// LUCENE-5461 public virtual void TestCRTReopen() { //test behaving badly //should be high enough int maxStaleSecs = 20; //build crap data just to store it. string s = " abcdefghijklmnopqrstuvwxyz "; char[] chars = s.ToCharArray(); StringBuilder builder = new StringBuilder(2048); for (int i = 0; i < 2048; i++) { builder.Append(chars[Random().Next(chars.Length)]); } string content = builder.ToString(); SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, new MockAnalyzer(Random())); config.SetIndexDeletionPolicy(sdp); config.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE_OR_APPEND); IndexWriter iw = new IndexWriter(dir, config); SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory()); TrackingIndexWriter tiw = new TrackingIndexWriter(iw); ControlledRealTimeReopenThread<IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread<IndexSearcher>(tiw, sm, maxStaleSecs, 0); controlledRealTimeReopenThread.SetDaemon(true); controlledRealTimeReopenThread.Start(); IList<Thread> commitThreads = new List<Thread>(); for (int i = 0; i < 500; i++) { if (i > 0 && i % 50 == 0) { Thread commitThread = new Thread(new RunnableAnonymousInnerClassHelper(this, sdp, dir, iw)); commitThread.Start(); commitThreads.Add(commitThread); } Document d = new Document(); d.Add(new TextField("count", i + "", Field.Store.NO)); d.Add(new TextField("content", content, Field.Store.YES)); long start = DateTime.Now.Millisecond; long l = tiw.AddDocument(d); controlledRealTimeReopenThread.WaitForGeneration(l); long wait = DateTime.Now.Millisecond - start; Assert.IsTrue(wait < (maxStaleSecs * 1000), "waited too long for generation " + wait); IndexSearcher searcher = sm.Acquire(); TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10); sm.Release(searcher); Assert.AreEqual(1, td.TotalHits); } foreach (Thread commitThread in commitThreads) { commitThread.Join(); } controlledRealTimeReopenThread.Dispose(); sm.Dispose(); iw.Dispose(); dir.Dispose(); }
public void Dispose() { stop = true; searcherManager.close(); searcherManager = null; }
public override void Build(IInputIterator iter) { if (searcherMgr != null) { searcherMgr.Dispose(); searcherMgr = null; } if (writer != null) { writer.Dispose(); writer = null; } AtomicReader r = null; bool success = false; try { // First pass: build a temporary normal Lucene index, // just indexing the suggestions as they iterate: writer = new IndexWriter(dir, GetIndexWriterConfig(matchVersion, GramAnalyzer, IndexWriterConfig.OpenMode_e.CREATE)); //long t0 = System.nanoTime(); // TODO: use threads? BytesRef text; while ((text = iter.Next()) != null) { BytesRef payload; if (iter.HasPayloads) { payload = iter.Payload; } else { payload = null; } Add(text, iter.Contexts, iter.Weight, payload); } //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec"); searcherMgr = new SearcherManager(writer, true, null); success = true; } finally { if (success) { IOUtils.Close(r); } else { IOUtils.CloseWhileHandlingException(writer, r); writer = null; } } }
internal SearchWrapper(SearcherManager searcherManager) { _searcherManager = searcherManager; _indexSearcher = (IndexSearcher) _searcherManager.acquire(); }
/// <summary> /// Create a new instance, loading from a previously built /// <see cref="AnalyzingInfixSuggester"/> directory, if it exists. This directory must be /// private to the infix suggester (i.e., not an external /// Lucene index). Note that <see cref="Dispose()"/> /// will also dispose the provided directory. /// </summary> /// <param name="minPrefixChars"> Minimum number of leading characters /// before <see cref="PrefixQuery"/> is used (default 4). /// Prefixes shorter than this are indexed as character /// ngrams (increasing index size but making lookups /// faster). </param> public AnalyzingInfixSuggester(LuceneVersion matchVersion, Directory dir, Analyzer indexAnalyzer, Analyzer queryAnalyzer, int minPrefixChars) { if (minPrefixChars < 0) { throw new System.ArgumentException("minPrefixChars must be >= 0; got: " + minPrefixChars); } this.queryAnalyzer = queryAnalyzer; this.indexAnalyzer = indexAnalyzer; this.matchVersion = matchVersion; this.dir = dir; this.minPrefixChars = minPrefixChars; if (DirectoryReader.IndexExists(dir)) { // Already built; open it: writer = new IndexWriter(dir, GetIndexWriterConfig(matchVersion, GramAnalyzer, IndexWriterConfig.OpenMode_e.APPEND)); searcherMgr = new SearcherManager(writer, true, null); } }
public virtual void TestCloseTwice() { // test that we can close SM twice (per IDisposable's contract). Directory dir = NewDirectory(); (new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose(); SearcherManager sm = new SearcherManager(dir, null); sm.Dispose(); sm.Dispose(); dir.Dispose(); }
/* * LUCENE-3528 - NRTManager hangs in certain situations */ public virtual void TestThreadStarvationNoDeleteNRTReader() { IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); conf.SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES); Directory d = NewDirectory(); CountDownLatch latch = new CountDownLatch(1); CountDownLatch signal = new CountDownLatch(1); LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal); TrackingIndexWriter writer = new TrackingIndexWriter(_writer); SearcherManager manager = new SearcherManager(_writer, false, null); Document doc = new Document(); doc.Add(NewTextField("test", "test", Field.Store.YES)); writer.AddDocument(doc); manager.MaybeRefresh(); ThreadClass t = new ThreadAnonymousInnerClassHelper(this, latch, signal, writer, manager); t.Start(); _writer.WaitAfterUpdate = true; // wait in addDocument to let some reopens go through long lastGen = writer.UpdateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen Assert.IsFalse(manager.SearcherCurrent); // false since there is a delete in the queue IndexSearcher searcher = manager.Acquire(); try { Assert.AreEqual(2, searcher.IndexReader.NumDocs()); } finally { manager.Release(searcher); } ControlledRealTimeReopenThread<IndexSearcher> thread = new ControlledRealTimeReopenThread<IndexSearcher>(writer, manager, 0.01, 0.01); thread.Start(); // start reopening if (VERBOSE) { Console.WriteLine("waiting now for generation " + lastGen); } AtomicBoolean finished = new AtomicBoolean(false); ThreadClass waiter = new ThreadAnonymousInnerClassHelper2(this, lastGen, thread, finished); waiter.Start(); manager.MaybeRefresh(); waiter.Join(1000); if (!finished.Get()) { waiter.Interrupt(); Assert.Fail("thread deadlocked on waitForGeneration"); } thread.Dispose(); thread.Join(); IOUtils.Close(manager, _writer, d); }
public virtual void TestEnsureOpen() { Directory dir = NewDirectory(); (new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose(); SearcherManager sm = new SearcherManager(dir, null); IndexSearcher s = sm.Acquire(); sm.Dispose(); // this should succeed; sm.Release(s); try { // this should fail sm.Acquire(); } catch (AlreadyClosedException e) { // ok } try { // this should fail sm.MaybeRefresh(); } catch (AlreadyClosedException e) { // ok } dir.Dispose(); }