Example #1
0
        // See PersonIndexer.cs for details on how fields were originally indexed.
        public IList <LuceneSearchResult> Search(string term, int numResults, bool includeRestrictedProfiles)
        {
            using (SearcherManager manager = new SearcherManager(PersonIndexWriterSingleton.Instance))
            {
                this.searcher = manager.Acquire().Searcher;

                try
                {
                    Query query = CreateQueryFromTerm(term);
                    if (query == null)
                    {
                        return(new List <LuceneSearchResult>());
                    }

                    // filter on Person.IsRestrictedProfile since restricted profiles are not viewable in screening
                    if (!includeRestrictedProfiles)
                    {
                        BooleanQuery booleanQuery = new BooleanQuery();
                        booleanQuery.Add(query, Occur.MUST);
                        booleanQuery.Add(new TermQuery(new Term("IsRestrictedProfile", "0")), Occur.MUST);
                        query = booleanQuery;
                    }

                    this.topDocs = this.searcher.Search(query, numResults);
                    return(TransformTopDocs());
                }
                catch (ParseException e)
                {
                    log.Error("Encountered problem parsing the search term: " + term, e);
                    return(new List <LuceneSearchResult>());
                }
            }
        }
Example #2
0
        public virtual void TestDeleteDocuments()
        {
            Directory directory = CreateIndex();

            IndexWriter writer = GetWriter(directory);

            ReferenceManager <IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory());

            IndexSearcher searcher = mgr.Acquire();

            TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);

            Assert.AreEqual(1, topDocs.TotalHits);

            TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer);
            long result = mgrWriter.DeleteDocuments(new TermQuery(new Term("foo", "0")));

            Assert.AreEqual(1, result);

            // writer.Commit();

            Assert.IsTrue(writer.HasDeletions());

            mgr.MaybeRefresh();

            searcher = mgr.Acquire();

            topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);

            Assert.AreEqual(0, topDocs.TotalHits);
        }
Example #3
0
 protected BaseSearchService(string indexDir)
 {
     this.log      = LogManager.GetLogger(GetType());
     this.IndexDir = indexDir;
     this._writer  = IndexWriterDic.GetOrAdd(indexDir, dic =>
     {
         lock (IndexWriterDic)
         {
             if (IndexWriterDic.TryGetValue(indexDir, out var writer))
             {
                 return(writer);
             }
             InfoStream.Default = new Log4NetInfoStream();
             var merge          = new ConcurrentMergeScheduler();
             merge.SetMaxMergesAndThreads(6, 2);
             return(new IndexWriter(IndexDirectory,
                                    new IndexWriterConfig(LuceneVersion.LUCENE_48, KeyWordAnalyzer)
             {
                 OpenMode = OpenMode.CREATE_OR_APPEND,
                 MergeScheduler = merge
             }));
         }
     });
     this._searcherManager = new SearcherManager(_writer, true, null);
 }
Example #4
0
        public IList <LuceneSearchResult> AllSourcesWithCaseNumbers(string term, int numResults, bool includeRestrictedSources, string sortField, bool descending)
        {
            using (SearcherManager manager = new SearcherManager(SourceIndexWriterSingleton.Instance))
            {
                this.searcher = manager.Acquire().Searcher;

                BooleanQuery booleanQuery = new BooleanQuery();
                if (string.IsNullOrEmpty(term))
                {
                    booleanQuery.Add(new MatchAllDocsQuery(), Occur.MUST);
                }
                else
                {
                    booleanQuery.Add(new WildcardQuery(new Term("JhroCaseNumber", "*" + term + "*")), Occur.MUST);
                }

                // value '0' set by SourceIndexer
                booleanQuery.Add(new TermQuery(new Term("JhroCaseNumber", "0")), Occur.MUST_NOT);

                if (!includeRestrictedSources)
                {
                    booleanQuery.Add(new TermQuery(new Term("IsRestricted", "0")), Occur.MUST);
                }

                log.Debug("Search query: " + booleanQuery.ToString());

                this.PerformSearch(booleanQuery, numResults, sortField, descending);
                return(this.TransformTopDocs());
            }
        }
Example #5
0
        public IList <LuceneSearchResult> Search(string term, string screeningEntityName, int numResults)
        {
            using (SearcherManager manager = new SearcherManager(ScreeningResponseIndexWriterSingleton.Instance))
            {
                this.searcher = manager.Acquire().Searcher;

                QueryParser parser = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30,
                                                               new string[] { "Reason", "Commentary" },
                                                               new SimpleAnalyzer());
                parser.DefaultOperator      = QueryParser.Operator.AND;
                parser.AllowLeadingWildcard = true;

                try
                {
                    Query query = parser.Parse(term);

                    BooleanQuery bq = new BooleanQuery();
                    bq.Add(query, Occur.MUST);
                    bq.Add(new TermQuery(new Term("ScreeningEntityName", screeningEntityName)), Occur.MUST);

                    log.Debug("Search query: " + bq.ToString());

                    this.topDocs = this.searcher.Search(bq, numResults);
                    return(TransformTopDocs());
                }
                catch (ParseException e)
                {
                    log.Error("Encountered problem parsing the search term: " + term, e);
                    return(new List <LuceneSearchResult>());
                }
            }
        }
Example #6
0
        public int GetMaxSourceID()
        {
            using (SearcherManager manager = new SearcherManager(SourceIndexWriterSingleton.Instance))
            {
                this.searcher = manager.Acquire().Searcher;

                Query query = NumericRangeQuery.NewIntRange("Id", 1, int.MaxValue, true, true);

                this.topDocs = this.searcher.Search(query, null, 1, new Sort(new SortField("Id", SortField.INT, true)));

                if (this.topDocs != null && this.topDocs.ScoreDocs != null && this.topDocs.ScoreDocs.Length > 0)
                {
                    ScoreDoc           scoreDoc = this.topDocs.ScoreDocs[0];
                    LuceneSearchResult result   = new LuceneSearchResult(this.searcher.Doc(scoreDoc.Doc), scoreDoc.Score, this.topDocs.TotalHits);

                    int id = 0;
                    if (result.FieldValues != null && result.FieldValues["Id"] != null && result.FieldValues["Id"].Count > 0 &&
                        int.TryParse(result.FieldValues["Id"][0], out id))
                    {
                        return(id);
                    }
                }
            }
            return(0);
        }
 public MovieIndex(string indexPath)
 {
     analyzer      = SetupAnalyzer();
     queryParser   = SetupQueryParser(analyzer);
     writer        = new IndexWriter(FSDirectory.Open(indexPath), new IndexWriterConfig(MATCH_LUCENE_VERSION, analyzer));
     searchManager = new SearcherManager(writer, true, null);
 }
Example #8
0
        public override void Build(IInputEnumerator enumerator)
        {
            if (m_searcherMgr != null)
            {
                m_searcherMgr.Dispose();
                m_searcherMgr = null;
            }

            if (writer != null)
            {
                writer.Dispose();
                writer = null;
            }

            AtomicReader r       = null;
            bool         success = false;

            try
            {
                // First pass: build a temporary normal Lucene index,
                // just indexing the suggestions as they iterate:
                writer = new IndexWriter(dir, GetIndexWriterConfig(matchVersion, GetGramAnalyzer(), OpenMode.CREATE));
                //long t0 = System.nanoTime();

                // TODO: use threads?
                BytesRef text;
                while (enumerator.MoveNext())
                {
                    text = enumerator.Current;
                    BytesRef payload;
                    if (enumerator.HasPayloads)
                    {
                        payload = enumerator.Payload;
                    }
                    else
                    {
                        payload = null;
                    }

                    Add(text, enumerator.Contexts, enumerator.Weight, payload);
                }

                //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec");

                m_searcherMgr = new SearcherManager(writer, true, null);
                success       = true;
            }
            finally
            {
                if (success)
                {
                    IOUtils.Dispose(r);
                }
                else
                {
                    IOUtils.DisposeWhileHandlingException(writer, r);
                    writer = null;
                }
            }
        }
Example #9
0
        protected override SearcherAndTaxonomy RefreshIfNeeded(SearcherAndTaxonomy @ref)
        {
            // Must re-open searcher first, otherwise we may get a
            // new reader that references ords not yet known to the
            // taxonomy reader:
            IndexReader r         = @ref.searcher.IndexReader;
            IndexReader newReader = DirectoryReader.OpenIfChanged((DirectoryReader)r);

            if (newReader == null)
            {
                return(null);
            }
            else
            {
                var tr = TaxonomyReader.OpenIfChanged(@ref.taxonomyReader);
                if (tr == null)
                {
                    @ref.taxonomyReader.IncRef();
                    tr = @ref.taxonomyReader;
                }
                else if (taxoWriter != null && taxoWriter.TaxonomyEpoch != taxoEpoch)
                {
                    IOUtils.Close(newReader, tr);
                    throw new ThreadStateException("DirectoryTaxonomyWriter.replaceTaxonomy was called, which is not allowed when using SearcherTaxonomyManager");
                }

                return(new SearcherAndTaxonomy(SearcherManager.GetSearcher(searcherFactory, newReader), tr));
            }
        }
        //LUCENENET specific -Support for LUCENE - 5889.
        private void EnsureOpen()
        {
            if (writer != null)
            {
                return;
            }

            UninterruptableMonitor.Enter(syncLock);
            try
            {
                if (writer == null)
                {
                    if (m_searcherMgr != null)
                    {
                        m_searcherMgr.Dispose();
                        m_searcherMgr = null;
                    }
                    writer        = new IndexWriter(dir, GetIndexWriterConfig(matchVersion, GetGramAnalyzer(), OpenMode.CREATE));
                    m_searcherMgr = new SearcherManager(writer, true, null);
                }
            }
            finally
            {
                UninterruptableMonitor.Exit(syncLock);
            }
        }
Example #11
0
 protected BaseSearchService(string indexDir)
 {
     this.log              = LogManager.GetLogger(GetType());
     this.IndexDir         = indexDir;
     this._writer          = GetIndexWriter();
     this._searcherManager = new SearcherManager(_writer, true, null);
 }
Example #12
0
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @BeforeEach void initLuceneResources() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
        internal virtual void InitLuceneResources()
        {
            _dirFactory = new Org.Neo4j.Kernel.Api.Impl.Index.storage.DirectoryFactory_InMemoryDirectoryFactory();
            Directory dir = _dirFactory.open(_testDir.directory("test"));

            _writer          = new IndexWriter(dir, IndexWriterConfigs.standard());
            _searcherManager = new SearcherManager(_writer, true, new SearcherFactory());
        }
Example #13
0
        public static IReadOnlyList <EntryWithScore> SearchFull(SearcherManager manager,
                                                                QueryParser queryParser,
                                                                string searchTerm)
        {
            IndexSearcher searcher = null;

            try
            {
                searcher = manager.Acquire();

                var query = queryParser.Parse(searchTerm);

                var searchResults = searcher.Search(query, null, int.MaxValue, IdSort, true, true);

                var results = new EntryWithScore[searchResults.TotalHits];

                for (var i = 0; i < searchResults.ScoreDocs.Length; i++)
                {
                    var doc  = searchResults.ScoreDocs[i];
                    var item = searcher.Doc(doc.Doc);
                    var id   = item.GetField(nameof(Entry.Id)).GetInt32Value();

                    if (!id.HasValue)
                    {
                        throw new InvalidOperationException($"Id did not have a value for document: {item}.");
                    }

                    var time = item.GetField(nameof(Entry.Time)).GetInt64Value();

                    if (!time.HasValue)
                    {
                        throw new InvalidOperationException($"Time did not have a value for document: {item}.");
                    }

                    var date = DateTimeOffset.FromUnixTimeSeconds(time.Value).UtcDateTime;

                    var entry = new EntryWithScore
                    {
                        Id    = id.Value,
                        Date  = date,
                        Score = doc.Score,
                        Time  = time.Value,
                        Url   = item.Get(nameof(Entry.Url)),
                        Title = item.Get(nameof(Entry.Title))
                    };

                    results[i] = entry;
                }

                return(results);
            }
            finally
            {
                manager.Release(searcher);
            }
        }
Example #14
0
        /// <summary>
        /// The get searcher.
        /// </summary>
        /// <returns>
        /// The <see cref="IndexSearcher"/>.
        /// </returns>
        private IndexSearcher GetSearcher()
        {
            this.searcherManager = this.indexWriter != null
                                       ? new SearcherManager(this.indexWriter, false, null)
                                       : new SearcherManager(FSDirectory.Open(SearchIndexFolder), null);

            this.searcherManager.MaybeRefreshBlocking();

            return(this.searcherManager.Acquire());
        }
Example #15
0
        public MenuSearchEngine()
        {
            analyzer = new StandardAnalyzer(LUCENE_VERSION, StandardAnalyzer.STOP_WORDS_SET);
            parser   = new QueryParser(LUCENE_VERSION, DESCRIPTION, analyzer);

            var directory = System.IO.Directory.CreateDirectory(Settings.SearchIndexPath);

            writer  = new IndexWriter(FSDirectory.Open(Settings.SearchIndexPath), new IndexWriterConfig(LUCENE_VERSION, analyzer));
            manager = new SearcherManager(writer, true, null);
        }
Example #16
0
        internal LuceneBatchInserterIndex(File dbStoreDir, IndexIdentifier identifier, IDictionary <string, string> config, RelationshipLookup relationshipLookup)
        {
            File storeDir = GetStoreDir(dbStoreDir);

            this._createdNow         = !LuceneDataSource.GetFileDirectory(storeDir, identifier).exists();
            this._identifier         = identifier;
            this._type               = IndexType.GetIndexType(config);
            this._relationshipLookup = relationshipLookup;
            this._writer             = InstantiateWriter(storeDir);
            this._searcherManager    = InstantiateSearcherManager(_writer);
        }
Example #17
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public static java.util.List<long> getAllNodes(org.apache.lucene.store.Directory directory, org.neo4j.values.storable.Value propertyValue) throws java.io.IOException
        public static IList <long> GetAllNodes(Directory directory, Value propertyValue)
        {
            using (SearcherManager manager = new SearcherManager(directory, new SearcherFactory()))
            {
                IndexSearcher     searcher  = manager.acquire();
                Query             query     = LuceneDocumentStructure.NewSeekQuery(propertyValue);
                AllNodesCollector collector = new AllNodesCollector();
                searcher.search(query, collector);
                return(collector._nodeIds);
            }
        }
Example #18
0
 public IList <LuceneSearchResult> Search(string term, string prefix, bool usePrefixQuery, DateTime?start, DateTime?end, int numResults,
                                          bool canViewAndSearchAll, bool includeRestrictedSources, string uploadedByUserId, IList <string> owners, string sortField, bool descending)
 {
     using (SearcherManager manager = new SearcherManager(SourceIndexWriterSingleton.Instance))
     {
         this.searcher = manager.Acquire().Searcher;
         Query query = this.BuildQuery(term, prefix, usePrefixQuery, start, end, canViewAndSearchAll, includeRestrictedSources, uploadedByUserId, owners);
         this.PerformSearch(query, numResults, sortField, descending);
         return(TransformTopDocs(query));
     }
 }
        public void Find_Single_Item()
        {
            var anders = "Anders";

            _indexWriter.AddDocument(CreateDocument(anders));
            _indexWriter.Commit();

            var manager = new SearcherManager(_dir);
            var result  = Search(manager, anders);

            CollectionAssert.AreEquivalent(result, new[] { anders });
        }
Example #20
0
        /// <summary>
        /// Creates search and taxonomy readers over the corresponding directories.
        ///
        /// <para>
        /// <b>NOTE:</b> you should only use this constructor if you commit and call
        /// <seealso cref="#maybeRefresh()"/> in the same thread. Otherwise it could lead to an
        /// unsync'd <seealso cref="IndexSearcher"/> and <seealso cref="TaxonomyReader"/> pair.
        /// </para>
        /// </summary>
        public SearcherTaxonomyManager(Store.Directory indexDir, Store.Directory taxoDir, SearcherFactory searcherFactory)
        {
            if (searcherFactory == null)
            {
                searcherFactory = new SearcherFactory();
            }
            this.searcherFactory = searcherFactory;
            var taxoReader = new DirectoryTaxonomyReader(taxoDir);

            Current         = new SearcherAndTaxonomy(SearcherManager.GetSearcher(searcherFactory, DirectoryReader.Open(indexDir)), taxoReader);
            this.taxoWriter = null;
            taxoEpoch       = -1;
        }
Example #21
0
        public void Check_IndexReader_Gets_Closed()
        {
            var dir     = new RAMDirectory();
            var writer  = new IndexWriter(dir, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
            var manager = new SearcherManager(dir);

            var searcher1 = manager.GetSearcher();

            manager.ReleaseSearcher(searcher1);
            manager.ReleaseSearcher(searcher1); // The SearcherManager ctor creates an IndexReader.

            Assert.Throws <AlreadyClosedException>(() => manager.ReleaseSearcher(searcher1));
        }
Example #22
0
        /// <summary>
        /// Creates near-real-time searcher and taxonomy reader
        ///  from the corresponding writers.
        /// </summary>
        public SearcherTaxonomyManager(IndexWriter writer, bool applyAllDeletes, SearcherFactory searcherFactory, DirectoryTaxonomyWriter taxoWriter)
        {
            if (searcherFactory == null)
            {
                searcherFactory = new SearcherFactory();
            }
            this.searcherFactory = searcherFactory;
            this.taxoWriter      = taxoWriter;
            var taxoReader = new DirectoryTaxonomyReader(taxoWriter);

            Current        = new SearcherAndTaxonomy(SearcherManager.GetSearcher(searcherFactory, DirectoryReader.Open(writer, applyAllDeletes)), taxoReader);
            this.taxoEpoch = taxoWriter.TaxonomyEpoch;
        }
Example #23
0
 public SearcherContextInternal(Directory dir, Analyzer defaultAnalyzer, TimeSpan targetMinStale, TimeSpan targetMaxStale)
 {
     Analyzer = new PerFieldAnalyzerWrapper(defaultAnalyzer);
     _writer  = new IndexWriter(dir, new IndexWriterConfig(LeoLuceneVersion.Version, Analyzer)
     {
         OpenMode = OpenMode.CREATE_OR_APPEND
     });
     _trackingWriter  = new TrackingIndexWriter(_writer);
     _searcherManager = new SearcherManager(_writer, true, null);
     _nrtReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(_trackingWriter, _searcherManager, targetMaxStale.TotalSeconds, targetMinStale.TotalSeconds);
     _nrtReopenThread.SetDaemon(true);
     _nrtReopenThread.Start();
 }
Example #24
0
        public static void AutoCompleteSample()
        {
            Directory dir = new RAMDirectory();

            // Create n-edge grams for field suggestionText
            Analyzer analyzer = new EdgeNGramAnalyzer(Version);

            var indexWriter = new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.UNLIMITED);

            var docs = CreateAutoCompleteDocuments(GetDeals());

            foreach (var doc in docs)
            {
                indexWriter.AddDocument(doc);
            }

            indexWriter.Commit();
            indexWriter.Dispose();

            var searcherManager = new SearcherManager(dir);

            // Default sort by dicount pct desc.
            var       sort           = new Sort(new SortField(DiscountPctFieldName, SortField.INT, true));
            const int maxSuggestions = 5;

            for (;;)
            {
                Console.Write("Enter a text for auto completion and press enter: ");
                var input = Console.ReadLine();

                Query query = new TermQuery(new Term(SuggestionTextFieldName, input));

                var     searcher = searcherManager.GetSearcher();
                TopDocs hits;
                try
                {
                    hits = searcher.Search(query, null, maxSuggestions, sort);
                }
                finally
                {
                    searcherManager.ReleaseSearcher(searcher);
                }

                foreach (ScoreDoc match in hits.ScoreDocs)
                {
                    Document doc = searcher.Doc(match.Doc);
                    Console.WriteLine("Matched: '{0}' in '{1}' by '{2}'",
                                      doc.Get(SuggestionTextFieldName), doc.Get(LocationNameFieldName), doc.Get(SupplierFieldName));
                }
            }
        }
Example #25
0
        // See UnitIndexer.cs for details on how fields were originally indexed.
        public IList <LuceneSearchResult> Search(string term, int numResults)
        {
            using (SearcherManager manager = new SearcherManager(UnitIndexWriterSingleton.Instance))
            {
                this.searcher = manager.Acquire().Searcher;

                QueryParser parser;
                if (!string.IsNullOrEmpty(term) && term.Trim().StartsWith("Id"))
                {
                    // Single and ranged numeric value search on Id field
                    parser = new NumericQueryParser(Lucene.Net.Util.Version.LUCENE_30, "Id", new PersonAnalyzer());
                }
                else
                {
                    // General search across text fields
                    parser = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30,
                                                       new string[] { "Name", "ParentNameChange", "ChildNameChange", "BackgroundInformation", "Organization" },
                                                       new PersonAnalyzer());

                    // We maintain OR as default for maximum results
                    parser.DefaultOperator = QueryParser.Operator.OR;

                    if (!string.IsNullOrEmpty(term))
                    {
                        if (!term.Contains(':'))
                        {
                            // Edit user's search string and add wildcards.
                            term = string.Join(" ", term.Split(new string[] { " " }, System.StringSplitOptions.RemoveEmptyEntries)
                                               .Select(x => "*" + x + "*")
                                               .ToArray()
                                               );
                        }
                    }
                }
                parser.AllowLeadingWildcard = true;

                try
                {
                    Query query = parser.Parse(term);
                    log.Debug("Search query: " + query.ToString());

                    this.topDocs = this.searcher.Search(query, numResults);
                    return(TransformTopDocs());
                }
                catch (ParseException e)
                {
                    log.Error("Encountered problem parsing the search term: " + term, e);
                    return(new List <LuceneSearchResult>());
                }
            }
        }
Example #26
0
 public void Dispose()
 {
     if (searcherMgr != null)
     {
         searcherMgr.Dispose();
         searcherMgr = null;
     }
     if (writer != null)
     {
         writer.Dispose();
         dir.Dispose();
         writer = null;
     }
 }
 public virtual void Dispose()
 {
     if (m_searcherMgr != null)
     {
         m_searcherMgr.Dispose();
         m_searcherMgr = null;
     }
     if (writer != null)
     {
         writer.Dispose();
         dir.Dispose();
         writer = null;
     }
 }
Example #28
0
        public virtual void TestTryDeleteDocument()
        {
            Directory directory = CreateIndex();

            IndexWriter writer = GetWriter(directory);

            ReferenceManager <IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory());

            TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer);

            IndexSearcher searcher = mgr.Acquire();

            TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);

            Assert.AreEqual(1, topDocs.TotalHits);

            long result;

            if (Random.NextBoolean())
            {
                IndexReader r = DirectoryReader.Open(writer, true);
                result = mgrWriter.TryDeleteDocument(r, 0);
                r.Dispose();
            }
            else
            {
                result = mgrWriter.TryDeleteDocument(searcher.IndexReader, 0);
            }

            // The tryDeleteDocument should have succeeded:
            Assert.IsTrue(result != -1);

            Assert.IsTrue(writer.HasDeletions());

            if (Random.NextBoolean())
            {
                writer.Commit();
            }

            Assert.IsTrue(writer.HasDeletions());

            mgr.MaybeRefresh();

            searcher = mgr.Acquire();

            topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);

            Assert.AreEqual(0, topDocs.TotalHits);
        }
Example #29
0
        public static IReadOnlyList <LocatedEntry> Search(SearcherManager manager,
                                                          QueryParser queryParser,
                                                          string searchTerm)
        {
            IndexSearcher searcher = null;

            try
            {
                manager.MaybeRefresh();

                searcher = manager.Acquire();

                var query = queryParser.Parse(searchTerm);

                var searchResults = searcher.Search(query, int.MaxValue, IdSort);

                var results = new LocatedEntry[searchResults.TotalHits];

                for (var i = 0; i < searchResults.ScoreDocs.Length; i++)
                {
                    var doc  = searchResults.ScoreDocs[i];
                    var item = searcher.Doc(doc.Doc);
                    var id   = item.GetField(nameof(Entry.Id)).GetInt32Value();

                    if (!id.HasValue)
                    {
                        throw new InvalidOperationException($"Id did not have a value for document: {item}.");
                    }

                    var time = item.GetField(nameof(Entry.Time)).GetInt64Value();

                    if (!time.HasValue)
                    {
                        throw new InvalidOperationException($"Time did not have a value for document: {item}.");
                    }

                    var date = DateTimeOffset.FromUnixTimeSeconds(time.Value).UtcDateTime;

                    results[i] = new LocatedEntry(id.Value, date);
                }

                return(results);
            }
            finally
            {
                manager.Release(searcher);
            }
        }
Example #30
0
        public SearchResults Search(string queryString)
        {
            int resultsPerPage = 10;
            var analyzer       = SetupAnalyzer();
            var queryParser    = SetupQueryParser(analyzer);
            IEnumerable <FieldDefinition> fields = new List <FieldDefinition> {
                new FieldDefinition {
                    Name = "title", isDefault = true
                },
                new FieldDefinition {
                    Name = "description", isDefault = false
                }
            };
            // Query query = BuildQuery(queryString,queryParser); // BuildQuery(queryString, fields); //
            Query query;

            if (queryString.EndsWith('~'))
            {
                query = BuildQuery(queryString, queryParser);
            }
            else
            {
                query = BuildQuery(queryString, fields);
            }

            using (var writer = new IndexWriter(_directory,
                                                new IndexWriterConfig(MATCH_LUCENE_VERSION, analyzer)))
            {
                var searchManager = new SearcherManager(writer, true, null);
                searchManager.MaybeRefreshBlocking();
                IndexSearcher searcher = searchManager.Acquire();

                try
                {
                    TopDocs topDocs = searcher.Search(query, resultsPerPage);
                    return(CompileResults(searcher, topDocs));
                }
                finally
                {
                    searchManager?.Release(searcher);
                    searchManager?.Dispose();
                    searcher = null;
                    analyzer?.Dispose();
                    ReleaseWriteLock();
                }
            }
        }
        protected internal override void DoAfterWriter(TaskScheduler es)
        {
            SearcherFactory factory = new SearcherFactoryAnonymousInnerClassHelper(this, es);
            if (Random().NextBoolean())
            {
                // TODO: can we randomize the applyAllDeletes?  But
                // somehow for final searcher we must apply
                // deletes...
                Mgr = new SearcherManager(Writer, true, factory);
                IsNRT = true;
            }
            else
            {
                // SearcherManager needs to see empty commit:
                Writer.Commit();
                Mgr = new SearcherManager(Dir, factory);
                IsNRT = false;
                AssertMergedSegmentsWarmed = false;
            }

            LifetimeMGR = new SearcherLifetimeManager();
        }
 public virtual void Dispose()
 {
     if (searcherMgr != null)
     {
         searcherMgr.Dispose();
         searcherMgr = null;
     }
     if (writer != null)
     {
         writer.Dispose();
         dir.Dispose();
         writer = null;
     }
 }
        protected internal override void DoAfterWriter(TaskScheduler es)
        {
            double minReopenSec = 0.01 + 0.05 * Random().NextDouble();
            double maxReopenSec = minReopenSec * (1.0 + 10 * Random().NextDouble());

            if (VERBOSE)
            {
              Console.WriteLine("TEST: make SearcherManager maxReopenSec=" + maxReopenSec + " minReopenSec=" + minReopenSec);
            }

            GenWriter = new TrackingIndexWriter(Writer);

            SearcherFactory sf = new SearcherFactoryAnonymousInnerClassHelper(this, es);

            NrtNoDeletes = new SearcherManager(Writer, false, sf);
            NrtDeletes = new SearcherManager(Writer, true, sf);

            NrtDeletesThread = new ControlledRealTimeReopenThread<>(GenWriter, NrtDeletes, maxReopenSec, minReopenSec);
            NrtDeletesThread.Name = "NRTDeletes Reopen Thread";
            NrtDeletesThread.Priority = Math.Min(Thread.CurrentThread.Priority + 2, Thread.MAX_PRIORITY);
            NrtDeletesThread.SetDaemon(true);
            NrtDeletesThread.Start();

            NrtNoDeletesThread = new ControlledRealTimeReopenThread<>(GenWriter, NrtNoDeletes, maxReopenSec, minReopenSec);
            NrtNoDeletesThread.Name = "NRTNoDeletes Reopen Thread";
            NrtNoDeletesThread.Priority = Math.Min(Thread.CurrentThread.Priority + 2, Thread.MAX_PRIORITY);
            NrtNoDeletesThread.SetDaemon(true);
            NrtNoDeletesThread.Start();
        }
 public ThreadAnonymousInnerClassHelper(TestControlledRealTimeReopenThread outerInstance, CountDownLatch latch, CountDownLatch signal, TrackingIndexWriter writer, SearcherManager manager)
 {
     this.OuterInstance = outerInstance;
     this.Latch = latch;
     this.Signal = signal;
     this.Writer = writer;
     this.Manager = manager;
 }
 public virtual void TestListenerCalled()
 {
     Directory dir = NewDirectory();
     IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
     AtomicBoolean afterRefreshCalled = new AtomicBoolean(false);
     SearcherManager sm = new SearcherManager(iw, false, new SearcherFactory());
     sm.AddListener(new RefreshListenerAnonymousInnerClassHelper(this, afterRefreshCalled));
     iw.AddDocument(new Document());
     iw.Commit();
     Assert.IsFalse(afterRefreshCalled.Get());
     sm.MaybeRefreshBlocking();
     Assert.IsTrue(afterRefreshCalled.Get());
     sm.Dispose();
     iw.Dispose();
     dir.Dispose();
 }
        public SearchManager(Directory dir)
        {
            searcherManager = new SearcherManager(dir, new SearcherFactory());

            Task.Run(Refresher);
        }
            public NodeState(ShardSearchingTestBase outerInstance, Random random, int nodeID, int numNodes)
            {
                this.OuterInstance = outerInstance;
                MyNodeID = nodeID;
                Dir = NewFSDirectory(CreateTempDir("ShardSearchingTestBase"));
                // TODO: set warmer
                MockAnalyzer analyzer = new MockAnalyzer(Random());
                analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
                IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
                iwc.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE);
                if (VERBOSE)
                {
                    iwc.InfoStream = new PrintStreamInfoStream(Console.Out);
                }
                Writer = new IndexWriter(Dir, iwc);
                Mgr = new SearcherManager(Writer, true, null);
                Searchers = new SearcherLifetimeManager();

                // Init w/ 0s... caller above will do initial
                // "broadcast" by calling initSearcher:
                CurrentNodeVersions = new long[numNodes];
            }
 public ThreadAnonymousInnerClassHelper2(TestSearcherManager outerInstance, SearcherManager sm)
 {
     this.OuterInstance = outerInstance;
     this.Sm = sm;
 }
        public virtual void TestReferenceDecrementIllegally()
        {
            Directory dir = NewDirectory();
            IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(new ConcurrentMergeScheduler()));
            SearcherManager sm = new SearcherManager(writer, false, new SearcherFactory());
            writer.AddDocument(new Document());
            writer.Commit();
            sm.MaybeRefreshBlocking();

            IndexSearcher acquire = sm.Acquire();
            IndexSearcher acquire2 = sm.Acquire();
            sm.Release(acquire);
            sm.Release(acquire2);

            acquire = sm.Acquire();
            acquire.IndexReader.DecRef();
            sm.Release(acquire);
            try
            {
                sm.Acquire();
                Assert.Fail("acquire should have thrown an InvalidOperationException since we modified the refCount outside of the manager");
            }
            catch (InvalidOperationException ex)
            {
                //
            }

            // sm.Dispose(); -- already closed
            writer.Dispose();
            dir.Dispose();
        }
 public RunnableAnonymousInnerClassHelper(TestSearcherManager outerInstance, AtomicBoolean triedReopen, SearcherManager searcherManager, AtomicBoolean success, Exception[] exc)
 {
     this.OuterInstance = outerInstance;
     this.TriedReopen = triedReopen;
     this.SearcherManager = searcherManager;
     this.Success = success;
     this.Exc = exc;
 }
        public virtual void TestReferenceDecrementIllegally([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler)
        {
            Directory dir = NewDirectory();
            var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
                            .SetMergeScheduler(scheduler);
            IndexWriter writer = new IndexWriter(dir, config);
            SearcherManager sm = new SearcherManager(writer, false, new SearcherFactory());
            writer.AddDocument(new Document());
            writer.Commit();
            sm.MaybeRefreshBlocking();

            IndexSearcher acquire = sm.Acquire();
            IndexSearcher acquire2 = sm.Acquire();
            sm.Release(acquire);
            sm.Release(acquire2);

            acquire = sm.Acquire();
            acquire.IndexReader.DecRef();
            sm.Release(acquire);

            Assert.Throws<InvalidOperationException>(() => sm.Acquire(), "acquire should have thrown an InvalidOperationException since we modified the refCount outside of the manager");

            // sm.Dispose(); -- already closed
            writer.Dispose();
            dir.Dispose();
        }
        public virtual void TestMaybeRefreshBlockingLock()
        {
            // make sure that maybeRefreshBlocking releases the lock, otherwise other
            // threads cannot obtain it.
            Directory dir = NewDirectory();
            RandomIndexWriter w = new RandomIndexWriter(Random(), dir);
            w.Dispose();

            SearcherManager sm = new SearcherManager(dir, null);

            ThreadClass t = new ThreadAnonymousInnerClassHelper2(this, sm);
            t.Start();
            t.Join();

            // if maybeRefreshBlocking didn't release the lock, this will fail.
            Assert.IsTrue(sm.MaybeRefresh(), "failde to obtain the refreshLock!");

            sm.Dispose();
            dir.Dispose();
        }
        // LUCENE-5461
        public virtual void TestCRTReopen()
        {
            //test behaving badly

            //should be high enough
            int maxStaleSecs = 20;

            //build crap data just to store it.
            string s = "        abcdefghijklmnopqrstuvwxyz     ";
            char[] chars = s.ToCharArray();
            StringBuilder builder = new StringBuilder(2048);
            for (int i = 0; i < 2048; i++)
            {
                builder.Append(chars[Random().Next(chars.Length)]);
            }
            string content = builder.ToString();

            SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
            Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128);
            IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, new MockAnalyzer(Random()));
            config.SetIndexDeletionPolicy(sdp);
            config.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE_OR_APPEND);
            IndexWriter iw = new IndexWriter(dir, config);
            SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory());
            TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
            ControlledRealTimeReopenThread<IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread<IndexSearcher>(tiw, sm, maxStaleSecs, 0);

            controlledRealTimeReopenThread.SetDaemon(true);
            controlledRealTimeReopenThread.Start();

            IList<Thread> commitThreads = new List<Thread>();

            for (int i = 0; i < 500; i++)
            {
                if (i > 0 && i % 50 == 0)
                {
                    Thread commitThread = new Thread(new RunnableAnonymousInnerClassHelper(this, sdp, dir, iw));
                    commitThread.Start();
                    commitThreads.Add(commitThread);
                }
                Document d = new Document();
                d.Add(new TextField("count", i + "", Field.Store.NO));
                d.Add(new TextField("content", content, Field.Store.YES));
                long start = DateTime.Now.Millisecond;
                long l = tiw.AddDocument(d);
                controlledRealTimeReopenThread.WaitForGeneration(l);
                long wait = DateTime.Now.Millisecond - start;
                Assert.IsTrue(wait < (maxStaleSecs * 1000), "waited too long for generation " + wait);
                IndexSearcher searcher = sm.Acquire();
                TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10);
                sm.Release(searcher);
                Assert.AreEqual(1, td.TotalHits);
            }

            foreach (Thread commitThread in commitThreads)
            {
                commitThread.Join();
            }

            controlledRealTimeReopenThread.Dispose();
            sm.Dispose();
            iw.Dispose();
            dir.Dispose();
        }
 public void Dispose()
 {
     stop = true;
     searcherManager.close();
     searcherManager = null;
 }
        public override void Build(IInputIterator iter)
        {
            if (searcherMgr != null)
            {
                searcherMgr.Dispose();
                searcherMgr = null;
            }

            if (writer != null)
            {
                writer.Dispose();
                writer = null;
            }

            AtomicReader r = null;
            bool success = false;
            try
            {
                // First pass: build a temporary normal Lucene index,
                // just indexing the suggestions as they iterate:
                writer = new IndexWriter(dir, GetIndexWriterConfig(matchVersion, GramAnalyzer, IndexWriterConfig.OpenMode_e.CREATE));
                //long t0 = System.nanoTime();

                // TODO: use threads?
                BytesRef text;
                while ((text = iter.Next()) != null)
                {
                    BytesRef payload;
                    if (iter.HasPayloads)
                    {
                        payload = iter.Payload;
                    }
                    else
                    {
                        payload = null;
                    }

                    Add(text, iter.Contexts, iter.Weight, payload);
                }

                //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec");

                searcherMgr = new SearcherManager(writer, true, null);
                success = true;
            }
            finally
            {
                if (success)
                {
                    IOUtils.Close(r);
                }
                else
                {
                    IOUtils.CloseWhileHandlingException(writer, r);
                    writer = null;
                }
            }
        }
 internal SearchWrapper(SearcherManager searcherManager)
 {
     _searcherManager = searcherManager;
     _indexSearcher = (IndexSearcher) _searcherManager.acquire();
 }
        /// <summary>
        /// Create a new instance, loading from a previously built
        /// <see cref="AnalyzingInfixSuggester"/> directory, if it exists.  This directory must be
        /// private to the infix suggester (i.e., not an external
        /// Lucene index).  Note that <see cref="Dispose()"/>
        /// will also dispose the provided directory.
        /// </summary>
        ///  <param name="minPrefixChars"> Minimum number of leading characters
        ///     before <see cref="PrefixQuery"/> is used (default 4).
        ///     Prefixes shorter than this are indexed as character
        ///     ngrams (increasing index size but making lookups
        ///     faster). </param>
        public AnalyzingInfixSuggester(LuceneVersion matchVersion, Directory dir, Analyzer indexAnalyzer, 
            Analyzer queryAnalyzer, int minPrefixChars)
        {
            if (minPrefixChars < 0)
            {
                throw new System.ArgumentException("minPrefixChars must be >= 0; got: " + minPrefixChars);
            }

            this.queryAnalyzer = queryAnalyzer;
            this.indexAnalyzer = indexAnalyzer;
            this.matchVersion = matchVersion;
            this.dir = dir;
            this.minPrefixChars = minPrefixChars;

            if (DirectoryReader.IndexExists(dir))
            {
                // Already built; open it:
                writer = new IndexWriter(dir, GetIndexWriterConfig(matchVersion, GramAnalyzer, IndexWriterConfig.OpenMode_e.APPEND));
                searcherMgr = new SearcherManager(writer, true, null);
            }
        }
 public virtual void TestCloseTwice()
 {
     // test that we can close SM twice (per IDisposable's contract).
     Directory dir = NewDirectory();
     (new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose();
     SearcherManager sm = new SearcherManager(dir, null);
     sm.Dispose();
     sm.Dispose();
     dir.Dispose();
 }
        /*
         * LUCENE-3528 - NRTManager hangs in certain situations
         */
        public virtual void TestThreadStarvationNoDeleteNRTReader()
        {
            IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
            conf.SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES);
            Directory d = NewDirectory();
            CountDownLatch latch = new CountDownLatch(1);
            CountDownLatch signal = new CountDownLatch(1);

            LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal);
            TrackingIndexWriter writer = new TrackingIndexWriter(_writer);
            SearcherManager manager = new SearcherManager(_writer, false, null);
            Document doc = new Document();
            doc.Add(NewTextField("test", "test", Field.Store.YES));
            writer.AddDocument(doc);
            manager.MaybeRefresh();
            ThreadClass t = new ThreadAnonymousInnerClassHelper(this, latch, signal, writer, manager);
            t.Start();
            _writer.WaitAfterUpdate = true; // wait in addDocument to let some reopens go through
            long lastGen = writer.UpdateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen

            Assert.IsFalse(manager.SearcherCurrent); // false since there is a delete in the queue

            IndexSearcher searcher = manager.Acquire();
            try
            {
                Assert.AreEqual(2, searcher.IndexReader.NumDocs());
            }
            finally
            {
                manager.Release(searcher);
            }
            ControlledRealTimeReopenThread<IndexSearcher> thread = new ControlledRealTimeReopenThread<IndexSearcher>(writer, manager, 0.01, 0.01);
            thread.Start(); // start reopening
            if (VERBOSE)
            {
                Console.WriteLine("waiting now for generation " + lastGen);
            }

            AtomicBoolean finished = new AtomicBoolean(false);
            ThreadClass waiter = new ThreadAnonymousInnerClassHelper2(this, lastGen, thread, finished);
            waiter.Start();
            manager.MaybeRefresh();
            waiter.Join(1000);
            if (!finished.Get())
            {
                waiter.Interrupt();
                Assert.Fail("thread deadlocked on waitForGeneration");
            }
            thread.Dispose();
            thread.Join();
            IOUtils.Close(manager, _writer, d);
        }
        public virtual void TestEnsureOpen()
        {
            Directory dir = NewDirectory();
            (new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose();
            SearcherManager sm = new SearcherManager(dir, null);
            IndexSearcher s = sm.Acquire();
            sm.Dispose();

            // this should succeed;
            sm.Release(s);

            try
            {
                // this should fail
                sm.Acquire();
            }
            catch (AlreadyClosedException e)
            {
                // ok
            }

            try
            {
                // this should fail
                sm.MaybeRefresh();
            }
            catch (AlreadyClosedException e)
            {
                // ok
            }
            dir.Dispose();
        }