public Result Search (string term, int count, int start) { try { term = term.ToLower (); Term htTerm = new Term ("hottext", term); Query qq1 = new FuzzyQuery (htTerm); Query qq2 = new TermQuery (htTerm); qq2.Boost = 10f; Query qq3 = new PrefixQuery (htTerm); qq3.Boost = 10f; DisjunctionMaxQuery q1 = new DisjunctionMaxQuery (0f); q1.Add (qq1); q1.Add (qq2); q1.Add (qq3); Query q2 = new TermQuery (new Term ("text", term)); q2.Boost = 3f; Query q3 = new TermQuery (new Term ("examples", term)); q3.Boost = 3f; DisjunctionMaxQuery q = new DisjunctionMaxQuery (0f); q.Add (q1); q.Add (q2); q.Add (q3); TopDocs top = SearchInternal (q, count, start); Result r = new Result (term, searcher, top.ScoreDocs); Results.Add (r); return r; } catch (IOException) { Console.WriteLine ("No index in {0}", dir); return null; } }
public virtual void TestPrefixQuery_Mem() { Directory directory = NewDirectory(); string[] categories = new string[] { "/Computers", "/Computers/Mac", "/Computers/Windows" }; RandomIndexWriter writer = new RandomIndexWriter(Random(), directory, Similarity, TimeZone); for (int i = 0; i < categories.Length; i++) { Document doc = new Document(); doc.Add(NewStringField("category", categories[i], Field.Store.YES)); writer.AddDocument(doc); } IndexReader reader = writer.Reader; PrefixQuery query = new PrefixQuery(new Term("category", "/Computers")); IndexSearcher searcher = NewSearcher(reader); ScoreDoc[] hits = searcher.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(3, hits.Length, "All documents in /Computers category and below"); query = new PrefixQuery(new Term("category", "/Computers/Mac")); hits = searcher.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(1, hits.Length, "One in /Computers/Mac"); query = new PrefixQuery(new Term("category", "")); Terms terms = MultiFields.GetTerms(searcher.IndexReader, "category"); Assert.IsFalse(query.GetTermsEnum(terms) is PrefixTermsEnum); hits = searcher.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(3, hits.Length, "everything"); writer.Dispose(); reader.Dispose(); directory.Dispose(); }
public override Query VisitPrefixQuery(PrefixQuery prefixq) { _dump.Append("PrefixQ("); var q = base.VisitPrefixQuery(prefixq); _dump.Append(BoostToString(q)); _dump.Append(")"); return q; }
public virtual void TestMethod() { RAMDirectory directory = new RAMDirectory(); System.String[] categories = new System.String[]{"food", "foodanddrink", "foodanddrinkandgoodtimes", "food and drink"}; Query rw1 = null; Query rw2 = null; try { IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true); for (int i = 0; i < categories.Length; i++) { Document doc = new Document(); doc.Add(Field.Keyword("category", categories[i])); writer.AddDocument(doc); } writer.Close(); IndexReader reader = IndexReader.Open(directory); PrefixQuery query = new PrefixQuery(new Term("category", "foo")); rw1 = query.Rewrite(reader); BooleanQuery bq = new BooleanQuery(); bq.Add(query, true, false); rw2 = bq.Rewrite(reader); } catch (System.IO.IOException e) { Assert.Fail(e.Message); } BooleanQuery bq1 = null; if (rw1 is BooleanQuery) { bq1 = (BooleanQuery) rw1; } BooleanQuery bq2 = null; if (rw2 is BooleanQuery) { bq2 = (BooleanQuery) rw2; } else { Assert.Fail("Rewrite"); } Assert.AreEqual(bq1.GetClauses().Length, bq2.GetClauses().Length, "Number of Clauses Mismatch"); }
public override Query Rewrite(IndexReader reader) { if (_termIsPrefix) { MultiTermQuery rewritten = new PrefixQuery(internalTerm.CreateTerm(internalTerm.Text.Substring(0, internalTerm.Text.IndexOf('*')))); rewritten.Boost = Boost; rewritten.RewriteMethod = RewriteMethod; return rewritten; } else { return base.Rewrite(reader); } }
public virtual Query Build(IQueryNode queryNode) { PrefixWildcardQueryNode wildcardNode = (PrefixWildcardQueryNode)queryNode; string text = wildcardNode.Text.SubSequence(0, wildcardNode.Text.Length - 1).ToString(); PrefixQuery q = new PrefixQuery(new Term(wildcardNode.GetFieldAsString(), text)); MultiTermQuery.RewriteMethod method = (MultiTermQuery.RewriteMethod)queryNode.GetTag(MultiTermRewriteMethodProcessor.TAG_ID); if (method != null) { q.SetRewriteMethod(method); } return q; }
public Query Parse(string text, string field) { if (string.IsNullOrWhiteSpace(text)) { return null; } var tokens = this.Tokenize(text).ToList(); if (!tokens.Any()) { return null; } var outerQuery = new BooleanQuery(); foreach (string token in tokens) { var query = new PrefixQuery(new Term(field, token)); outerQuery.Add(query, Occur.MUST); } return outerQuery; }
public virtual void TestPrefixQuery_Mem() { Directory directory = NewDirectory(); string[] categories = new string[] { "/Computers", "/Computers/Mac", "/Computers/Windows" }; RandomIndexWriter writer = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, directory); for (int i = 0; i < categories.Length; i++) { Document doc = new Document(); doc.Add(NewStringField("category", categories[i], Field.Store.YES)); writer.AddDocument(doc); } IndexReader reader = writer.GetReader(); PrefixQuery query = new PrefixQuery(new Term("category", "/Computers")); IndexSearcher searcher = NewSearcher(reader); ScoreDoc[] hits = searcher.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(3, hits.Length, "All documents in /Computers category and below"); query = new PrefixQuery(new Term("category", "/Computers/Mac")); hits = searcher.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(1, hits.Length, "One in /Computers/Mac"); query = new PrefixQuery(new Term("category", "")); Terms terms = MultiFields.GetTerms(searcher.IndexReader, "category"); Assert.IsFalse(query.GetTermsEnum(terms) is PrefixTermsEnum); hits = searcher.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(3, hits.Length, "everything"); writer.Dispose(); reader.Dispose(); directory.Dispose(); }
public virtual void TestPrefixQuery_Renamed_Method() { RAMDirectory directory = new RAMDirectory(); System.String[] categories = new System.String[]{"/Computers", "/Computers/Mac", "/Computers/Windows"}; IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true); for (int i = 0; i < categories.Length; i++) { Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document(); doc.Add(new Field("category", categories[i], Field.Store.YES, Field.Index.UN_TOKENIZED)); writer.AddDocument(doc); } writer.Close(); PrefixQuery query = new PrefixQuery(new Term("category", "/Computers")); IndexSearcher searcher = new IndexSearcher(directory); Hits hits = searcher.Search(query); Assert.AreEqual(3, hits.Length(), "All documents in /Computers category and below"); query = new PrefixQuery(new Term("category", "/Computers/Mac")); hits = searcher.Search(query); Assert.AreEqual(1, hits.Length(), "One in /Computers/Mac"); }
public virtual void TestPrefixQuery_Renamed() { RAMDirectory directory = new RAMDirectory(); System.String[] categories = new System.String[]{"/Computers", "/Computers/Mac", "/Computers/Windows"}; IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); for (int i = 0; i < categories.Length; i++) { Document doc = new Document(); doc.Add(new Field("category", categories[i], Field.Store.YES, Field.Index.NOT_ANALYZED)); writer.AddDocument(doc); } writer.Close(); PrefixQuery query = new PrefixQuery(new Term("category", "/Computers")); IndexSearcher searcher = new IndexSearcher(directory); ScoreDoc[] hits = searcher.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(3, hits.Length, "All documents in /Computers category and below"); query = new PrefixQuery(new Term("category", "/Computers/Mac")); hits = searcher.Search(query, null, 1000).ScoreDocs; Assert.AreEqual(1, hits.Length, "One in /Computers/Mac"); }
public virtual void TestMethod() { RAMDirectory directory = new RAMDirectory(); System.String[] categories = new System.String[] { "food", "foodanddrink", "foodanddrinkandgoodtimes", "food and drink" }; Query rw1 = null; Query rw2 = null; IndexReader reader = null; try { IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); for (int i = 0; i < categories.Length; i++) { Document doc = new Document(); doc.Add(new Field("category", categories[i], Field.Store.YES, Field.Index.NOT_ANALYZED)); writer.AddDocument(doc); } writer.Close(); reader = IndexReader.Open(directory); PrefixQuery query = new PrefixQuery(new Term("category", "foo")); rw1 = query.Rewrite(reader); BooleanQuery bq = new BooleanQuery(); bq.Add(query, BooleanClause.Occur.MUST); rw2 = bq.Rewrite(reader); } catch (System.IO.IOException e) { Assert.Fail(e.Message); } Assert.AreEqual(GetCount(reader, rw1), GetCount(reader, rw2), "Number of Clauses Mismatch"); }
public void TestPrefixTerm() { RAMDirectory indexStore = GetIndexStore("field", new String[] { "prefix", "prefixx" }); IndexSearcher searcher = new IndexSearcher(indexStore, true); MultiTermQuery wq = new WildcardQuery(new Term("field", "prefix*")); AssertMatches(searcher, wq, 2); MultiTermQuery expected = new PrefixQuery(new Term("field", "prefix")); wq.RewriteMethod = MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE; wq.Boost = 0.1F; expected.RewriteMethod = wq.RewriteMethod; expected.Boost = wq.Boost; Assert.AreEqual(searcher.Rewrite(expected), searcher.Rewrite(wq)); wq.RewriteMethod = MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE; wq.Boost = 0.2F; expected.RewriteMethod = wq.RewriteMethod; expected.Boost = wq.Boost; Assert.AreEqual(searcher.Rewrite(expected), searcher.Rewrite(wq)); wq.RewriteMethod = MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT; wq.Boost = 0.3F; expected.RewriteMethod = wq.RewriteMethod; expected.Boost = wq.Boost; Assert.AreEqual(searcher.Rewrite(expected), searcher.Rewrite(wq)); wq.RewriteMethod = MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE; wq.Boost = 0.4F; expected.RewriteMethod = wq.RewriteMethod; expected.Boost = wq.Boost; Assert.AreEqual(searcher.Rewrite(expected), searcher.Rewrite(wq)); }
protected Query Preq(float boost, String field, String text) { Query query = new PrefixQuery(new Term(field, text)); query.Boost = boost; return query; }
public Result FastSearch (string term, int number) { try { term = term.ToLower (); Query q1 = new TermQuery (new Term ("hottext", term)); Query q2 = new PrefixQuery (new Term ("hottext", term)); q2.Boost = 0.5f; DisjunctionMaxQuery q = new DisjunctionMaxQuery (0f); q.Add (q1); q.Add (q2); TopDocs top = searcher.Search (q, number); return new Result (term, searcher, top.ScoreDocs); } catch (IOException) { Console.WriteLine ("No index in {0}", dir); return null; } }
public virtual void TestParsingAndSearching() { string field = "content"; string[] docs = new string[] { "\\ abcdefg1", "\\x00079 hijklmn1", "\\\\ opqrstu1" }; // queries that should find all docs Query[] matchAll = new Query[] { new WildcardQuery(new Term(field, "*")), new WildcardQuery(new Term(field, "*1")), new WildcardQuery(new Term(field, "**1")), new WildcardQuery(new Term(field, "*?")), new WildcardQuery(new Term(field, "*?1")), new WildcardQuery(new Term(field, "?*1")), new WildcardQuery(new Term(field, "**")), new WildcardQuery(new Term(field, "***")), new WildcardQuery(new Term(field, "\\\\*")) }; // queries that should find no docs Query[] matchNone = new Query[] { new WildcardQuery(new Term(field, "a*h")), new WildcardQuery(new Term(field, "a?h")), new WildcardQuery(new Term(field, "*a*h")), new WildcardQuery(new Term(field, "?a")), new WildcardQuery(new Term(field, "a?")) }; PrefixQuery[][] matchOneDocPrefix = new PrefixQuery[][] { new PrefixQuery[] { new PrefixQuery(new Term(field, "a")), new PrefixQuery(new Term(field, "ab")), new PrefixQuery(new Term(field, "abc")) }, new PrefixQuery[] { new PrefixQuery(new Term(field, "h")), new PrefixQuery(new Term(field, "hi")), new PrefixQuery(new Term(field, "hij")), new PrefixQuery(new Term(field, "\\x0007")) }, new PrefixQuery[] { new PrefixQuery(new Term(field, "o")), new PrefixQuery(new Term(field, "op")), new PrefixQuery(new Term(field, "opq")), new PrefixQuery(new Term(field, "\\\\")) } }; WildcardQuery[][] matchOneDocWild = new WildcardQuery[][] { new WildcardQuery[] { new WildcardQuery(new Term(field, "*a*")), new WildcardQuery(new Term(field, "*ab*")), new WildcardQuery(new Term(field, "*abc**")), new WildcardQuery(new Term(field, "ab*e*")), new WildcardQuery(new Term(field, "*g?")), new WildcardQuery(new Term(field, "*f?1")) }, new WildcardQuery[] { new WildcardQuery(new Term(field, "*h*")), new WildcardQuery(new Term(field, "*hi*")), new WildcardQuery(new Term(field, "*hij**")), new WildcardQuery(new Term(field, "hi*k*")), new WildcardQuery(new Term(field, "*n?")), new WildcardQuery(new Term(field, "*m?1")), new WildcardQuery(new Term(field, "hij**")) }, new WildcardQuery[] { new WildcardQuery(new Term(field, "*o*")), new WildcardQuery(new Term(field, "*op*")), new WildcardQuery(new Term(field, "*opq**")), new WildcardQuery(new Term(field, "op*q*")), new WildcardQuery(new Term(field, "*u?")), new WildcardQuery(new Term(field, "*t?1")), new WildcardQuery(new Term(field, "opq**")) } }; // prepare the index Directory dir = NewDirectory(); RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy())); for (int i = 0; i < docs.Length; i++) { Document doc = new Document(); doc.Add(NewTextField(field, docs[i], Field.Store.NO)); iw.AddDocument(doc); } iw.Dispose(); IndexReader reader = DirectoryReader.Open(dir); IndexSearcher searcher = NewSearcher(reader); // test queries that must find all foreach (Query q in matchAll) { if (VERBOSE) { Console.WriteLine("matchAll: q=" + q + " " + q.GetType().Name); } ScoreDoc[] hits = searcher.Search(q, null, 1000).ScoreDocs; Assert.AreEqual(docs.Length, hits.Length); } // test queries that must find none foreach (Query q in matchNone) { if (VERBOSE) { Console.WriteLine("matchNone: q=" + q + " " + q.GetType().Name); } ScoreDoc[] hits = searcher.Search(q, null, 1000).ScoreDocs; Assert.AreEqual(0, hits.Length); } // thest the prefi queries find only one doc for (int i = 0; i < matchOneDocPrefix.Length; i++) { for (int j = 0; j < matchOneDocPrefix[i].Length; j++) { Query q = matchOneDocPrefix[i][j]; if (VERBOSE) { Console.WriteLine("match 1 prefix: doc=" + docs[i] + " q=" + q + " " + q.GetType().Name); } ScoreDoc[] hits = searcher.Search(q, null, 1000).ScoreDocs; Assert.AreEqual(1, hits.Length); Assert.AreEqual(i, hits[0].Doc); } } // test the wildcard queries find only one doc for (int i = 0; i < matchOneDocWild.Length; i++) { for (int j = 0; j < matchOneDocWild[i].Length; j++) { Query q = matchOneDocWild[i][j]; if (VERBOSE) { Console.WriteLine("match 1 wild: doc=" + docs[i] + " q=" + q + " " + q.GetType().Name); } ScoreDoc[] hits = searcher.Search(q, null, 1000).ScoreDocs; Assert.AreEqual(1, hits.Length); Assert.AreEqual(i, hits[0].Doc); } } reader.Dispose(); dir.Dispose(); }
/// <summary> /// macro for readability </summary> public static Query Cspq(Term prefix) { PrefixQuery query = new PrefixQuery(prefix); query.SetRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE); return query; }
/// <summary> /// check that the # of hits is the same as from a very /// simple prefixquery implementation. /// </summary> private void AssertSame(string prefix) { PrefixQuery smart = new PrefixQuery(new Term("field", prefix)); DumbPrefixQuery dumb = new DumbPrefixQuery(this, new Term("field", prefix)); TopDocs smartDocs = Searcher.Search(smart, 25); TopDocs dumbDocs = Searcher.Search(dumb, 25); CheckHits.CheckEqual(smart, smartDocs.ScoreDocs, dumbDocs.ScoreDocs); }
/// <summary> /// 批量添加TermQuery或PrefixQuery搜索条件 /// </summary> /// <param name="fieldNameAndBoosts">字段名称及权重集合</param> /// <param name="values">字段值集合</param> /// <param name="exactMatch">是否精确搜索</param> /// <param name="occur">搜索条件间的关系</param> /// <param name="asFilter">是否作为过滤条件</param> /// <returns>LuceneSearchBuilder</returns> public LuceneSearchBuilder WithFields(Dictionary<string, BoostLevel> fieldNameAndBoosts, IEnumerable<string> values, bool exactMatch, BooleanClause.Occur occur, bool asFilter = false) { foreach (var fieldNameAndBoost in fieldNameAndBoosts) { BooleanQuery query = new BooleanQuery(); foreach (string value in values) { string filteredValue = ClauseScrubber.LuceneKeywordsScrub(value); if (string.IsNullOrEmpty(filteredValue)) continue; Term term = new Term(fieldNameAndBoost.Key, filteredValue); Query q; if (exactMatch) q = new TermQuery(term); else q = new PrefixQuery(term); SetBoost(q, fieldNameAndBoost.Value); query.Add(q, BooleanClause.Occur.SHOULD); } if (asFilter) filters.Add(new BooleanClause(query, occur)); else clauses.Add(new BooleanClause(query, occur)); } return this; }
/// <summary> /// 批量添加PhraseQuery /// </summary> /// <param name="phrases">待搜索的短语集合</param> /// <param name="fieldNameAndBoosts">字段名称及权重集合</param> /// <param name="occur">搜索条件间的关系</param> /// <param name="asFilter">是否作为过滤器条件</param> /// <returns></returns> public LuceneSearchBuilder WithPhrases(Dictionary<string, BoostLevel> fieldNameAndBoosts, IEnumerable<string> phrases, BooleanClause.Occur occur, bool asFilter = false) { foreach (var fieldNameAndBoost in fieldNameAndBoosts) { BooleanQuery query = new BooleanQuery(); foreach (string phrase in phrases) { string filteredPhrase = ClauseScrubber.LuceneKeywordsScrub(phrase); if (string.IsNullOrEmpty(filteredPhrase)) continue; if (filteredPhrase.Length == 1) { Term term = new Term(fieldNameAndBoost.Key, filteredPhrase); Query q = new PrefixQuery(term); SetBoost(q, fieldNameAndBoost.Value); query.Add(q, BooleanClause.Occur.SHOULD); continue; } string[] nameSegments = ClauseScrubber.SegmentForPhraseQuery(filteredPhrase); PhraseQuery phraseQuery = new PhraseQuery(); foreach (var nameSegment in nameSegments) phraseQuery.Add(new Term(fieldNameAndBoost.Key, nameSegment)); phraseQuery.SetSlop(PhraseQuerySlop); SetBoost(phraseQuery, fieldNameAndBoost.Value); query.Add(phraseQuery, BooleanClause.Occur.SHOULD); } if (asFilter) filters.Add(new BooleanClause(query, occur)); else clauses.Add(new BooleanClause(query, occur)); } return this; }
/// <summary> /// 根据多个关键字添加TermQuery或PrefixQuery搜索条件 /// </summary> /// <param name="fieldName">字段名称</param> /// <param name="values">字段值列表</param> /// <param name="exactMatch">是否精确搜索</param> /// <param name="boostLevel">权重级别</param> /// <param name="asFilter">是否作为过滤条件</param> public LuceneSearchBuilder WithFields(string fieldName, IEnumerable<string> values, bool exactMatch = true, BoostLevel? boostLevel = null, bool asFilter = false) { BooleanQuery query = new BooleanQuery(); foreach (string value in values) { string filteredValue = ClauseScrubber.LuceneKeywordsScrub(value); if (string.IsNullOrEmpty(filteredValue)) continue; Term term = new Term(fieldName, filteredValue); Query q; if (exactMatch) q = new TermQuery(term); else q = new PrefixQuery(term); if (boostLevel.HasValue) SetBoost(q, boostLevel.Value); query.Add(q, BooleanClause.Occur.SHOULD); } if (asFilter) filters.Add(new BooleanClause(query, BooleanClause.Occur.MUST)); else clauses.Add(new BooleanClause(query, BooleanClause.Occur.MUST)); return this; }
/// <summary> /// 添加TermQuery或PrefixQuery搜索条件 /// </summary> /// <param name="fieldName">字段名称</param> /// <param name="value">字段值</param> /// <param name="exactMatch">是否精确搜索</param> /// <param name="boostLevel">权重级别</param> /// <param name="asFilter">是否作为过滤条件</param> /// <returns>LuceneSearchBuilder</returns> public LuceneSearchBuilder WithField(string fieldName, string value, bool exactMatch = true, BoostLevel? boostLevel = null, bool asFilter = false) { string filteredValue = ClauseScrubber.LuceneKeywordsScrub(value); if (string.IsNullOrEmpty(filteredValue)) return this; Term term = new Term(fieldName, filteredValue); Query query; if (exactMatch) query = new TermQuery(term); else query = new PrefixQuery(term); if (boostLevel.HasValue) SetBoost(query, boostLevel.Value); if (asFilter) filters.Add(new BooleanClause(query, BooleanClause.Occur.MUST)); else clauses.Add(new BooleanClause(query, BooleanClause.Occur.MUST)); return this; }
public override Query ProcessQuery(string indexName, Query query, IndexQuery originalQuery) { if (indexName != SpecificIndexName) return query; var customQuery = new PrefixQuery(new Term("CustomField", "CustomPrefix")); return new BooleanQuery { { query, Occur.MUST }, { customQuery, Occur.MUST} }; }
public virtual void TestSimple() { int numNodes = TestUtil.NextInt(Random(), 1, 10); double runTimeSec = AtLeast(3); int minDocsToMakeTerms = TestUtil.NextInt(Random(), 5, 20); int maxSearcherAgeSeconds = TestUtil.NextInt(Random(), 1, 3); if (VERBOSE) { Console.WriteLine("TEST: numNodes=" + numNodes + " runTimeSec=" + runTimeSec + " maxSearcherAgeSeconds=" + maxSearcherAgeSeconds); } Start(numNodes, runTimeSec, maxSearcherAgeSeconds); List<PreviousSearchState> priorSearches = new List<PreviousSearchState>(); List<BytesRef> terms = null; while (TimeHelper.NanoTime() < EndTimeNanos) { bool doFollowon = priorSearches.Count > 0 && Random().Next(7) == 1; // Pick a random node; we will run the query on this node: int myNodeID = Random().Next(numNodes); NodeState.ShardIndexSearcher localShardSearcher; PreviousSearchState prevSearchState; if (doFollowon) { // Pretend user issued a followon query: prevSearchState = priorSearches[Random().Next(priorSearches.Count)]; if (VERBOSE) { Console.WriteLine("\nTEST: follow-on query age=" + ((TimeHelper.NanoTime() - prevSearchState.SearchTimeNanos) / 1000000000.0)); } try { localShardSearcher = Nodes[myNodeID].Acquire(prevSearchState.Versions); } catch (SearcherExpiredException see) { // Expected, sometimes; in a "real" app we would // either forward this error to the user ("too // much time has passed; please re-run your // search") or sneakily just switch to newest // searcher w/o telling them... if (VERBOSE) { Console.WriteLine(" searcher expired during local shard searcher init: " + see); } priorSearches.Remove(prevSearchState); continue; } } else { if (VERBOSE) { Console.WriteLine("\nTEST: fresh query"); } // Do fresh query: localShardSearcher = Nodes[myNodeID].Acquire(); prevSearchState = null; } IndexReader[] subs = new IndexReader[numNodes]; PreviousSearchState searchState = null; try { // Mock: now make a single reader (MultiReader) from all node // searchers. In a real shard env you can't do this... we // do it to confirm results from the shard searcher // are correct: int docCount = 0; try { for (int nodeID = 0; nodeID < numNodes; nodeID++) { long subVersion = localShardSearcher.NodeVersions[nodeID]; IndexSearcher sub = Nodes[nodeID].Searchers.Acquire(subVersion); if (sub == null) { nodeID--; while (nodeID >= 0) { subs[nodeID].DecRef(); subs[nodeID] = null; nodeID--; } throw new SearcherExpiredException("nodeID=" + nodeID + " version=" + subVersion); } subs[nodeID] = sub.IndexReader; docCount += subs[nodeID].MaxDoc(); } } catch (SearcherExpiredException see) { // Expected if (VERBOSE) { Console.WriteLine(" searcher expired during mock reader init: " + see); } continue; } IndexReader mockReader = new MultiReader(subs); IndexSearcher mockSearcher = new IndexSearcher(mockReader); Query query; Sort sort; if (prevSearchState != null) { query = prevSearchState.Query; sort = prevSearchState.Sort; } else { if (terms == null && docCount > minDocsToMakeTerms) { // TODO: try to "focus" on high freq terms sometimes too // TODO: maybe also periodically reset the terms...? TermsEnum termsEnum = MultiFields.GetTerms(mockReader, "body").Iterator(null); terms = new List<BytesRef>(); while (termsEnum.Next() != null) { terms.Add(BytesRef.DeepCopyOf(termsEnum.Term())); } if (VERBOSE) { Console.WriteLine("TEST: init terms: " + terms.Count + " terms"); } if (terms.Count == 0) { terms = null; } } if (VERBOSE) { Console.WriteLine(" maxDoc=" + mockReader.MaxDoc()); } if (terms != null) { if (Random().NextBoolean()) { query = new TermQuery(new Term("body", terms[Random().Next(terms.Count)])); } else { string t = terms[Random().Next(terms.Count)].Utf8ToString(); string prefix; if (t.Length <= 1) { prefix = t; } else { prefix = t.Substring(0, TestUtil.NextInt(Random(), 1, 2)); } query = new PrefixQuery(new Term("body", prefix)); } if (Random().NextBoolean()) { sort = null; } else { // TODO: sort by more than 1 field int what = Random().Next(3); if (what == 0) { sort = new Sort(SortField.FIELD_SCORE); } else if (what == 1) { // TODO: this sort doesn't merge // correctly... it's tricky because you // could have > 2.1B docs across all shards: //sort = new Sort(SortField.FIELD_DOC); sort = null; } else if (what == 2) { sort = new Sort(new SortField[] { new SortField("docid", SortField.Type_e.INT, Random().NextBoolean()) }); } else { sort = new Sort(new SortField[] { new SortField("title", SortField.Type_e.STRING, Random().NextBoolean()) }); } } } else { query = null; sort = null; } } if (query != null) { try { searchState = AssertSame(mockSearcher, localShardSearcher, query, sort, prevSearchState); } catch (SearcherExpiredException see) { // Expected; in a "real" app we would // either forward this error to the user ("too // much time has passed; please re-run your // search") or sneakily just switch to newest // searcher w/o telling them... if (VERBOSE) { Console.WriteLine(" searcher expired during search: " + see); Console.Out.Write(see.StackTrace); } // We can't do this in general: on a very slow // computer it's possible the local searcher // expires before we can finish our search: // assert prevSearchState != null; if (prevSearchState != null) { priorSearches.Remove(prevSearchState); } } } } finally { Nodes[myNodeID].Release(localShardSearcher); foreach (IndexReader sub in subs) { if (sub != null) { sub.DecRef(); } } } if (searchState != null && searchState.SearchAfterLocal != null && Random().Next(5) == 3) { priorSearches.Add(searchState); if (priorSearches.Count > 200) { priorSearches = (List<PreviousSearchState>)CollectionsHelper.Shuffle(priorSearches); priorSearches.SubList(100, priorSearches.Count).Clear(); } } } Finish(); }
private static Query BuildTermQuery(string termQuery) { Query mainQuery; if (termQuery.Length < 3) mainQuery = new PrefixQuery(new Term("name", termQuery)); else { BooleanQuery boolQuery = new BooleanQuery(); var fuzzQuery = new FuzzyQuery(new Term("name", termQuery)); var prefQuery = new PrefixQuery(new Term("name", termQuery)); boolQuery.Add(prefQuery, BooleanClause.Occur.SHOULD); boolQuery.Add(fuzzQuery, BooleanClause.Occur.SHOULD); mainQuery = boolQuery; //first implemetation //mainQuery = fuzzQuery; } return mainQuery; }
/// <summary> /// Factory method to generate a prefix query. /// </summary> protected virtual Query NewPrefixQuery(string text) { BooleanQuery bq = new BooleanQuery(true); foreach (var entry in weights) { PrefixQuery prefix = new PrefixQuery(new Term(entry.Key, text)); prefix.Boost = entry.Value; bq.Add(prefix, BooleanClause.Occur.SHOULD); } return Simplify(bq); }
public virtual void TestMethod() { RAMDirectory directory = new RAMDirectory(); System.String[] categories = new System.String[]{"food", "foodanddrink", "foodanddrinkandgoodtimes", "food and drink"}; Query rw1 = null; Query rw2 = null; IndexReader reader = null; try { IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); for (int i = 0; i < categories.Length; i++) { Document doc = new Document(); doc.Add(new Field("category", categories[i], Field.Store.YES, Field.Index.NOT_ANALYZED)); writer.AddDocument(doc); } writer.Close(); reader = IndexReader.Open(directory); PrefixQuery query = new PrefixQuery(new Term("category", "foo")); rw1 = query.Rewrite(reader); BooleanQuery bq = new BooleanQuery(); bq.Add(query, BooleanClause.Occur.MUST); rw2 = bq.Rewrite(reader); } catch (System.IO.IOException e) { Assert.Fail(e.Message); } Assert.AreEqual(GetCount(reader, rw1), GetCount(reader, rw2), "Number of Clauses Mismatch"); }
public virtual void TestSimple() { int numNodes = TestUtil.NextInt(Random(), 1, 10); double runTimeSec = AtLeast(3); int minDocsToMakeTerms = TestUtil.NextInt(Random(), 5, 20); int maxSearcherAgeSeconds = TestUtil.NextInt(Random(), 1, 3); if (VERBOSE) { Console.WriteLine("TEST: numNodes=" + numNodes + " runTimeSec=" + runTimeSec + " maxSearcherAgeSeconds=" + maxSearcherAgeSeconds); } Start(numNodes, runTimeSec, maxSearcherAgeSeconds); List <PreviousSearchState> priorSearches = new List <PreviousSearchState>(); List <BytesRef> terms = null; while (Time.NanoTime() < endTimeNanos) { bool doFollowon = priorSearches.Count > 0 && Random().Next(7) == 1; // Pick a random node; we will run the query on this node: int myNodeID = Random().Next(numNodes); NodeState.ShardIndexSearcher localShardSearcher; PreviousSearchState prevSearchState; if (doFollowon) { // Pretend user issued a followon query: prevSearchState = priorSearches[Random().Next(priorSearches.Count)]; if (VERBOSE) { Console.WriteLine("\nTEST: follow-on query age=" + ((Time.NanoTime() - prevSearchState.SearchTimeNanos) / 1000000000.0)); } try { localShardSearcher = Nodes[myNodeID].Acquire(prevSearchState.Versions); } catch (SearcherExpiredException see) { // Expected, sometimes; in a "real" app we would // either forward this error to the user ("too // much time has passed; please re-run your // search") or sneakily just switch to newest // searcher w/o telling them... if (VERBOSE) { Console.WriteLine(" searcher expired during local shard searcher init: " + see); } priorSearches.Remove(prevSearchState); continue; } } else { if (VERBOSE) { Console.WriteLine("\nTEST: fresh query"); } // Do fresh query: localShardSearcher = Nodes[myNodeID].Acquire(); prevSearchState = null; } IndexReader[] subs = new IndexReader[numNodes]; PreviousSearchState searchState = null; try { // Mock: now make a single reader (MultiReader) from all node // searchers. In a real shard env you can't do this... we // do it to confirm results from the shard searcher // are correct: int docCount = 0; try { for (int nodeID = 0; nodeID < numNodes; nodeID++) { long subVersion = localShardSearcher.NodeVersions[nodeID]; IndexSearcher sub = Nodes[nodeID].Searchers.Acquire(subVersion); if (sub == null) { nodeID--; while (nodeID >= 0) { subs[nodeID].DecRef(); subs[nodeID] = null; nodeID--; } throw new SearcherExpiredException("nodeID=" + nodeID + " version=" + subVersion); } subs[nodeID] = sub.IndexReader; docCount += subs[nodeID].MaxDoc; } } catch (SearcherExpiredException see) { // Expected if (VERBOSE) { Console.WriteLine(" searcher expired during mock reader init: " + see); } continue; } IndexReader mockReader = new MultiReader(subs); IndexSearcher mockSearcher = new IndexSearcher(mockReader); Query query; Sort sort; if (prevSearchState != null) { query = prevSearchState.Query; sort = prevSearchState.Sort; } else { if (terms == null && docCount > minDocsToMakeTerms) { // TODO: try to "focus" on high freq terms sometimes too // TODO: maybe also periodically reset the terms...? TermsEnum termsEnum = MultiFields.GetTerms(mockReader, "body").GetIterator(null); terms = new List <BytesRef>(); while (termsEnum.Next() != null) { terms.Add(BytesRef.DeepCopyOf(termsEnum.Term)); } if (VERBOSE) { Console.WriteLine("TEST: init terms: " + terms.Count + " terms"); } if (terms.Count == 0) { terms = null; } } if (VERBOSE) { Console.WriteLine(" maxDoc=" + mockReader.MaxDoc); } if (terms != null) { if (Random().NextBoolean()) { query = new TermQuery(new Term("body", terms[Random().Next(terms.Count)])); } else { string t = terms[Random().Next(terms.Count)].Utf8ToString(); string prefix; if (t.Length <= 1) { prefix = t; } else { prefix = t.Substring(0, TestUtil.NextInt(Random(), 1, 2)); } query = new PrefixQuery(new Term("body", prefix)); } if (Random().NextBoolean()) { sort = null; } else { // TODO: sort by more than 1 field int what = Random().Next(3); if (what == 0) { sort = new Sort(SortField.FIELD_SCORE); } else if (what == 1) { // TODO: this sort doesn't merge // correctly... it's tricky because you // could have > 2.1B docs across all shards: //sort = new Sort(SortField.FIELD_DOC); sort = null; } else if (what == 2) { sort = new Sort(new SortField[] { new SortField("docid", SortFieldType.INT32, Random().NextBoolean()) }); } else { sort = new Sort(new SortField[] { new SortField("title", SortFieldType.STRING, Random().NextBoolean()) }); } } } else { query = null; sort = null; } } if (query != null) { try { searchState = AssertSame(mockSearcher, localShardSearcher, query, sort, prevSearchState); } catch (SearcherExpiredException see) { // Expected; in a "real" app we would // either forward this error to the user ("too // much time has passed; please re-run your // search") or sneakily just switch to newest // searcher w/o telling them... if (VERBOSE) { Console.WriteLine(" searcher expired during search: " + see); Console.Out.Write(see.StackTrace); } // We can't do this in general: on a very slow // computer it's possible the local searcher // expires before we can finish our search: // assert prevSearchState != null; if (prevSearchState != null) { priorSearches.Remove(prevSearchState); } } } } finally { Nodes[myNodeID].Release(localShardSearcher); foreach (IndexReader sub in subs) { if (sub != null) { sub.DecRef(); } } } if (searchState != null && searchState.SearchAfterLocal != null && Random().Next(5) == 3) { priorSearches.Add(searchState); if (priorSearches.Count > 200) { Collections.Shuffle(priorSearches); priorSearches.SubList(100, priorSearches.Count).Clear(); } } } Finish(); }
private ParameterizedSql BuildPrefix(PrefixQuery prefixQuery) { string field = prefixQuery.Prefix.Field; if (field != null) { return BuildQuery(new WildcardQuery(new Term(field, prefixQuery.Prefix.Text + "*"))); } return null; }
public virtual void TestPrefixQuery() { Query query = new PrefixQuery(new Term(FIELD, "tang")); Assert.AreEqual(2, Searcher.Search(query, null, 1000).TotalHits, "Number of matched documents"); }
public virtual void TestPrefixQuery() { IndexSearcher indexSearcher = new IndexSearcher(directory, true); Query query = new PrefixQuery(new Term(FIELD, "tang")); Assert.AreEqual(2, indexSearcher.Search(query, null, 1000).TotalHits, "Number of matched documents"); }
/// <summary>Delets an item from the index and any descendants.</summary> /// <param name="itemID">The id of the item to delete.</param> public virtual void Delete(int itemID) { logger.Debug("Deleting item #" + itemID); lock (accessor) { var iw = accessor.GetWriter(); var s = accessor.GetSearcher(); string trail = GetTrail(s, new Term(Properties.ID, itemID.ToString())); if (trail == null) return; // not indexed var query = new PrefixQuery(new Term(Properties.Trail, trail)); iw.DeleteDocuments(query); iw.Commit(); accessor.RecreateSearcher(); } }
public virtual void TestPrefix() { PrefixQuery expected = new PrefixQuery(new Term("field", "foobar")); assertEquals(expected, Parse("foobar*")); }
private static void MultiTermQuery(PrefixQuery query, AzureQueryLogger.IndentedTextWriter writer) { AzureQueryLogger.VisitTerm(query.Prefix, "Prefix Term", writer); }