public virtual void TestSpanNearExact() { SpanTermQuery term1 = new SpanTermQuery(new Term("Field", "seventy")); SpanTermQuery term2 = new SpanTermQuery(new Term("Field", "seven")); SpanNearQuery query = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 0, true); CheckHits(query, new int[] { 77, 177, 277, 377, 477, 577, 677, 777, 877, 977 }); Assert.IsTrue(searcher.Explain(query, 77).GetValue() > 0.0f); Assert.IsTrue(searcher.Explain(query, 977).GetValue() > 0.0f); }
public virtual void TestSpanNearScorerExplain() { SpanNearQuery q = MakeQuery(); Explanation e = Searcher.Explain(q, 1); Assert.IsTrue(0.0f < e.Value, "Scorer explanation value for doc#1 isn't positive: " + e.ToString()); }
public IEnumerable <SampleDataFileRow> _search(string searchTerm, int precision, global::Lucene.Net.Store.Directory indexDirectory) { Debug.Assert(!String.IsNullOrEmpty(searchTerm)); List <SampleDataFileRow> results = new List <SampleDataFileRow>(); if (String.IsNullOrEmpty(searchTerm)) { return(results); } using (IndexSearcher searcher = new IndexSearcher(indexDirectory)) { var analyzer = new StandardAnalyzer(Version.LUCENE_30, ListStopWords); QueryParser parser = new QueryParser(Version.LUCENE_30, "LineText", analyzer); if (precision > 0 && precision < 100) { parser.FuzzyMinSim = ((float)precision) * 0.01f; } else if (precision == 100) { parser.FuzzyMinSim = 0.99f; } else { parser.FuzzyMinSim = 0.8f; } //parser.PhraseSlop = 5; var query = ParseQuery(searchTerm, parser); //var query = fparseQuery(searchTerm); ScoreDoc[] hitsFound = searcher.Search(query, null, CountSearchResults).ScoreDocs; foreach (var t in hitsFound) { var sampleDataFileRow = new SampleDataFileRow(); int docId = t.Doc; float score = t.Score; Explanation explanation = searcher.Explain(query, t.Doc); Document doc = searcher.Doc(docId); sampleDataFileRow.LineNumber = int.Parse(doc.Get("LineNumber")); sampleDataFileRow.LineText = doc.Get("LineText"); sampleDataFileRow.Score = score; _explanationResult = explanation.ToString(); results.Add(sampleDataFileRow); } analyzer.Close(); searcher.Dispose(); } return(results.OrderByDescending(x => x.Score).ToList()); }
/// <summary> /// Test that FieldScoreQuery returns docs in expected order. /// </summary> /// <param name="valueSource"></param> private void DoTestRank(ValueSource valueSource) { FunctionQuery functionQuery = new FunctionQuery(valueSource); IndexReader r = DirectoryReader.Open(dir); IndexSearcher s = NewSearcher(r); Log("test: " + functionQuery); QueryUtils.Check( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, functionQuery, s); ScoreDoc[] h = s.Search(functionQuery, null, 1000).ScoreDocs; assertEquals("All docs should be matched!", N_DOCS, h.Length); string prevID = "ID" + (N_DOCS + 1); // greater than all ids of docs in this test for (int i = 0; i < h.Length; i++) { string resID = s.Doc(h[i].Doc).Get(ID_FIELD); Log(i + ". score=" + h[i].Score + " - " + resID); Log(s.Explain(functionQuery, h[i].Doc)); assertTrue("res id " + resID + " should be < prev res id " + prevID, resID.CompareToOrdinal(prevID) < 0); prevID = resID; } r.Dispose(); }
public virtual void TestBasic() { // create a sort field and sort by it (reverse order) Query query = new TermQuery(new Term("body", "contents")); IndexReader r = searcher.IndexReader; // Just first pass query TopDocs hits = searcher.Search(query, 10); AreEqual(3, hits.TotalHits); AreEqual("3", r.Document(hits.ScoreDocs[0].Doc).Get("id")); AreEqual("1", r.Document(hits.ScoreDocs[1].Doc).Get("id")); AreEqual("2", r.Document(hits.ScoreDocs[2].Doc).Get("id")); // Now, rescore: Expression e = JavascriptCompiler.Compile("sqrt(_score) + ln(popularity)"); SimpleBindings bindings = new SimpleBindings(); bindings.Add(new SortField("popularity", SortField.Type_e.INT)); bindings.Add(new SortField("_score", SortField.Type_e.SCORE)); Rescorer rescorer = e.GetRescorer(bindings); hits = rescorer.Rescore(searcher, hits, 10); AreEqual(3, hits.TotalHits); AreEqual("2", r.Document(hits.ScoreDocs[0].Doc).Get("id")); AreEqual("1", r.Document(hits.ScoreDocs[1].Doc).Get("id")); AreEqual("3", r.Document(hits.ScoreDocs[2].Doc).Get("id")); string expl = rescorer.Explain(searcher, searcher.Explain(query, hits.ScoreDocs[0].Doc), hits.ScoreDocs[0].Doc).ToString(); // Confirm the explanation breaks out the individual // variables: IsTrue(expl.Contains("= variable \"popularity\"")); // Confirm the explanation includes first pass details: IsTrue(expl.Contains("= first pass score")); IsTrue(expl.Contains("body:contents in")); }
/// <summary> /// Performs the explanation. /// </summary> /// <param name="luceneVersion">The lucene version.</param> /// <param name="fsDirectory">The fs directory.</param> /// <param name="searchQuery">The search query.</param> /// <param name="resultId">The result identifier.</param> /// <returns></returns> protected virtual string PerformExplain(Version luceneVersion, FSDirectory fsDirectory, string searchQuery, int resultId) { /* * The obvious problem here is that we're not using the exact same search as the real one. */ var explanation = string.Empty; using (var indexSearcher = new IndexSearcher(fsDirectory, false)) { var analyzer = new StandardAnalyzer(luceneVersion); var queryParser = new MultiFieldQueryParser(luceneVersion, new[] { "Id".ToLowerInvariant() }, analyzer) { DefaultOperator = QueryParser.Operator.AND }; var query = this.searchQueryParser.ParseQuery(searchQuery, queryParser); explanation = indexSearcher.Explain(query, resultId).ToHtml(); analyzer.Close(); } return(explanation); }
private ExplanationResult GetQueryExplanations(ExplanationOptions options, Query luceneQuery, IndexSearcher searcher, ScoreDoc scoreDoc, Document document, global::Lucene.Net.Documents.Document luceneDocument) { string key; var hasGroupKey = options != null && string.IsNullOrWhiteSpace(options.GroupKey) == false; if (_indexType.IsMapReduce()) { if (hasGroupKey) { key = luceneDocument.Get(options.GroupKey, _state); if (key == null && document.Data.TryGet(options.GroupKey, out object value)) { key = value?.ToString(); } } else { key = luceneDocument.Get(Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName, _state); } } else { key = hasGroupKey ? luceneDocument.Get(options.GroupKey, _state) : document.Id; } return(new ExplanationResult { Key = key, Explanation = searcher.Explain(luceneQuery, scoreDoc.Doc, _state) }); }
/// <summary> /// Test that queries based on reverse/ordFieldScore returns docs with expected score. /// </summary> /// <param name="field"></param> /// <param name="inOrder"></param> private void DoTestExactScore(string field, bool inOrder) { IndexReader r = DirectoryReader.Open(dir); IndexSearcher s = NewSearcher(r); ValueSource vs; if (inOrder) { vs = new OrdFieldSource(field); } else { vs = new ReverseOrdFieldSource(field); } Query q = new FunctionQuery(vs); TopDocs td = s.Search(q, null, 1000); assertEquals("All docs should be matched!", N_DOCS, td.TotalHits); ScoreDoc[] sd = td.ScoreDocs; for (int i = 0; i < sd.Length; i++) { float score = sd[i].Score; string id = s.IndexReader.Document(sd[i].Doc).Get(ID_FIELD); Log("-------- " + i + ". Explain doc " + id); Log(s.Explain(q, sd[i].Doc)); float expectedScore = N_DOCS - i - 1; assertEquals("score of result " + i + " shuould be " + expectedScore + " != " + score, expectedScore, score, TEST_SCORE_TOLERANCE_DELTA); string expectedId = inOrder ? Id2String(N_DOCS - i) : Id2String(i + 1); // reverse ==> smaller values first - in-order ==> larger values first assertTrue("id of result " + i + " shuould be " + expectedId + " != " + score, expectedId.Equals(id, StringComparison.Ordinal)); } r.Dispose(); }
/// <summary> /// Checks to see if the hits are what we expected. /// /// LUCENENET specific /// Is non-static because it depends on the non-static variable, <see cref="LuceneTestCase.Similarity"/> /// </summary> /// <param name="query"> the query to execute </param> /// <param name="description"> the description of the search </param> /// <param name="expectedIds"> the expected document ids of the hits </param> /// <param name="expectedScores"> the expected scores of the hits </param> protected internal void AssertHits(IndexSearcher s, Query query, string description, string[] expectedIds, float[] expectedScores) { QueryUtils.Check( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, query, s); const float tolerance = 1e-5f; // Hits hits = searcher.Search(query); // hits normalizes and throws things off if one score is greater than 1.0 TopDocs topdocs = s.Search(query, null, 10000); /* * /// // display the hits System.out.println(hits.Length() + * /// " hits for search: \"" + description + '\"'); for (int i = 0; i < * /// hits.Length(); i++) { System.out.println(" " + FIELD_ID + ':' + * /// hits.Doc(i).Get(FIELD_ID) + " (score:" + hits.Score(i) + ')'); } * /// **** */ // did we get the hits we expected Assert.AreEqual(expectedIds.Length, topdocs.TotalHits); for (int i = 0; i < topdocs.TotalHits; i++) { // System.out.println(i + " exp: " + expectedIds[i]); // System.out.println(i + " field: " + hits.Doc(i).Get(FIELD_ID)); int id = topdocs.ScoreDocs[i].Doc; float score = topdocs.ScoreDocs[i].Score; Document doc = s.Doc(id); Assert.AreEqual(expectedIds[i], doc.Get(FIELD_ID)); bool scoreEq = Math.Abs(expectedScores[i] - score) < tolerance; if (!scoreEq) { Console.WriteLine(i + " warning, expected score: " + expectedScores[i] + ", actual " + score); Console.WriteLine(s.Explain(query, id)); } Assert.AreEqual(expectedScores[i], score, tolerance); Assert.AreEqual(s.Explain(query, id).Value, score, tolerance); } }
public LuceneDotNetHowToExamplesFacade explain(FuzzyQuery query, int docID) { if (IndexSearcher == null) { throw new Exception("IndexParser not created."); } trace("Explaining DocumentID=={0} Query={1}", docID, query); this.Explanation = IndexSearcher.Explain(query, docID); trace(this.Explanation); return(this); }
private static StringBuilder GetSearcResultExplanation(LuceneQuery luceneQuery, IEnumerable <ScoreDoc> scoreDocs, IndexSearcher searcher) { var sb = new StringBuilder(); sb.AppendLine("Query: " + luceneQuery.Query.ToString()); foreach (var match in scoreDocs) { var explanation = searcher.Explain(luceneQuery.Query, match.Doc); sb.AppendLine("-------------------"); var doc = searcher.Doc(match.Doc); sb.AppendLine(doc.Get(Constants.TitleTag)); sb.AppendLine(explanation.ToString()); } return(sb); }
public virtual void TestNegativeQueryBoost() { Query q = new TermQuery(new Term("foo", "bar")); q.Boost = -42f; Assert.AreEqual(-42f, q.Boost, 0.0f); Store.Directory directory = NewDirectory(); try { Analyzer analyzer = new MockAnalyzer(Random); IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); IndexWriter writer = new IndexWriter(directory, conf); try { Documents.Document d = new Documents.Document(); d.Add(NewTextField("foo", "bar", Field.Store.YES)); writer.AddDocument(d); } finally { writer.Dispose(); } IndexReader reader = DirectoryReader.Open(directory); try { IndexSearcher searcher = NewSearcher(reader); ScoreDoc[] hits = searcher.Search(q, null, 1000).ScoreDocs; Assert.AreEqual(1, hits.Length); Assert.IsTrue(hits[0].Score < 0, "score is not negative: " + hits[0].Score); Explanation explain = searcher.Explain(q, hits[0].Doc); Assert.AreEqual(hits[0].Score, explain.Value, 0.001f, "score doesn't match explanation"); Assert.IsTrue(explain.IsMatch, "explain doesn't think doc is a match"); } finally { reader.Dispose(); } } finally { directory.Dispose(); } }
private static string AddExplanation(IndexSearcher searcher, string data, Query query, ScoreDoc scoreDoc, IDictionary <string, int> rankings) { Explanation explanation = searcher.Explain(query, scoreDoc.Doc); JObject diagnostics = new JObject(); int rankVal; string id = searcher.Doc(scoreDoc.Doc).Get("Id"); if (rankings.TryGetValue(id, out rankVal)) { float rankingScore = RankingScoreQuery.GetRankingScore(rankings, id); diagnostics.Add("Rank", rankVal); diagnostics.Add("RankScore", rankingScore); diagnostics.Add("LuceneScore", scoreDoc.Score / rankingScore); } diagnostics.Add("Score", scoreDoc.Score.ToString()); diagnostics.Add("Explanation", explanation.ToString()); diagnostics.Add("IdTerms", GetTerms(searcher, scoreDoc.Doc, "Id")); diagnostics.Add("TokenizedIdTerms", GetTerms(searcher, scoreDoc.Doc, "TokenizedId")); diagnostics.Add("ShingledIdTerms", GetTerms(searcher, scoreDoc.Doc, "ShingledId")); diagnostics.Add("VersionTerms", GetTerms(searcher, scoreDoc.Doc, "Version")); diagnostics.Add("TitleTerms", GetTerms(searcher, scoreDoc.Doc, "Title")); diagnostics.Add("TagsTerms", GetTerms(searcher, scoreDoc.Doc, "Tags")); diagnostics.Add("DescriptionTerms", GetTerms(searcher, scoreDoc.Doc, "Description")); diagnostics.Add("AuthorsTerms", GetTerms(searcher, scoreDoc.Doc, "Authors")); diagnostics.Add("OwnersTerms", GetTerms(searcher, scoreDoc.Doc, "Owners")); diagnostics.Add("PublishedDate", GetInt(searcher, scoreDoc.Doc, "PublishedDate")); diagnostics.Add("EditedDate", GetInt(searcher, scoreDoc.Doc, "EditedDate")); diagnostics.Add("CuratedFeed", GetMultiValue(searcher, scoreDoc.Doc, "CuratedFeed")); diagnostics.Add("Key", GetInt(searcher, scoreDoc.Doc, "Key")); diagnostics.Add("Checksum", GetInt(searcher, scoreDoc.Doc, "Checksum")); diagnostics.Add("ProjectGuidRankings", GetProjectGuidRankings(searcher, scoreDoc.Doc)); JObject obj = JObject.Parse(data); obj.Add("diagnostics", diagnostics); data = obj.ToString(); return(data); }
/// <summary> /// Test that queries based on reverse/ordFieldScore scores correctly /// </summary> /// <param name="field"></param> /// <param name="inOrder"></param> private void DoTestRank(string field, bool inOrder) { IndexReader r = DirectoryReader.Open(dir); IndexSearcher s = NewSearcher(r); ValueSource vs; if (inOrder) { vs = new OrdFieldSource(field); } else { vs = new ReverseOrdFieldSource(field); } Query q = new FunctionQuery(vs); Log("test: " + q); QueryUtils.Check( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, q, s); ScoreDoc[] h = s.Search(q, null, 1000).ScoreDocs; assertEquals("All docs should be matched!", N_DOCS, h.Length); string prevID = inOrder ? "IE" // greater than all ids of docs in this test ("ID0001", etc.) : "IC"; // smaller than all ids of docs in this test ("ID0001", etc.) for (int i = 0; i < h.Length; i++) { string resID = s.Doc(h[i].Doc).Get(ID_FIELD); Log(i + ". score=" + h[i].Score + " - " + resID); Log(s.Explain(q, h[i].Doc)); if (inOrder) { assertTrue("res id " + resID + " should be < prev res id " + prevID, resID.CompareToOrdinal(prevID) < 0); } else { assertTrue("res id " + resID + " should be > prev res id " + prevID, resID.CompareToOrdinal(prevID) > 0); } prevID = resID; } r.Dispose(); }
/// <summary> /// Searches the index for the querytext /// </summary> /// <param name="querytext">The text to search the index</param> //public string SearchIndext(string querytext) public List <Dictionary <string, string> > SearchIndext(string querytext) { List <Dictionary <string, string> > resultListDict = new List <Dictionary <string, string> >(); // Initiate a result list Query query = DisplayQueries(querytext); Console.WriteLine("query is " + query); TopDocs results = searcher.Search(query, 100); System.Console.WriteLine("Number of results is " + results.TotalHits); int rank = 0; // ScoreDocs : a array stores pointers of a query // scoreDoc : a pointer of a query points to doc_ID and score (of the doc for the query) //string output = ""; if (results.TotalHits != 0) // Check if there are results { foreach (ScoreDoc scoreDoc in results.ScoreDocs) { rank++; Lucene.Net.Documents.Document doc = searcher.Doc(scoreDoc.Doc); string myFieldValue = doc.Get(TEXT_FN_PASS_TEXT); string myURL = doc.Get(TEXT_FN_URL); string passId = doc.Get(TEXT_FN_PASS_ID); string score = scoreDoc.Score.ToString(); string queryId = doc.Get(TEXT_FN_QUERY_ID); Explanation e = searcher.Explain(query, scoreDoc.Doc); char delimiters = '/'; string[] urlSeg = myURL.Split(delimiters); resultListDict.Add(new Dictionary <string, string> { { "rank", rank.ToString() }, { "passId", passId }, { "score", score }, { "title", urlSeg[2] }, { "url", myURL }, { "text", myFieldValue }, { "queryId", queryId } }); //Console.WriteLine("Rank " + rank + " text " + myFieldValue + " URL " + myURL); //Console.WriteLine(e); } } return(resultListDict); }
public static JToken MakeResultData(IndexSearcher searcher, string scheme, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query) { Uri registrationBaseAddress = searcherManager.RegistrationBaseAddress[scheme]; JArray array = new JArray(); for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++) { ScoreDoc scoreDoc = topDocs.ScoreDocs[i]; Document document = searcher.Doc(scoreDoc.Doc); string url = document.Get("Url"); string id = document.Get("Id"); string version = document.Get("Version"); JObject obj = new JObject(); obj["@id"] = new Uri(registrationBaseAddress, url).AbsoluteUri; obj["@type"] = "Package"; obj["registration"] = new Uri(registrationBaseAddress, string.Format("{0}/index.json", id.ToLowerInvariant())).AbsoluteUri; obj["id"] = id; AddField(obj, document, "domain", "Domain"); AddField(obj, document, "description", "Description"); AddField(obj, document, "summary", "Summary"); AddField(obj, document, "title", "Title"); AddField(obj, document, "iconUrl", "IconUrl"); AddFieldAsArray(obj, document, "tags", "Tags"); AddFieldAsArray(obj, document, "authors", "Authors"); obj["version"] = version; obj["versions"] = searcherManager.GetVersions(scheme, scoreDoc.Doc); if (includeExplanation) { Explanation explanation = searcher.Explain(query, scoreDoc.Doc); obj["explanation"] = explanation.ToString(); } array.Add(obj); } return(array); }
private ExplanationResult GetQueryExplanations(ExplanationOptions options, Query luceneQuery, IndexSearcher searcher, ScoreDoc scoreDoc, Document document, global::Lucene.Net.Documents.Document luceneDocument) { string key; if (options != null && string.IsNullOrWhiteSpace(options.GroupKey) == false) { key = luceneDocument.Get(options.GroupKey, _state); } else { key = document.Id; } return(new ExplanationResult { Key = key, Explanation = searcher.Explain(luceneQuery, scoreDoc.Doc, _state) }); }
// Test that FieldScoreQuery returns docs with expected score. //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: private void doTestExactScore(ValueSource valueSource) throws Exception private void doTestExactScore(ValueSource valueSource) { FunctionQuery functionQuery = new FunctionQuery(valueSource); IndexReader r = DirectoryReader.Open(dir); IndexSearcher s = NewSearcher(r); TopDocs td = s.Search(functionQuery, null, 1000); assertEquals("All docs should be matched!", N_DOCS, td.TotalHits); ScoreDoc[] sd = td.ScoreDocs; foreach (ScoreDoc aSd in sd) { float score = aSd.Score; Log(s.Explain(functionQuery, aSd.Doc)); string id = s.IndexReader.Document(aSd.Doc).Get(ID_FIELD); float expectedScore = ExpectedFieldScore(id); // "ID7" --> 7.0 assertEquals("score of " + id + " shuould be " + expectedScore + " != " + score, expectedScore, score, TEST_SCORE_TOLERANCE_DELTA); } r.Dispose(); }
/// <summary> /// Searches the index for the querytext /// </summary> /// <param name="querytext">The text to search the index</param> public void SearchText(string querytext) { System.Console.WriteLine("Searching for " + querytext); querytext = querytext.ToLower(); Query query = parser.Parse(querytext); TopDocs results = searcher.Search(query, 100); System.Console.WriteLine("Number of results is " + results.TotalHits); int rank = 0; foreach (ScoreDoc scoreDoc in results.ScoreDocs) { rank++; Lucene.Net.Documents.Document doc = searcher.Doc(scoreDoc.Doc); string myFieldValue = doc.Get(TEXT_FN).ToString(); Console.WriteLine("Rank " + rank + " Score" + scoreDoc.Score + " text " + myFieldValue); Explanation ex = searcher.Explain(query, scoreDoc.Doc); Console.WriteLine(ex.ToString()); } }
public override void Collect(int doc) { Explanation exp = null; doc = doc + @base; try { exp = s.Explain(q, doc); } catch (IOException e) { throw new Exception ("exception in hitcollector of [[" + d + "]] for #" + doc, e); } assertNotNull("Explanation of [[" + d + "]] for #" + doc + " is null", exp); verifyExplanation(d, doc, scorer.Score(), deep, exp); assertTrue("Explanation of [[" + d + "]] for #" + doc + " does not indicate match: " + exp.ToString(), exp.IsMatch); }
private void btnExplain_Click(object sender, System.EventArgs e) { if (listSearch.SelectedItems.Count == 0) { return; } if (searchedDocIds == null || searchedDocIds.Length < listSearch.Items.Count) { return; } if (_luke.IndexReader == null) { _luke.ShowStatus(_luke.resources.GetString("NoIndex")); return; } if (query == null) { return; } IndexSearcher searcher = null; try { searcher = new IndexSearcher(_luke.Directory, true); Lucene.Net.Search.Explanation expl = searcher.Explain(query, searchedDocIds[listSearch.SelectedIndices[0]]); ExplanationDialog explDialog = new ExplanationDialog(expl); explDialog.ShowDialog(this); } catch (Exception exc) { _luke.ErrorMessage(exc.Message); } finally { searcher.Close(); } }
static void Main(string[] args) { RAMDirectory dir = new RAMDirectory(); IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer()); AddDocument(writer, "John Doe", "NY"); AddDocument(writer, "John Foo", "New Jersey"); AddDocument(writer, "XYZ", "NY"); writer.Commit(); BooleanQuery query = new BooleanQuery(); query.Add(new TermQuery(new Term("Name", "john")), BooleanClause.Occur.SHOULD); query.Add(new TermQuery(new Term("Location", "NY")), BooleanClause.Occur.SHOULD); IndexReader reader = writer.GetReader(); IndexSearcher searcher = new IndexSearcher(reader); var hits = searcher.Search(query, null, 10); for (int i = 0; i < hits.totalHits; i++) { Document doc = searcher.Doc(hits.scoreDocs[i].doc); var explain = searcher.Explain(query, hits.scoreDocs[i].doc); Console.WriteLine("{0} - {1} - {2}", hits.scoreDocs[i].score, doc.ToString(), explain.ToString()); } }
public static JToken AutoCompleteMakeResult(IndexSearcher searcher, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query) { JArray array = new JArray(); for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++) { ScoreDoc scoreDoc = topDocs.ScoreDocs[i]; Document document = searcher.Doc(scoreDoc.Doc); string id = document.Get("Id"); array.Add(id); } JObject result = new JObject(); result.Add("@context", new JObject { { "@vocab", "http://schema.nuget.org/schema#" } }); result.Add("totalHits", topDocs.TotalHits); result.Add("indexName", searcherManager.IndexName); result.Add("data", array); if (includeExplanation) { JArray explanations = new JArray(); for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++) { ScoreDoc scoreDoc = topDocs.ScoreDocs[i]; Explanation explanation = searcher.Explain(query, scoreDoc.Doc); explanations.Add(explanation.ToString()); } result.Add("explanations", explanations); } return(result); }
/// <summary> /// Searches the index for the querytext /// </summary> /// <param name="querytext">The text to search the index</param> //public string SearchIndext(string querytext) public List <Dictionary <string, string> > SearchIndext(string querytext) { List <Dictionary <string, string> > resultListDict = new List <Dictionary <string, string> >(); // Initiate a result list Query query = DisplayQueries(querytext); Console.WriteLine("query is " + query); TopDocs results = searcher.Search(query, 100); System.Console.WriteLine("Number of results is " + results.TotalHits); // Setup the configuration of Highlighter IFormatter formatter = new SimpleHTMLFormatter("<span style=\"font-weight:bold; background-color:yellow;\">", "</span>"); SimpleFragmenter fragmenter = new SimpleFragmenter(2000); QueryScorer scorer = new QueryScorer(query); Highlighter highlighter = new Highlighter(formatter, scorer); highlighter.TextFragmenter = fragmenter; int rank = 0; // ScoreDocs : a array stores pointers of a query // scoreDoc : a pointer of a query points to doc_ID and score (of the doc for the query) //string output = ""; if (results.TotalHits != 0) // Check if there are results { foreach (ScoreDoc scoreDoc in results.ScoreDocs) { rank++; Lucene.Net.Documents.Document doc = searcher.Doc(scoreDoc.Doc); string myFieldValue = doc.Get(TEXT_FN_PASS_TEXT); string myURL = doc.Get(TEXT_FN_URL); string passId = doc.Get(TEXT_FN_PASS_ID); string score = scoreDoc.Score.ToString(); string queryId = doc.Get(TEXT_FN_QUERY_ID); int jsonId = Int32.Parse(doc.Get(TEXT_FN_JSON_ARRAY_ID)); // passage_text field store as Field.Store.NO foreach (var itemP in jArr[jsonId][PASSAGES]) { if (itemP[TEXT_FN_PASS_ID].ToString() == passId) { myFieldValue = itemP[TEXT_FN_PASS_TEXT].ToString(); } } //Add the Highlighter tag into passage_text of query //TokenStream HLstream = analyzer.TokenStream("", new StringReader(doc.Get(TEXT_FN_PASS_TEXT))); //string HLmyFieldValue = highlighter.GetBestFragment(HLstream, doc.Get(TEXT_FN_PASS_TEXT)); TokenStream HLstream = analyzer.TokenStream("", new StringReader(myFieldValue)); string HLmyFieldValue = highlighter.GetBestFragment(HLstream, myFieldValue); Explanation e = searcher.Explain(query, scoreDoc.Doc); //Extract title from URL char delimiters = '/'; string[] urlSeg = myURL.Split(delimiters); string title; if (urlSeg[urlSeg.Length - 1].Length == 0) { title = urlSeg[urlSeg.Length - 2]; } else { title = urlSeg[urlSeg.Length - 1]; } resultListDict.Add(new Dictionary <string, string> { { "rank", rank.ToString() }, { "passId", passId }, { "score", score }, { "title", title }, { "url", myURL }, { "text", myFieldValue }, { "queryId", queryId }, { "highlighter", HLmyFieldValue } }); //Console.WriteLine("Rank " + rank + " text " + myFieldValue + " URL " + myURL); //Console.WriteLine(e); } } return(resultListDict); }
public virtual void TestSpanNearExact() { SpanTermQuery term1 = new SpanTermQuery(new Term("field", "seventy")); SpanTermQuery term2 = new SpanTermQuery(new Term("field", "seven")); SpanNearQuery query = new SpanNearQuery(new SpanQuery[] { term1, term2 }, 0, true); CheckHits(query, new int[] { 77, 177, 277, 377, 477, 577, 677, 777, 877, 977, 1077, 1177, 1277, 1377, 1477, 1577, 1677, 1777, 1877, 1977 }); Assert.IsTrue(Searcher.Explain(query, 77).Value > 0.0f); Assert.IsTrue(Searcher.Explain(query, 977).Value > 0.0f); QueryUtils.Check(term1); QueryUtils.Check(term2); QueryUtils.CheckUnequal(term1, term2); }
public static JToken MakeResultData(IndexSearcher searcher, string currentOwner, string scheme, TopDocs topDocs, int skip, int take, SecureSearcherManager searcherManager, bool includeExplanation, Query query) { Uri registrationBaseAddress = searcherManager.RegistrationBaseAddress[scheme]; JArray array = new JArray(); for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++) { ScoreDoc scoreDoc = topDocs.ScoreDocs[i]; Document document = searcher.Doc(scoreDoc.Doc); string url = document.Get("Url"); string id = document.Get("Id"); string version = document.Get("Version"); string owner = document.Get("Owner"); string ns = document.Get("Namespace"); if (ns != null) { id = string.Format("{0}.{1}", ns, id); } JObject obj = new JObject(); obj["@id"] = new Uri(registrationBaseAddress, url).AbsoluteUri; obj["@type"] = document.Get("@type"); obj["registration"] = new Uri(registrationBaseAddress, string.Format("{0}/index.json", id.ToLowerInvariant())).AbsoluteUri; obj["id"] = id; obj["isOwner"] = (owner == currentOwner); ServiceHelpers.AddField(obj, document, "packageContent", "PackageContent"); ServiceHelpers.AddField(obj, document, "catalogEntry", "CatalogEntry"); ServiceHelpers.AddFieldBool(obj, document, "listed", "Listed"); ServiceHelpers.AddField(obj, document, "tenantId", "TenantId"); ServiceHelpers.AddField(obj, document, "namespace", "Namespace"); ServiceHelpers.AddField(obj, document, "visibility", "Visibility"); ServiceHelpers.AddField(obj, document, "description", "Description"); ServiceHelpers.AddField(obj, document, "summary", "Summary"); ServiceHelpers.AddField(obj, document, "title", "Title"); ServiceHelpers.AddField(obj, document, "iconUrl", "IconUrl"); ServiceHelpers.AddFieldAsObject(obj, document, "owner", "OwnerDetails"); ServiceHelpers.AddFieldAsArray(obj, document, "tags", "Tags"); ServiceHelpers.AddFieldAsArray(obj, document, "authors", "Authors"); obj["version"] = version; obj["versions"] = searcherManager.GetVersions(scheme, scoreDoc.Doc); if (includeExplanation) { Explanation explanation = searcher.Explain(query, scoreDoc.Doc); obj["explanation"] = explanation.ToString(); } array.Add(obj); } return(array); }
private void LogResult(string msg, IndexSearcher s, Query q, int doc, float?score1) { Log(msg + " " + score1); Log("Explain by: " + q); Log(s.Explain(q, doc)); }
private void ExecuteRandomJoin(bool multipleValuesPerDocument, int maxIndexIter, int maxSearchIter, int numberOfDocumentsToIndex) { for (int indexIter = 1; indexIter <= maxIndexIter; indexIter++) { if (Verbose) { Console.WriteLine("indexIter=" + indexIter); } Directory dir = NewDirectory(); RandomIndexWriter w = new RandomIndexWriter(Random, dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random, MockTokenizer.KEYWORD, false)) .SetMergePolicy(NewLogMergePolicy())); bool scoreDocsInOrder = TestJoinUtil.Random.NextBoolean(); IndexIterationContext context = CreateContext(numberOfDocumentsToIndex, w, multipleValuesPerDocument, scoreDocsInOrder); IndexReader topLevelReader = w.GetReader(); w.Dispose(); for (int searchIter = 1; searchIter <= maxSearchIter; searchIter++) { if (Verbose) { Console.WriteLine("searchIter=" + searchIter); } IndexSearcher indexSearcher = NewSearcher(topLevelReader); int r = Random.Next(context.RandomUniqueValues.Length); bool from = context.RandomFrom[r]; string randomValue = context.RandomUniqueValues[r]; FixedBitSet expectedResult = CreateExpectedResult(randomValue, from, indexSearcher.IndexReader, context); Query actualQuery = new TermQuery(new Term("value", randomValue)); if (Verbose) { Console.WriteLine("actualQuery=" + actualQuery); } var scoreModeLength = Enum.GetNames(typeof(ScoreMode)).Length; ScoreMode scoreMode = (ScoreMode)Random.Next(scoreModeLength); if (Verbose) { Console.WriteLine("scoreMode=" + scoreMode); } Query joinQuery; if (from) { joinQuery = JoinUtil.CreateJoinQuery("from", multipleValuesPerDocument, "to", actualQuery, indexSearcher, scoreMode); } else { joinQuery = JoinUtil.CreateJoinQuery("to", multipleValuesPerDocument, "from", actualQuery, indexSearcher, scoreMode); } if (Verbose) { Console.WriteLine("joinQuery=" + joinQuery); } // Need to know all documents that have matches. TopDocs doesn't give me that and then I'd be also testing TopDocsCollector... FixedBitSet actualResult = new FixedBitSet(indexSearcher.IndexReader.MaxDoc); TopScoreDocCollector topScoreDocCollector = TopScoreDocCollector.Create(10, false); indexSearcher.Search(joinQuery, new CollectorAnonymousClass2(scoreDocsInOrder, actualResult, topScoreDocCollector)); // Asserting bit set... if (Verbose) { Console.WriteLine("expected cardinality:" + expectedResult.Cardinality); DocIdSetIterator iterator = expectedResult.GetIterator(); for (int doc = iterator.NextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.NextDoc()) { Console.WriteLine(string.Format("Expected doc[{0}] with id value {1}", doc, indexSearcher.Doc(doc).Get("id"))); } Console.WriteLine("actual cardinality:" + actualResult.Cardinality); iterator = actualResult.GetIterator(); for (int doc = iterator.NextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.NextDoc()) { Console.WriteLine(string.Format("Actual doc[{0}] with id value {1}", doc, indexSearcher.Doc(doc).Get("id"))); } } assertEquals(expectedResult, actualResult); // Asserting TopDocs... TopDocs expectedTopDocs = CreateExpectedTopDocs(randomValue, from, scoreMode, context); TopDocs actualTopDocs = topScoreDocCollector.GetTopDocs(); assertEquals(expectedTopDocs.TotalHits, actualTopDocs.TotalHits); assertEquals(expectedTopDocs.ScoreDocs.Length, actualTopDocs.ScoreDocs.Length); if (scoreMode == ScoreMode.None) { continue; } assertEquals(expectedTopDocs.MaxScore, actualTopDocs.MaxScore, 0.0f); for (int i = 0; i < expectedTopDocs.ScoreDocs.Length; i++) { if (Verbose) { Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "Expected doc: {0} | Actual doc: {1}\n", expectedTopDocs.ScoreDocs[i].Doc, actualTopDocs.ScoreDocs[i].Doc)); Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "Expected score: {0} | Actual score: {1}\n", expectedTopDocs.ScoreDocs[i].Score, actualTopDocs.ScoreDocs[i].Score)); } assertEquals(expectedTopDocs.ScoreDocs[i].Doc, actualTopDocs.ScoreDocs[i].Doc); assertEquals(expectedTopDocs.ScoreDocs[i].Score, actualTopDocs.ScoreDocs[i].Score, 0.0f); Explanation explanation = indexSearcher.Explain(joinQuery, expectedTopDocs.ScoreDocs[i].Doc); assertEquals(expectedTopDocs.ScoreDocs[i].Score, explanation.Value, 0.0f); } } topLevelReader.Dispose(); dir.Dispose(); } }
public static string Explain(IndexSearcher searcher, Query query, ScoreDoc match) { Explanation explanation = searcher.Explain(query, match.Doc); return(explanation.ToString()); }