private IQueryable<Package> SearchCore(SearchFilter searchFilter, out int totalHits)
        {
            int numRecords = searchFilter.Skip + searchFilter.Take;

            var searcher = new IndexSearcher(_directory, readOnly: true);
            var query = ParseQuery(searchFilter);

            // IF searching by relevance, boost scores by download count.
            if (searchFilter.SortProperty == SortProperty.Relevance)
            {
                var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
                query = new CustomScoreQuery(query, downloadCountBooster);
            }

            var filterTerm = searchFilter.IncludePrerelease ? "IsLatest" : "IsLatestStable";
            var termQuery = new TermQuery(new Term(filterTerm, Boolean.TrueString));
            var filter = new QueryWrapperFilter(termQuery);
            var results = searcher.Search(query, filter: filter, n: numRecords, sort: new Sort(GetSortField(searchFilter)));
            totalHits = results.totalHits;

            if (results.totalHits == 0 || searchFilter.CountOnly)
            {
                return Enumerable.Empty<Package>().AsQueryable();
            }

            var packages = results.scoreDocs
                                  .Skip(searchFilter.Skip)
                                  .Select(sd => PackageFromDoc(searcher.Doc(sd.doc)))
                                  .ToList();
            return packages.AsQueryable();
        }
        // Test that FieldScoreQuery returns docs with expected score.
        private void  DoTestCustomScore(System.String field, FieldScoreQuery.Type tp, double dboost)
        {
            float           boost   = (float)dboost;
            IndexSearcher   s       = new IndexSearcher(dir, true);
            FieldScoreQuery qValSrc = new FieldScoreQuery(field, tp); // a query that would score by the field
            QueryParser     qp      = new QueryParser(Util.Version.LUCENE_CURRENT, TEXT_FIELD, anlzr);

            System.String qtxt = "first aid text"; // from the doc texts in FunctionQuerySetup.

            // regular (boolean) query.
            Query q1 = qp.Parse(qtxt);

            Log(q1);

            // custom query, that should score the same as q1.
            CustomScoreQuery q2CustomNeutral = new CustomScoreQuery(q1);

            q2CustomNeutral.Boost = boost;
            Log(q2CustomNeutral);

            // custom query, that should (by default) multiply the scores of q1 by that of the field
            CustomScoreQuery q3CustomMul = new CustomScoreQuery(q1, qValSrc);

            q3CustomMul.SetStrict(true);
            q3CustomMul.Boost = boost;
            Log(q3CustomMul);

            // custom query, that should add the scores of q1 to that of the field
            CustomScoreQuery q4CustomAdd = new CustomAddQuery(q1, qValSrc);

            q4CustomAdd.SetStrict(true);
            q4CustomAdd.Boost = boost;
            Log(q4CustomAdd);

            // custom query, that multiplies and adds the field score to that of q1
            CustomScoreQuery q5CustomMulAdd = new CustomMulAddQuery(q1, qValSrc, qValSrc);

            q5CustomMulAdd.SetStrict(true);
            q5CustomMulAdd.Boost = boost;
            Log(q5CustomMulAdd);

            // do al the searches
            TopDocs td1 = s.Search(q1, null, 1000);
            TopDocs td2CustomNeutral = s.Search(q2CustomNeutral, null, 1000);
            TopDocs td3CustomMul     = s.Search(q3CustomMul, null, 1000);
            TopDocs td4CustomAdd     = s.Search(q4CustomAdd, null, 1000);
            TopDocs td5CustomMulAdd  = s.Search(q5CustomMulAdd, null, 1000);

            // put results in map so we can verify the scores although they have changed
            System.Collections.Hashtable h1 = TopDocsToMap(td1);
            System.Collections.Hashtable h2CustomNeutral = TopDocsToMap(td2CustomNeutral);
            System.Collections.Hashtable h3CustomMul     = TopDocsToMap(td3CustomMul);
            System.Collections.Hashtable h4CustomAdd     = TopDocsToMap(td4CustomAdd);
            System.Collections.Hashtable h5CustomMulAdd  = TopDocsToMap(td5CustomMulAdd);

            VerifyResults(boost, s, h1, h2CustomNeutral, h3CustomMul, h4CustomAdd, h5CustomMulAdd, q1, q2CustomNeutral, q3CustomMul, q4CustomAdd, q5CustomMulAdd);
        }
Пример #3
0
        // Test that FieldScoreQuery returns docs with expected score.
        private void  DoTestExactScore(System.String field, FieldScoreQuery.Type tp)
        {
            IndexSearcher s  = new IndexSearcher(dir, true);
            Query         q  = new FieldScoreQuery(field, tp);
            TopDocs       td = s.Search(q, null, 1000);

            Assert.AreEqual(N_DOCS, td.TotalHits, "All docs should be matched!");
            ScoreDoc[] sd = td.ScoreDocs;
            for (int i = 0; i < sd.Length; i++)
            {
                float score = sd[i].Score;
                Log(s.Explain(q, sd[i].Doc));
                System.String id            = s.IndexReader.Document(sd[i].Doc).Get(ID_FIELD);
                float         expectedScore = ExpectedFieldScore(id);         // "ID7" --> 7.0
                Assert.AreEqual(expectedScore, score, TEST_SCORE_TOLERANCE_DELTA, "score of " + id + " shuould be " + expectedScore + " != " + score);
            }
        }
        private SearchResults SearchCore(SearchFilter searchFilter)
        {
            // Get index timestamp
            DateTime timestamp = File.GetLastWriteTimeUtc(LuceneCommon.GetIndexMetadataPath());

            int numRecords = searchFilter.Skip + searchFilter.Take;

            var searcher = new IndexSearcher(_directory, readOnly: true);
            var query = ParseQuery(searchFilter);

            // IF searching by relevance, boost scores by download count.
            if (searchFilter.SortOrder == SortOrder.Relevance)
            {
                var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
                query = new CustomScoreQuery(query, downloadCountBooster);
            }

            var filterTerm = searchFilter.IncludePrerelease ? "IsLatest" : "IsLatestStable";
            Query filterQuery = new TermQuery(new Term(filterTerm, Boolean.TrueString));
            if (searchFilter.CuratedFeed != null)
            {
                var feedFilterQuery = new TermQuery(new Term("CuratedFeedKey", searchFilter.CuratedFeed.Key.ToString(CultureInfo.InvariantCulture)));
                BooleanQuery conjunctionQuery = new BooleanQuery();
                conjunctionQuery.Add(filterQuery, Occur.MUST);
                conjunctionQuery.Add(feedFilterQuery, Occur.MUST);
                filterQuery = conjunctionQuery;
            }

            Filter filter = new QueryWrapperFilter(filterQuery);
            var results = searcher.Search(query, filter: filter, n: numRecords, sort: new Sort(GetSortField(searchFilter)));
            
            if (results.TotalHits == 0 || searchFilter.CountOnly)
            {
                return new SearchResults(results.TotalHits, timestamp);
            }

            var packages = results.ScoreDocs
                                  .Skip(searchFilter.Skip)
                                  .Select(sd => PackageFromDoc(searcher.Doc(sd.Doc)))
                                  .ToList();
            return new SearchResults(
                results.TotalHits,
                timestamp,
                packages.AsQueryable());
        }
Пример #5
0
        // Test that FieldScoreQuery returns docs in expected order.
        private void  DoTestRank(System.String field, FieldScoreQuery.Type tp)
        {
            IndexSearcher s = new IndexSearcher(dir, true);
            Query         q = new FieldScoreQuery(field, tp);

            Log("test: " + q);
            QueryUtils.Check(q, s);
            ScoreDoc[] h = s.Search(q, null, 1000).ScoreDocs;
            Assert.AreEqual(N_DOCS, h.Length, "All docs should be matched!");
            System.String prevID = "ID" + (N_DOCS + 1);             // greater than all ids of docs in this test
            for (int i = 0; i < h.Length; i++)
            {
                System.String resID = s.Doc(h[i].Doc).Get(ID_FIELD);
                Log(i + ".   score=" + h[i].Score + "  -  " + resID);
                Log(s.Explain(q, h[i].Doc));
                Assert.IsTrue(String.CompareOrdinal(resID, prevID) < 0, "res id " + resID + " should be < prev res id " + prevID);
                prevID = resID;
            }
        }
Пример #6
0
        private IQueryable<Package> SearchCore(SearchFilter searchFilter, out int totalHits)
        {
            int numRecords = searchFilter.Skip + searchFilter.Take;

            var searcher = new IndexSearcher(_directory, readOnly: true);
            var query = ParseQuery(searchFilter);

            // IF searching by relevance, boost scores by download count.
            if (searchFilter.SortProperty == SortProperty.Relevance)
            {
                var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
                query = new CustomScoreQuery(query, downloadCountBooster);
            }

            var filterTerm = searchFilter.IncludePrerelease ? "IsLatest" : "IsLatestStable";
            Query filterQuery = new TermQuery(new Term(filterTerm, Boolean.TrueString));
            if (searchFilter.CuratedFeedKey.HasValue)
            {
                var feedFilterQuery = new TermQuery(new Term("CuratedFeedKey", searchFilter.CuratedFeedKey.Value.ToString(CultureInfo.InvariantCulture)));
                BooleanQuery conjunctionQuery = new BooleanQuery();
                conjunctionQuery.Add(filterQuery, Occur.MUST);
                conjunctionQuery.Add(feedFilterQuery, Occur.MUST);
                filterQuery = conjunctionQuery;
            }

            Filter filter = new QueryWrapperFilter(filterQuery);
            var results = searcher.Search(query, filter: filter, n: numRecords, sort: new Sort(GetSortField(searchFilter)));
            totalHits = results.TotalHits;

            if (results.TotalHits == 0 || searchFilter.CountOnly)
            {
                return Enumerable.Empty<Package>().AsQueryable();
            }

            var packages = results.ScoreDocs
                                  .Skip(searchFilter.Skip)
                                  .Select(sd => PackageFromDoc(searcher.Doc(sd.Doc)))
                                  .ToList();
            return packages.AsQueryable();
        }
        private SearchResults SearchCore(SearchFilter searchFilter)
        {
            // Get index timestamp
            DateTime timestamp = File.GetLastWriteTimeUtc(LuceneCommon.IndexMetadataPath);

            int numRecords = searchFilter.Skip + searchFilter.Take;

            var searcher = new IndexSearcher(_directory, readOnly: true);
            var query = ParseQuery(searchFilter);

            // If searching by relevance, boost scores by download count.
            if (searchFilter.SortProperty == SortProperty.Relevance)
            {
                var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
                query = new CustomScoreQuery(query, downloadCountBooster);
            }

            var filterTerm = searchFilter.IncludePrerelease ? "IsLatest" : "IsLatestStable";
            Query filterQuery = new TermQuery(new Term(filterTerm, Boolean.TrueString));

            Filter filter = new QueryWrapperFilter(filterQuery);
            var results = searcher.Search(query, filter: filter, n: numRecords, sort: new Sort(GetSortField(searchFilter)));

            if (results.TotalHits == 0 || searchFilter.CountOnly)
            {
                return new SearchResults(results.TotalHits, timestamp);
            }

            var packages = results.ScoreDocs
                                  .Skip(searchFilter.Skip)
                                  .Select(sd => PackageFromDoc(searcher.Doc(sd.Doc)))
                                  .ToList();

            return new SearchResults(
                results.TotalHits,
                timestamp,
                packages.AsQueryable());
        }
		// Test that values loaded for FieldScoreQuery are cached properly and consumes the proper RAM resources.
		private void  DoTestCaching(System.String field, FieldScoreQuery.Type tp)
		{
			// prepare expected array types for comparison
			System.Collections.Hashtable expectedArrayTypes = new System.Collections.Hashtable();
			expectedArrayTypes[FieldScoreQuery.Type.BYTE] = new byte[0];
			expectedArrayTypes[FieldScoreQuery.Type.SHORT] = new short[0];
			expectedArrayTypes[FieldScoreQuery.Type.INT] = new int[0];
			expectedArrayTypes[FieldScoreQuery.Type.FLOAT] = new float[0];
			
			IndexSearcher s = new IndexSearcher(dir);
			System.Object innerArray = null;
			
			bool warned = false; // print warning once.
			for (int i = 0; i < 10; i++)
			{
				FieldScoreQuery q = new FieldScoreQuery(field, tp);
				Hits h = s.Search(q);
				Assert.AreEqual(N_DOCS, h.Length(), "All docs should be matched!");
				try
				{
					if (i == 0)
					{
						innerArray = q.ValSrc_ForNUnitTest.GetValues(s.GetIndexReader()).GetInnerArray();
						Log(i + ".  compare: " + innerArray.GetType() + " to " + expectedArrayTypes[tp].GetType());
						Assert.AreEqual(innerArray.GetType(), expectedArrayTypes[tp].GetType(), "field values should be cached in the correct array type!");
					}
					else
					{
						Log(i + ".  compare: " + innerArray + " to " + q.ValSrc_ForNUnitTest.GetValues(s.GetIndexReader()).GetInnerArray());
						Assert.AreSame(innerArray, q.ValSrc_ForNUnitTest.GetValues(s.GetIndexReader()).GetInnerArray(), "field values should be cached and reused!");
					}
				}
				catch (System.NotSupportedException)
				{
					if (!warned)
					{
						System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q);
						warned = true;
					}
				}
			}
			
			// verify new values are reloaded (not reused) for a new reader
			s = new IndexSearcher(dir);
			FieldScoreQuery q2 = new FieldScoreQuery(field, tp);
			Hits h2 = s.Search(q2);
			Assert.AreEqual(N_DOCS, h2.Length(), "All docs should be matched!");
			try
			{
				Log("compare: " + innerArray + " to " + q2.ValSrc_ForNUnitTest.GetValues(s.GetIndexReader()).GetInnerArray());
				Assert.AreNotSame(innerArray, q2.ValSrc_ForNUnitTest.GetValues(s.GetIndexReader()).GetInnerArray(), "cached field values should not be reused if reader as changed!");
			}
			catch (System.NotSupportedException)
			{
				if (!warned)
				{
					System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q2);
					warned = true;
				}
			}
		}
Пример #9
0
 // Test that FieldScoreQuery returns docs in expected order.
 private void  DoTestRank(System.String field, FieldScoreQuery.Type tp)
 {
     IndexSearcher s = new IndexSearcher(dir, true);
     Query q = new FieldScoreQuery(field, tp);
     Log("test: " + q);
     QueryUtils.Check(q, s);
     ScoreDoc[] h = s.Search(q, null, 1000).ScoreDocs;
     Assert.AreEqual(N_DOCS, h.Length, "All docs should be matched!");
     System.String prevID = "ID" + (N_DOCS + 1); // greater than all ids of docs in this test
     for (int i = 0; i < h.Length; i++)
     {
         System.String resID = s.Doc(h[i].Doc).Get(ID_FIELD);
         Log(i + ".   score=" + h[i].Score + "  -  " + resID);
         Log(s.Explain(q, h[i].Doc));
         Assert.IsTrue(String.CompareOrdinal(resID, prevID) < 0, "res id " + resID + " should be < prev res id " + prevID);
         prevID = resID;
     }
 }
Пример #10
0
 // Test that values loaded for FieldScoreQuery are cached properly and consumes the proper RAM resources.
 private void  DoTestCaching(System.String field, FieldScoreQuery.Type tp)
 {
     // prepare expected array types for comparison
     System.Collections.Hashtable expectedArrayTypes = new System.Collections.Hashtable();
     expectedArrayTypes[FieldScoreQuery.Type.BYTE] = new sbyte[0];
     expectedArrayTypes[FieldScoreQuery.Type.SHORT] = new short[0];
     expectedArrayTypes[FieldScoreQuery.Type.INT] = new int[0];
     expectedArrayTypes[FieldScoreQuery.Type.FLOAT] = new float[0];
     
     IndexSearcher s = new IndexSearcher(dir, true);
     System.Object[] innerArray = new Object[s.IndexReader.GetSequentialSubReaders().Length];
     
     bool warned = false; // print warning once.
     for (int i = 0; i < 10; i++)
     {
         FieldScoreQuery q = new FieldScoreQuery(field, tp);
         ScoreDoc[] h = s.Search(q, null, 1000).ScoreDocs;
         Assert.AreEqual(N_DOCS, h.Length, "All docs should be matched!");
         IndexReader[] readers = s.IndexReader.GetSequentialSubReaders();
         for (int j = 0; j < readers.Length; j++)
         {
             IndexReader reader = readers[j];
             try
             {
                 if (i == 0)
                 {
                     innerArray[j] = q.valSrc.GetValues(reader).InnerArray;
                     Log(i + ".  compare: " + innerArray[j].GetType() + " to " + expectedArrayTypes[tp].GetType());
                     Assert.AreEqual(innerArray[j].GetType(), expectedArrayTypes[tp].GetType(), "field values should be cached in the correct array type!");
                 }
                 else
                 {
                     Log(i + ".  compare: " + innerArray[j] + " to " + q.valSrc.GetValues(reader).InnerArray);
                     Assert.AreSame(innerArray[j], q.valSrc.GetValues(reader).InnerArray, "field values should be cached and reused!");
                 }
             }
             catch (System.NotSupportedException)
             {
                 if (!warned)
                 {
                     System.Console.Error.WriteLine("WARNING: " +  TestName() + " cannot fully test values of " + q);
                     warned = true;
                 }
             }
         }
     }
     
     // verify new values are reloaded (not reused) for a new reader
     s = new IndexSearcher(dir, true);
     FieldScoreQuery q2 = new FieldScoreQuery(field, tp);
     ScoreDoc[] h2 = s.Search(q2, null, 1000).ScoreDocs;
     Assert.AreEqual(N_DOCS, h2.Length, "All docs should be matched!");
     IndexReader[] readers2 = s.IndexReader.GetSequentialSubReaders();
     for (int j = 0; j < readers2.Length; j++)
     {
         IndexReader reader = readers2[j];
         try
         {
             Log("compare: " + innerArray + " to " + q2.valSrc.GetValues(reader).InnerArray);
             Assert.AreNotSame(innerArray, q2.valSrc.GetValues(reader).InnerArray, "cached field values should not be reused if reader as changed!");
         }
         catch (System.NotSupportedException)
         {
             if (!warned)
             {
                 System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q2);
                 warned = true;
             }
         }
     }
 }
Пример #11
0
 // Test that FieldScoreQuery returns docs with expected score.
 private void  DoTestExactScore(System.String field, FieldScoreQuery.Type tp)
 {
     IndexSearcher s = new IndexSearcher(dir, true);
     Query q = new FieldScoreQuery(field, tp);
     TopDocs td = s.Search(q, null, 1000);
     Assert.AreEqual(N_DOCS, td.TotalHits, "All docs should be matched!");
     ScoreDoc[] sd = td.ScoreDocs;
     for (int i = 0; i < sd.Length; i++)
     {
         float score = sd[i].Score;
         Log(s.Explain(q, sd[i].Doc));
         System.String id = s.IndexReader.Document(sd[i].Doc).Get(ID_FIELD);
         float expectedScore = ExpectedFieldScore(id); // "ID7" --> 7.0
         Assert.AreEqual(expectedScore, score, TEST_SCORE_TOLERANCE_DELTA, "score of " + id + " shuould be " + expectedScore + " != " + score);
     }
 }
        private static Query ParseQuery(string searchTerm)
        {
            var fields = new Dictionary<string, float> { { "Id", 1.2f }, { "Title", 1.0f }, { "Tags", 0.8f }, { "Description", 0.1f },
                                                         { "Author", 1.0f } };
            var analyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion);
            var queryParser = new MultiFieldQueryParser(LuceneCommon.LuceneVersion, fields.Keys.ToArray(), analyzer, fields);

            var conjuctionQuery = new BooleanQuery();
            conjuctionQuery.SetBoost(2.0f);
            var disjunctionQuery = new BooleanQuery();
            disjunctionQuery.SetBoost(0.1f);
            var wildCardQuery = new BooleanQuery();
            wildCardQuery.SetBoost(0.5f);

            // Escape the entire term we use for exact searches.
            var escapedSearchTerm = Escape(searchTerm);
            var exactIdQuery = new TermQuery(new Term("Id-Exact", escapedSearchTerm));
            exactIdQuery.SetBoost(2.5f);
            var wildCardIdQuery = new WildcardQuery(new Term("Id-Exact", "*" + escapedSearchTerm + "*"));

            foreach(var term in GetSearchTerms(searchTerm))
            {
                var termQuery = queryParser.Parse(term);
                conjuctionQuery.Add(termQuery, BooleanClause.Occur.MUST);
                disjunctionQuery.Add(termQuery, BooleanClause.Occur.SHOULD);

                foreach (var field in fields)
                {
                    var wildCardTermQuery = new WildcardQuery(new Term(field.Key, term + "*"));
                    wildCardTermQuery.SetBoost(0.7f * field.Value);
                    wildCardQuery.Add(wildCardTermQuery, BooleanClause.Occur.SHOULD);
                }
            }

            var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
            return new CustomScoreQuery(conjuctionQuery.Combine(new Query[] { exactIdQuery, wildCardIdQuery, conjuctionQuery, disjunctionQuery, wildCardQuery }),
                                       downloadCountBooster);
        }
Пример #13
0
		// Test that FieldScoreQuery returns docs with expected score.
		private void  DoTestCustomScore(System.String field, FieldScoreQuery.Type tp, double dboost)
		{
			float boost = (float) dboost;
			IndexSearcher s = new IndexSearcher(dir, true);
			FieldScoreQuery qValSrc = new FieldScoreQuery(field, tp); // a query that would score by the field
			QueryParser qp = new QueryParser(Util.Version.LUCENE_CURRENT, TEXT_FIELD, anlzr);
			System.String qtxt = "first aid text"; // from the doc texts in FunctionQuerySetup.
			
			// regular (boolean) query.
			Query q1 = qp.Parse(qtxt);
			Log(q1);
			
			// custom query, that should score the same as q1.
			CustomScoreQuery q2CustomNeutral = new CustomScoreQuery(q1);
			q2CustomNeutral.Boost = boost;
			Log(q2CustomNeutral);
			
			// custom query, that should (by default) multiply the scores of q1 by that of the field
			CustomScoreQuery q3CustomMul = new CustomScoreQuery(q1, qValSrc);
			q3CustomMul.SetStrict(true);
			q3CustomMul.Boost = boost;
			Log(q3CustomMul);
			
			// custom query, that should add the scores of q1 to that of the field
			CustomScoreQuery q4CustomAdd = new CustomAddQuery(q1, qValSrc);
			q4CustomAdd.SetStrict(true);
			q4CustomAdd.Boost = boost;
			Log(q4CustomAdd);
			
			// custom query, that multiplies and adds the field score to that of q1
			CustomScoreQuery q5CustomMulAdd = new CustomMulAddQuery(q1, qValSrc, qValSrc);
			q5CustomMulAdd.SetStrict(true);
			q5CustomMulAdd.Boost = boost;
			Log(q5CustomMulAdd);
			
			// do al the searches 
			TopDocs td1 = s.Search(q1, null, 1000);
			TopDocs td2CustomNeutral = s.Search(q2CustomNeutral, null, 1000);
			TopDocs td3CustomMul = s.Search(q3CustomMul, null, 1000);
			TopDocs td4CustomAdd = s.Search(q4CustomAdd, null, 1000);
			TopDocs td5CustomMulAdd = s.Search(q5CustomMulAdd, null, 1000);
			
			// put results in map so we can verify the scores although they have changed
			System.Collections.Hashtable h1 = TopDocsToMap(td1);
			System.Collections.Hashtable h2CustomNeutral = TopDocsToMap(td2CustomNeutral);
			System.Collections.Hashtable h3CustomMul = TopDocsToMap(td3CustomMul);
			System.Collections.Hashtable h4CustomAdd = TopDocsToMap(td4CustomAdd);
			System.Collections.Hashtable h5CustomMulAdd = TopDocsToMap(td5CustomMulAdd);
			
			VerifyResults(boost, s, h1, h2CustomNeutral, h3CustomMul, h4CustomAdd, h5CustomMulAdd, q1, q2CustomNeutral, q3CustomMul, q4CustomAdd, q5CustomMulAdd);
		}
Пример #14
0
        private static Query ParseQuery(SearchFilter searchFilter)
        {
            var fields = new[] { "Id", "Title", "Tags", "Description", "Author" };
            var analyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion);
            var queryParser = new MultiFieldQueryParser(LuceneCommon.LuceneVersion, fields, analyzer);

            // All terms in the multi-term query appear in at least one of the fields.
            var conjuctionQuery = new BooleanQuery();
            conjuctionQuery.SetBoost(2.0f);

            // Some terms in the multi-term query appear in at least one of the fields.
            var disjunctionQuery = new BooleanQuery();
            disjunctionQuery.SetBoost(0.1f);

            // Suffix wildcard search e.g. jquer*
            var wildCardQuery = new BooleanQuery();
            wildCardQuery.SetBoost(0.5f);

            // Escape the entire term we use for exact searches.
            var escapedSearchTerm = Escape(searchFilter.SearchTerm);
            var exactIdQuery = new TermQuery(new Term("Id-Exact", escapedSearchTerm));
            exactIdQuery.SetBoost(2.5f);
            var wildCardIdQuery = new WildcardQuery(new Term("Id-Exact", "*" + escapedSearchTerm + "*"));

            foreach (var term in GetSearchTerms(searchFilter.SearchTerm))
            {
                var termQuery = queryParser.Parse(term);
                conjuctionQuery.Add(termQuery, BooleanClause.Occur.MUST);
                disjunctionQuery.Add(termQuery, BooleanClause.Occur.SHOULD);

                foreach (var field in fields)
                {
                    var wildCardTermQuery = new WildcardQuery(new Term(field, term + "*"));
                    wildCardTermQuery.SetBoost(0.7f);
                    wildCardQuery.Add(wildCardTermQuery, BooleanClause.Occur.SHOULD);
                }
            }

            // Create an OR of all the queries that we have
            var combinedQuery = conjuctionQuery.Combine(new Query[] { exactIdQuery, wildCardIdQuery, conjuctionQuery, disjunctionQuery, wildCardQuery });

            if (searchFilter.SortProperty == SortProperty.Relevance)
            {
                // If searching by relevance, boost scores by download count.
                var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
                return new CustomScoreQuery(combinedQuery, downloadCountBooster);
            }
            return combinedQuery;
        }
 private static void VisitQuery(FieldScoreQuery query, AzureQueryLogger.IndentedTextWriter writer)
 {
 }
Пример #16
0
        /// <summary>
        /// Perform search
        /// </summary>
        /// <param name="query"></param>
        /// <param name="startIndex"></param>
        /// <param name="blockSize"></param>
        /// <param name="indexDirEs"></param>
        /// <param name="indexDirEn"></param>
        /// <param name="sortBy"></param>
        /// <returns></returns>
        public List<IssueDocument> MedesSearch(Query query, int startIndex, int blockSize, Directory indexDirEs, Directory indexDirEn, Directory indexDirHe, string sortBy)
        {
            #if DEBUG
            T.TraceMessage(string.Format("Begin search , query: '{0}'", query.ToString()));
            #endif

            List<IssueDocument> result = new List<IssueDocument>();
            try
            {
                // build a multi searcher across the 2 indexes
                MultiSearcher mSearcher = CombineSearchers(indexDirEs, indexDirEn, indexDirHe);

                TopDocs tDocs = null;
                int iterateLast = startIndex + blockSize;

                string customScoreField = "article_id";
                FieldScoreQuery dateBooster = new FieldScoreQuery(customScoreField, FieldScoreQuery.Type.FLOAT);
                CustomScoreQuery customQuery = new CustomScoreQuery(query, dateBooster);

                tDocs = mSearcher.Search(customQuery, 1000);
                //ScoreDoc[] hits = tpDcs.scoreDocs;
                if (startIndex + blockSize > tDocs.TotalHits) iterateLast = tDocs.TotalHits;

                for (int i = startIndex; i < iterateLast; i++)
                {
                    // Document hitDoc = mSearcher.Doc(hits[i].doc);

                    Document hitDoc = mSearcher.Doc(i);

                    result.Add(new IssueDocument() { Id = Int32.Parse(hitDoc.Get("issue_id").ToString())});
                }

                // close the searcher and indexes
                mSearcher.Dispose();
                indexDirEs.Dispose();
                indexDirEn.Dispose();
                indexDirHe.Dispose();
            }
            catch (Exception ex)
            {
                T.TraceError("Error MedesSearch, query '{0}'", query.ToString());
                T.TraceError(ex);
                throw ex;
            }
            return result;
        }
        private static Query ParseQuery(SearchFilter searchFilter)
        {
            if (String.IsNullOrWhiteSpace(searchFilter.SearchTerm))
            {
                return new MatchAllDocsQuery();
            }

            var fields = new[] { "Id", "Title", "Tags", "Description", "Author" };
            var analyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion);
            var queryParser = new MultiFieldQueryParser(LuceneCommon.LuceneVersion, fields, analyzer);

            // All terms in the multi-term query appear in at least one of the fields.
            var conjuctionQuery = new BooleanQuery();
            conjuctionQuery.Boost = 2.0f;

            // Some terms in the multi-term query appear in at least one of the fields.
            var disjunctionQuery = new BooleanQuery();
            disjunctionQuery.Boost = 0.1f;

            // Suffix wildcard search e.g. jquer*
            var wildCardQuery = new BooleanQuery();
            wildCardQuery.Boost = 0.5f;

            // Escape the entire term we use for exact searches.
            var escapedSearchTerm = Escape(searchFilter.SearchTerm).Replace("id\\:", string.Empty).Replace("author\\:", string.Empty).Replace("tag\\:", string.Empty);

            bool searchLimiter = false;
            bool onlySearchById = false;
            bool onlySearchByAuthor = false;
            bool onlySearchByTag = false;

            var exactIdQuery = new TermQuery(new Term("Id-Exact", escapedSearchTerm));
            exactIdQuery.Boost = 2.5f;
            var wildCardIdQuery = new WildcardQuery(new Term("Id-Exact", "*" + escapedSearchTerm + "*"));

            foreach (var term in GetSearchTerms(searchFilter.SearchTerm))
            {
                var localTerm = term.to_lower_invariant();
                onlySearchById = localTerm.StartsWith("id\\:");
                onlySearchByAuthor = localTerm.StartsWith("author\\:");
                onlySearchByTag = localTerm.StartsWith("tag\\:");
                if (onlySearchById || onlySearchByAuthor || onlySearchByTag) searchLimiter = true; 
                
                localTerm = term.Replace("id\\:", string.Empty).Replace("author\\:", string.Empty).Replace("tag\\:", string.Empty);
                var termQuery = queryParser.Parse(localTerm);
                conjuctionQuery.Add(termQuery, Occur.MUST);
                disjunctionQuery.Add(termQuery, Occur.SHOULD);

                foreach (var field in fields)
                {
                    if (onlySearchById && field != "Id") continue;
                    if (onlySearchByAuthor && field != "Author") continue;
                    if (onlySearchByTag && field != "Tags") continue;

                    var wildCardTermQuery = new WildcardQuery(new Term(field, localTerm + "*"));
                    wildCardTermQuery.Boost = searchLimiter ? 2.5f : 0.7f;
                    wildCardQuery.Add(wildCardTermQuery, Occur.SHOULD);
                }
            }
            
            // Create an OR of all the queries that we have
            var combinedQuery = conjuctionQuery.Combine(new Query[] { exactIdQuery, wildCardIdQuery, conjuctionQuery, disjunctionQuery, wildCardQuery });

            if (onlySearchById)
            {
                combinedQuery = conjuctionQuery.Combine(new Query[] { exactIdQuery, wildCardIdQuery, wildCardQuery });
            } else if (onlySearchByAuthor || onlySearchByTag)
            {
                combinedQuery = conjuctionQuery.Combine(new Query[] { wildCardQuery });
            }
            
            if (searchFilter.SortProperty == SortProperty.Relevance)
            {
                // If searching by relevance, boost scores by download count.
                var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
                return new CustomScoreQuery(combinedQuery, downloadCountBooster);
            }

            return combinedQuery;
        }
Пример #18
0
        // Test that values loaded for FieldScoreQuery are cached properly and consumes the proper RAM resources.
        private void  DoTestCaching(System.String field, FieldScoreQuery.Type tp)
        {
            // prepare expected array types for comparison
            System.Collections.Hashtable expectedArrayTypes = new System.Collections.Hashtable();
            expectedArrayTypes[FieldScoreQuery.Type.BYTE]  = new sbyte[0];
            expectedArrayTypes[FieldScoreQuery.Type.SHORT] = new short[0];
            expectedArrayTypes[FieldScoreQuery.Type.INT]   = new int[0];
            expectedArrayTypes[FieldScoreQuery.Type.FLOAT] = new float[0];

            IndexSearcher s = new IndexSearcher(dir, true);

            System.Object[] innerArray = new Object[s.IndexReader.GetSequentialSubReaders().Length];

            bool warned = false;             // print warning once.

            for (int i = 0; i < 10; i++)
            {
                FieldScoreQuery q = new FieldScoreQuery(field, tp);
                ScoreDoc[]      h = s.Search(q, null, 1000).ScoreDocs;
                Assert.AreEqual(N_DOCS, h.Length, "All docs should be matched!");
                IndexReader[] readers = s.IndexReader.GetSequentialSubReaders();
                for (int j = 0; j < readers.Length; j++)
                {
                    IndexReader reader = readers[j];
                    try
                    {
                        if (i == 0)
                        {
                            innerArray[j] = q.valSrc.GetValues(reader).InnerArray;
                            Log(i + ".  compare: " + innerArray[j].GetType() + " to " + expectedArrayTypes[tp].GetType());
                            Assert.AreEqual(innerArray[j].GetType(), expectedArrayTypes[tp].GetType(), "field values should be cached in the correct array type!");
                        }
                        else
                        {
                            Log(i + ".  compare: " + innerArray[j] + " to " + q.valSrc.GetValues(reader).InnerArray);
                            Assert.AreSame(innerArray[j], q.valSrc.GetValues(reader).InnerArray, "field values should be cached and reused!");
                        }
                    }
                    catch (System.NotSupportedException)
                    {
                        if (!warned)
                        {
                            System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q);
                            warned = true;
                        }
                    }
                }
            }

            // verify new values are reloaded (not reused) for a new reader
            s = new IndexSearcher(dir, true);
            FieldScoreQuery q2 = new FieldScoreQuery(field, tp);

            ScoreDoc[] h2 = s.Search(q2, null, 1000).ScoreDocs;
            Assert.AreEqual(N_DOCS, h2.Length, "All docs should be matched!");
            IndexReader[] readers2 = s.IndexReader.GetSequentialSubReaders();
            for (int j = 0; j < readers2.Length; j++)
            {
                IndexReader reader = readers2[j];
                try
                {
                    Log("compare: " + innerArray + " to " + q2.valSrc.GetValues(reader).InnerArray);
                    Assert.AreNotSame(innerArray, q2.valSrc.GetValues(reader).InnerArray, "cached field values should not be reused if reader as changed!");
                }
                catch (System.NotSupportedException)
                {
                    if (!warned)
                    {
                        System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q2);
                        warned = true;
                    }
                }
            }
        }
Пример #19
0
 public virtual Query VisitFieldScoreQuery(FieldScoreQuery fieldScoreq) { throw new NotImplementedException(); }
Пример #20
0
        private static Query ParseQuery(SearchFilter searchFilter)
        {
            if (String.IsNullOrWhiteSpace(searchFilter.SearchTerm))
            {
                return new MatchAllDocsQuery();
            }

            var fields = new[] { "Title", "Description", "MediaCateName", "CityName" };

            var analyzer = new ChineseAnalyzer();
            //var analyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion);
            var queryParser = new MultiFieldQueryParser(LuceneCommon.LuceneVersion, fields, analyzer);
            var query = queryParser.Parse(searchFilter.SearchTerm);
            // All terms in the multi-term query appear in at least one of the fields.
            var conjuctionQuery = new BooleanQuery();
            conjuctionQuery.Boost = 2.0f;

            // Some terms in the multi-term query appear in at least one of the fields.
            var disjunctionQuery = new BooleanQuery();
            disjunctionQuery.Boost = 0.1f;

            // Suffix wildcard search e.g. jquer*
            var wildCardQuery = new BooleanQuery();
            wildCardQuery.Boost = 0.5f;

            //// Escape the entire term we use for exact searches.
            var escapedSearchTerm = Escape(searchFilter.SearchTerm);

            var exactIdQuery = new TermQuery(new Term("Title", escapedSearchTerm));

            exactIdQuery.Boost = 2.5f;

            var wildCardIdQuery = new WildcardQuery(new Term("Title", "*" + escapedSearchTerm + "*"));

            foreach (var term in GetSearchTerms(searchFilter.SearchTerm))
            {
                var termQuery = queryParser.Parse(term);
                conjuctionQuery.Add(termQuery, Occur.MUST);
                disjunctionQuery.Add(termQuery, Occur.SHOULD);

                foreach (var field in fields)
                {
                    var wildCardTermQuery = new WildcardQuery(new Term(field, term + "*"));
                    wildCardTermQuery.Boost = 0.7f;
                    wildCardQuery.Add(wildCardTermQuery, Occur.SHOULD);
                }
            }

            //var combinedQuery =
            //    conjuctionQuery.Combine(new Query[] { exactIdQuery, conjuctionQuery });
            //// Create an OR of all the queries that we have

            var combinedQuery =
                conjuctionQuery.Combine(new Query[] { exactIdQuery, wildCardIdQuery, conjuctionQuery, disjunctionQuery, wildCardQuery });

            if (searchFilter.SortProperty == SortProperty.Hit)
            {
                // If searching by relevance, boost scores by download count.
                var downloadCountBooster = new FieldScoreQuery("Hit", FieldScoreQuery.Type.INT);
                return new CustomScoreQuery(combinedQuery, downloadCountBooster);
            }
            return combinedQuery;
        }