private IQueryable<Package> SearchCore(SearchFilter searchFilter, out int totalHits)
{
int numRecords = searchFilter.Skip + searchFilter.Take;
var searcher = new IndexSearcher(_directory, readOnly: true);
var query = ParseQuery(searchFilter);
// IF searching by relevance, boost scores by download count.
if (searchFilter.SortProperty == SortProperty.Relevance)
{
var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
query = new CustomScoreQuery(query, downloadCountBooster);
}
var filterTerm = searchFilter.IncludePrerelease ? "IsLatest" : "IsLatestStable";
var termQuery = new TermQuery(new Term(filterTerm, Boolean.TrueString));
var filter = new QueryWrapperFilter(termQuery);
var results = searcher.Search(query, filter: filter, n: numRecords, sort: new Sort(GetSortField(searchFilter)));
totalHits = results.totalHits;
if (results.totalHits == 0 || searchFilter.CountOnly)
{
return Enumerable.Empty<Package>().AsQueryable();
}
var packages = results.scoreDocs
.Skip(searchFilter.Skip)
.Select(sd => PackageFromDoc(searcher.Doc(sd.doc)))
.ToList();
return packages.AsQueryable();
}
// Test that FieldScoreQuery returns docs with expected score.
private void DoTestCustomScore(System.String field, FieldScoreQuery.Type tp, double dboost)
{
float boost = (float)dboost;
IndexSearcher s = new IndexSearcher(dir, true);
FieldScoreQuery qValSrc = new FieldScoreQuery(field, tp); // a query that would score by the field
QueryParser qp = new QueryParser(Util.Version.LUCENE_CURRENT, TEXT_FIELD, anlzr);
System.String qtxt = "first aid text"; // from the doc texts in FunctionQuerySetup.
// regular (boolean) query.
Query q1 = qp.Parse(qtxt);
Log(q1);
// custom query, that should score the same as q1.
CustomScoreQuery q2CustomNeutral = new CustomScoreQuery(q1);
q2CustomNeutral.Boost = boost;
Log(q2CustomNeutral);
// custom query, that should (by default) multiply the scores of q1 by that of the field
CustomScoreQuery q3CustomMul = new CustomScoreQuery(q1, qValSrc);
q3CustomMul.SetStrict(true);
q3CustomMul.Boost = boost;
Log(q3CustomMul);
// custom query, that should add the scores of q1 to that of the field
CustomScoreQuery q4CustomAdd = new CustomAddQuery(q1, qValSrc);
q4CustomAdd.SetStrict(true);
q4CustomAdd.Boost = boost;
Log(q4CustomAdd);
// custom query, that multiplies and adds the field score to that of q1
CustomScoreQuery q5CustomMulAdd = new CustomMulAddQuery(q1, qValSrc, qValSrc);
q5CustomMulAdd.SetStrict(true);
q5CustomMulAdd.Boost = boost;
Log(q5CustomMulAdd);
// do al the searches
TopDocs td1 = s.Search(q1, null, 1000);
TopDocs td2CustomNeutral = s.Search(q2CustomNeutral, null, 1000);
TopDocs td3CustomMul = s.Search(q3CustomMul, null, 1000);
TopDocs td4CustomAdd = s.Search(q4CustomAdd, null, 1000);
TopDocs td5CustomMulAdd = s.Search(q5CustomMulAdd, null, 1000);
// put results in map so we can verify the scores although they have changed
System.Collections.Hashtable h1 = TopDocsToMap(td1);
System.Collections.Hashtable h2CustomNeutral = TopDocsToMap(td2CustomNeutral);
System.Collections.Hashtable h3CustomMul = TopDocsToMap(td3CustomMul);
System.Collections.Hashtable h4CustomAdd = TopDocsToMap(td4CustomAdd);
System.Collections.Hashtable h5CustomMulAdd = TopDocsToMap(td5CustomMulAdd);
VerifyResults(boost, s, h1, h2CustomNeutral, h3CustomMul, h4CustomAdd, h5CustomMulAdd, q1, q2CustomNeutral, q3CustomMul, q4CustomAdd, q5CustomMulAdd);
}
// Test that FieldScoreQuery returns docs with expected score.
private void DoTestExactScore(System.String field, FieldScoreQuery.Type tp)
{
IndexSearcher s = new IndexSearcher(dir, true);
Query q = new FieldScoreQuery(field, tp);
TopDocs td = s.Search(q, null, 1000);
Assert.AreEqual(N_DOCS, td.TotalHits, "All docs should be matched!");
ScoreDoc[] sd = td.ScoreDocs;
for (int i = 0; i < sd.Length; i++)
{
float score = sd[i].Score;
Log(s.Explain(q, sd[i].Doc));
System.String id = s.IndexReader.Document(sd[i].Doc).Get(ID_FIELD);
float expectedScore = ExpectedFieldScore(id); // "ID7" --> 7.0
Assert.AreEqual(expectedScore, score, TEST_SCORE_TOLERANCE_DELTA, "score of " + id + " shuould be " + expectedScore + " != " + score);
}
}
private SearchResults SearchCore(SearchFilter searchFilter)
{
// Get index timestamp
DateTime timestamp = File.GetLastWriteTimeUtc(LuceneCommon.GetIndexMetadataPath());
int numRecords = searchFilter.Skip + searchFilter.Take;
var searcher = new IndexSearcher(_directory, readOnly: true);
var query = ParseQuery(searchFilter);
// IF searching by relevance, boost scores by download count.
if (searchFilter.SortOrder == SortOrder.Relevance)
{
var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
query = new CustomScoreQuery(query, downloadCountBooster);
}
var filterTerm = searchFilter.IncludePrerelease ? "IsLatest" : "IsLatestStable";
Query filterQuery = new TermQuery(new Term(filterTerm, Boolean.TrueString));
if (searchFilter.CuratedFeed != null)
{
var feedFilterQuery = new TermQuery(new Term("CuratedFeedKey", searchFilter.CuratedFeed.Key.ToString(CultureInfo.InvariantCulture)));
BooleanQuery conjunctionQuery = new BooleanQuery();
conjunctionQuery.Add(filterQuery, Occur.MUST);
conjunctionQuery.Add(feedFilterQuery, Occur.MUST);
filterQuery = conjunctionQuery;
}
Filter filter = new QueryWrapperFilter(filterQuery);
var results = searcher.Search(query, filter: filter, n: numRecords, sort: new Sort(GetSortField(searchFilter)));
if (results.TotalHits == 0 || searchFilter.CountOnly)
{
return new SearchResults(results.TotalHits, timestamp);
}
var packages = results.ScoreDocs
.Skip(searchFilter.Skip)
.Select(sd => PackageFromDoc(searcher.Doc(sd.Doc)))
.ToList();
return new SearchResults(
results.TotalHits,
timestamp,
packages.AsQueryable());
}
// Test that FieldScoreQuery returns docs in expected order.
private void DoTestRank(System.String field, FieldScoreQuery.Type tp)
{
IndexSearcher s = new IndexSearcher(dir, true);
Query q = new FieldScoreQuery(field, tp);
Log("test: " + q);
QueryUtils.Check(q, s);
ScoreDoc[] h = s.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(N_DOCS, h.Length, "All docs should be matched!");
System.String prevID = "ID" + (N_DOCS + 1); // greater than all ids of docs in this test
for (int i = 0; i < h.Length; i++)
{
System.String resID = s.Doc(h[i].Doc).Get(ID_FIELD);
Log(i + ". score=" + h[i].Score + " - " + resID);
Log(s.Explain(q, h[i].Doc));
Assert.IsTrue(String.CompareOrdinal(resID, prevID) < 0, "res id " + resID + " should be < prev res id " + prevID);
prevID = resID;
}
}
private IQueryable<Package> SearchCore(SearchFilter searchFilter, out int totalHits)
{
int numRecords = searchFilter.Skip + searchFilter.Take;
var searcher = new IndexSearcher(_directory, readOnly: true);
var query = ParseQuery(searchFilter);
// IF searching by relevance, boost scores by download count.
if (searchFilter.SortProperty == SortProperty.Relevance)
{
var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
query = new CustomScoreQuery(query, downloadCountBooster);
}
var filterTerm = searchFilter.IncludePrerelease ? "IsLatest" : "IsLatestStable";
Query filterQuery = new TermQuery(new Term(filterTerm, Boolean.TrueString));
if (searchFilter.CuratedFeedKey.HasValue)
{
var feedFilterQuery = new TermQuery(new Term("CuratedFeedKey", searchFilter.CuratedFeedKey.Value.ToString(CultureInfo.InvariantCulture)));
BooleanQuery conjunctionQuery = new BooleanQuery();
conjunctionQuery.Add(filterQuery, Occur.MUST);
conjunctionQuery.Add(feedFilterQuery, Occur.MUST);
filterQuery = conjunctionQuery;
}
Filter filter = new QueryWrapperFilter(filterQuery);
var results = searcher.Search(query, filter: filter, n: numRecords, sort: new Sort(GetSortField(searchFilter)));
totalHits = results.TotalHits;
if (results.TotalHits == 0 || searchFilter.CountOnly)
{
return Enumerable.Empty<Package>().AsQueryable();
}
var packages = results.ScoreDocs
.Skip(searchFilter.Skip)
.Select(sd => PackageFromDoc(searcher.Doc(sd.Doc)))
.ToList();
return packages.AsQueryable();
}
private SearchResults SearchCore(SearchFilter searchFilter)
{
// Get index timestamp
DateTime timestamp = File.GetLastWriteTimeUtc(LuceneCommon.IndexMetadataPath);
int numRecords = searchFilter.Skip + searchFilter.Take;
var searcher = new IndexSearcher(_directory, readOnly: true);
var query = ParseQuery(searchFilter);
// If searching by relevance, boost scores by download count.
if (searchFilter.SortProperty == SortProperty.Relevance)
{
var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
query = new CustomScoreQuery(query, downloadCountBooster);
}
var filterTerm = searchFilter.IncludePrerelease ? "IsLatest" : "IsLatestStable";
Query filterQuery = new TermQuery(new Term(filterTerm, Boolean.TrueString));
Filter filter = new QueryWrapperFilter(filterQuery);
var results = searcher.Search(query, filter: filter, n: numRecords, sort: new Sort(GetSortField(searchFilter)));
if (results.TotalHits == 0 || searchFilter.CountOnly)
{
return new SearchResults(results.TotalHits, timestamp);
}
var packages = results.ScoreDocs
.Skip(searchFilter.Skip)
.Select(sd => PackageFromDoc(searcher.Doc(sd.Doc)))
.ToList();
return new SearchResults(
results.TotalHits,
timestamp,
packages.AsQueryable());
}
// Test that values loaded for FieldScoreQuery are cached properly and consumes the proper RAM resources.
private void DoTestCaching(System.String field, FieldScoreQuery.Type tp)
{
// prepare expected array types for comparison
System.Collections.Hashtable expectedArrayTypes = new System.Collections.Hashtable();
expectedArrayTypes[FieldScoreQuery.Type.BYTE] = new byte[0];
expectedArrayTypes[FieldScoreQuery.Type.SHORT] = new short[0];
expectedArrayTypes[FieldScoreQuery.Type.INT] = new int[0];
expectedArrayTypes[FieldScoreQuery.Type.FLOAT] = new float[0];
IndexSearcher s = new IndexSearcher(dir);
System.Object innerArray = null;
bool warned = false; // print warning once.
for (int i = 0; i < 10; i++)
{
FieldScoreQuery q = new FieldScoreQuery(field, tp);
Hits h = s.Search(q);
Assert.AreEqual(N_DOCS, h.Length(), "All docs should be matched!");
try
{
if (i == 0)
{
innerArray = q.ValSrc_ForNUnitTest.GetValues(s.GetIndexReader()).GetInnerArray();
Log(i + ". compare: " + innerArray.GetType() + " to " + expectedArrayTypes[tp].GetType());
Assert.AreEqual(innerArray.GetType(), expectedArrayTypes[tp].GetType(), "field values should be cached in the correct array type!");
}
else
{
Log(i + ". compare: " + innerArray + " to " + q.ValSrc_ForNUnitTest.GetValues(s.GetIndexReader()).GetInnerArray());
Assert.AreSame(innerArray, q.ValSrc_ForNUnitTest.GetValues(s.GetIndexReader()).GetInnerArray(), "field values should be cached and reused!");
}
}
catch (System.NotSupportedException)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q);
warned = true;
}
}
}
// verify new values are reloaded (not reused) for a new reader
s = new IndexSearcher(dir);
FieldScoreQuery q2 = new FieldScoreQuery(field, tp);
Hits h2 = s.Search(q2);
Assert.AreEqual(N_DOCS, h2.Length(), "All docs should be matched!");
try
{
Log("compare: " + innerArray + " to " + q2.ValSrc_ForNUnitTest.GetValues(s.GetIndexReader()).GetInnerArray());
Assert.AreNotSame(innerArray, q2.ValSrc_ForNUnitTest.GetValues(s.GetIndexReader()).GetInnerArray(), "cached field values should not be reused if reader as changed!");
}
catch (System.NotSupportedException)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q2);
warned = true;
}
}
}
// Test that FieldScoreQuery returns docs in expected order.
private void DoTestRank(System.String field, FieldScoreQuery.Type tp)
{
IndexSearcher s = new IndexSearcher(dir, true);
Query q = new FieldScoreQuery(field, tp);
Log("test: " + q);
QueryUtils.Check(q, s);
ScoreDoc[] h = s.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(N_DOCS, h.Length, "All docs should be matched!");
System.String prevID = "ID" + (N_DOCS + 1); // greater than all ids of docs in this test
for (int i = 0; i < h.Length; i++)
{
System.String resID = s.Doc(h[i].Doc).Get(ID_FIELD);
Log(i + ". score=" + h[i].Score + " - " + resID);
Log(s.Explain(q, h[i].Doc));
Assert.IsTrue(String.CompareOrdinal(resID, prevID) < 0, "res id " + resID + " should be < prev res id " + prevID);
prevID = resID;
}
}
// Test that values loaded for FieldScoreQuery are cached properly and consumes the proper RAM resources.
private void DoTestCaching(System.String field, FieldScoreQuery.Type tp)
{
// prepare expected array types for comparison
System.Collections.Hashtable expectedArrayTypes = new System.Collections.Hashtable();
expectedArrayTypes[FieldScoreQuery.Type.BYTE] = new sbyte[0];
expectedArrayTypes[FieldScoreQuery.Type.SHORT] = new short[0];
expectedArrayTypes[FieldScoreQuery.Type.INT] = new int[0];
expectedArrayTypes[FieldScoreQuery.Type.FLOAT] = new float[0];
IndexSearcher s = new IndexSearcher(dir, true);
System.Object[] innerArray = new Object[s.IndexReader.GetSequentialSubReaders().Length];
bool warned = false; // print warning once.
for (int i = 0; i < 10; i++)
{
FieldScoreQuery q = new FieldScoreQuery(field, tp);
ScoreDoc[] h = s.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(N_DOCS, h.Length, "All docs should be matched!");
IndexReader[] readers = s.IndexReader.GetSequentialSubReaders();
for (int j = 0; j < readers.Length; j++)
{
IndexReader reader = readers[j];
try
{
if (i == 0)
{
innerArray[j] = q.valSrc.GetValues(reader).InnerArray;
Log(i + ". compare: " + innerArray[j].GetType() + " to " + expectedArrayTypes[tp].GetType());
Assert.AreEqual(innerArray[j].GetType(), expectedArrayTypes[tp].GetType(), "field values should be cached in the correct array type!");
}
else
{
Log(i + ". compare: " + innerArray[j] + " to " + q.valSrc.GetValues(reader).InnerArray);
Assert.AreSame(innerArray[j], q.valSrc.GetValues(reader).InnerArray, "field values should be cached and reused!");
}
}
catch (System.NotSupportedException)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q);
warned = true;
}
}
}
}
// verify new values are reloaded (not reused) for a new reader
s = new IndexSearcher(dir, true);
FieldScoreQuery q2 = new FieldScoreQuery(field, tp);
ScoreDoc[] h2 = s.Search(q2, null, 1000).ScoreDocs;
Assert.AreEqual(N_DOCS, h2.Length, "All docs should be matched!");
IndexReader[] readers2 = s.IndexReader.GetSequentialSubReaders();
for (int j = 0; j < readers2.Length; j++)
{
IndexReader reader = readers2[j];
try
{
Log("compare: " + innerArray + " to " + q2.valSrc.GetValues(reader).InnerArray);
Assert.AreNotSame(innerArray, q2.valSrc.GetValues(reader).InnerArray, "cached field values should not be reused if reader as changed!");
}
catch (System.NotSupportedException)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q2);
warned = true;
}
}
}
}
// Test that FieldScoreQuery returns docs with expected score.
private void DoTestExactScore(System.String field, FieldScoreQuery.Type tp)
{
IndexSearcher s = new IndexSearcher(dir, true);
Query q = new FieldScoreQuery(field, tp);
TopDocs td = s.Search(q, null, 1000);
Assert.AreEqual(N_DOCS, td.TotalHits, "All docs should be matched!");
ScoreDoc[] sd = td.ScoreDocs;
for (int i = 0; i < sd.Length; i++)
{
float score = sd[i].Score;
Log(s.Explain(q, sd[i].Doc));
System.String id = s.IndexReader.Document(sd[i].Doc).Get(ID_FIELD);
float expectedScore = ExpectedFieldScore(id); // "ID7" --> 7.0
Assert.AreEqual(expectedScore, score, TEST_SCORE_TOLERANCE_DELTA, "score of " + id + " shuould be " + expectedScore + " != " + score);
}
}
private static Query ParseQuery(string searchTerm)
{
var fields = new Dictionary<string, float> { { "Id", 1.2f }, { "Title", 1.0f }, { "Tags", 0.8f }, { "Description", 0.1f },
{ "Author", 1.0f } };
var analyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion);
var queryParser = new MultiFieldQueryParser(LuceneCommon.LuceneVersion, fields.Keys.ToArray(), analyzer, fields);
var conjuctionQuery = new BooleanQuery();
conjuctionQuery.SetBoost(2.0f);
var disjunctionQuery = new BooleanQuery();
disjunctionQuery.SetBoost(0.1f);
var wildCardQuery = new BooleanQuery();
wildCardQuery.SetBoost(0.5f);
// Escape the entire term we use for exact searches.
var escapedSearchTerm = Escape(searchTerm);
var exactIdQuery = new TermQuery(new Term("Id-Exact", escapedSearchTerm));
exactIdQuery.SetBoost(2.5f);
var wildCardIdQuery = new WildcardQuery(new Term("Id-Exact", "*" + escapedSearchTerm + "*"));
foreach(var term in GetSearchTerms(searchTerm))
{
var termQuery = queryParser.Parse(term);
conjuctionQuery.Add(termQuery, BooleanClause.Occur.MUST);
disjunctionQuery.Add(termQuery, BooleanClause.Occur.SHOULD);
foreach (var field in fields)
{
var wildCardTermQuery = new WildcardQuery(new Term(field.Key, term + "*"));
wildCardTermQuery.SetBoost(0.7f * field.Value);
wildCardQuery.Add(wildCardTermQuery, BooleanClause.Occur.SHOULD);
}
}
var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
return new CustomScoreQuery(conjuctionQuery.Combine(new Query[] { exactIdQuery, wildCardIdQuery, conjuctionQuery, disjunctionQuery, wildCardQuery }),
downloadCountBooster);
}
// Test that FieldScoreQuery returns docs with expected score.
private void DoTestCustomScore(System.String field, FieldScoreQuery.Type tp, double dboost)
{
float boost = (float) dboost;
IndexSearcher s = new IndexSearcher(dir, true);
FieldScoreQuery qValSrc = new FieldScoreQuery(field, tp); // a query that would score by the field
QueryParser qp = new QueryParser(Util.Version.LUCENE_CURRENT, TEXT_FIELD, anlzr);
System.String qtxt = "first aid text"; // from the doc texts in FunctionQuerySetup.
// regular (boolean) query.
Query q1 = qp.Parse(qtxt);
Log(q1);
// custom query, that should score the same as q1.
CustomScoreQuery q2CustomNeutral = new CustomScoreQuery(q1);
q2CustomNeutral.Boost = boost;
Log(q2CustomNeutral);
// custom query, that should (by default) multiply the scores of q1 by that of the field
CustomScoreQuery q3CustomMul = new CustomScoreQuery(q1, qValSrc);
q3CustomMul.SetStrict(true);
q3CustomMul.Boost = boost;
Log(q3CustomMul);
// custom query, that should add the scores of q1 to that of the field
CustomScoreQuery q4CustomAdd = new CustomAddQuery(q1, qValSrc);
q4CustomAdd.SetStrict(true);
q4CustomAdd.Boost = boost;
Log(q4CustomAdd);
// custom query, that multiplies and adds the field score to that of q1
CustomScoreQuery q5CustomMulAdd = new CustomMulAddQuery(q1, qValSrc, qValSrc);
q5CustomMulAdd.SetStrict(true);
q5CustomMulAdd.Boost = boost;
Log(q5CustomMulAdd);
// do al the searches
TopDocs td1 = s.Search(q1, null, 1000);
TopDocs td2CustomNeutral = s.Search(q2CustomNeutral, null, 1000);
TopDocs td3CustomMul = s.Search(q3CustomMul, null, 1000);
TopDocs td4CustomAdd = s.Search(q4CustomAdd, null, 1000);
TopDocs td5CustomMulAdd = s.Search(q5CustomMulAdd, null, 1000);
// put results in map so we can verify the scores although they have changed
System.Collections.Hashtable h1 = TopDocsToMap(td1);
System.Collections.Hashtable h2CustomNeutral = TopDocsToMap(td2CustomNeutral);
System.Collections.Hashtable h3CustomMul = TopDocsToMap(td3CustomMul);
System.Collections.Hashtable h4CustomAdd = TopDocsToMap(td4CustomAdd);
System.Collections.Hashtable h5CustomMulAdd = TopDocsToMap(td5CustomMulAdd);
VerifyResults(boost, s, h1, h2CustomNeutral, h3CustomMul, h4CustomAdd, h5CustomMulAdd, q1, q2CustomNeutral, q3CustomMul, q4CustomAdd, q5CustomMulAdd);
}
private static Query ParseQuery(SearchFilter searchFilter)
{
var fields = new[] { "Id", "Title", "Tags", "Description", "Author" };
var analyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion);
var queryParser = new MultiFieldQueryParser(LuceneCommon.LuceneVersion, fields, analyzer);
// All terms in the multi-term query appear in at least one of the fields.
var conjuctionQuery = new BooleanQuery();
conjuctionQuery.SetBoost(2.0f);
// Some terms in the multi-term query appear in at least one of the fields.
var disjunctionQuery = new BooleanQuery();
disjunctionQuery.SetBoost(0.1f);
// Suffix wildcard search e.g. jquer*
var wildCardQuery = new BooleanQuery();
wildCardQuery.SetBoost(0.5f);
// Escape the entire term we use for exact searches.
var escapedSearchTerm = Escape(searchFilter.SearchTerm);
var exactIdQuery = new TermQuery(new Term("Id-Exact", escapedSearchTerm));
exactIdQuery.SetBoost(2.5f);
var wildCardIdQuery = new WildcardQuery(new Term("Id-Exact", "*" + escapedSearchTerm + "*"));
foreach (var term in GetSearchTerms(searchFilter.SearchTerm))
{
var termQuery = queryParser.Parse(term);
conjuctionQuery.Add(termQuery, BooleanClause.Occur.MUST);
disjunctionQuery.Add(termQuery, BooleanClause.Occur.SHOULD);
foreach (var field in fields)
{
var wildCardTermQuery = new WildcardQuery(new Term(field, term + "*"));
wildCardTermQuery.SetBoost(0.7f);
wildCardQuery.Add(wildCardTermQuery, BooleanClause.Occur.SHOULD);
}
}
// Create an OR of all the queries that we have
var combinedQuery = conjuctionQuery.Combine(new Query[] { exactIdQuery, wildCardIdQuery, conjuctionQuery, disjunctionQuery, wildCardQuery });
if (searchFilter.SortProperty == SortProperty.Relevance)
{
// If searching by relevance, boost scores by download count.
var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
return new CustomScoreQuery(combinedQuery, downloadCountBooster);
}
return combinedQuery;
}
private static void VisitQuery(FieldScoreQuery query, AzureQueryLogger.IndentedTextWriter writer)
{
}
/// <summary>
/// Perform search
/// </summary>
/// <param name="query"></param>
/// <param name="startIndex"></param>
/// <param name="blockSize"></param>
/// <param name="indexDirEs"></param>
/// <param name="indexDirEn"></param>
/// <param name="sortBy"></param>
/// <returns></returns>
public List<IssueDocument> MedesSearch(Query query, int startIndex, int blockSize, Directory indexDirEs, Directory indexDirEn, Directory indexDirHe, string sortBy)
{
#if DEBUG
T.TraceMessage(string.Format("Begin search , query: '{0}'", query.ToString()));
#endif
List<IssueDocument> result = new List<IssueDocument>();
try
{
// build a multi searcher across the 2 indexes
MultiSearcher mSearcher = CombineSearchers(indexDirEs, indexDirEn, indexDirHe);
TopDocs tDocs = null;
int iterateLast = startIndex + blockSize;
string customScoreField = "article_id";
FieldScoreQuery dateBooster = new FieldScoreQuery(customScoreField, FieldScoreQuery.Type.FLOAT);
CustomScoreQuery customQuery = new CustomScoreQuery(query, dateBooster);
tDocs = mSearcher.Search(customQuery, 1000);
//ScoreDoc[] hits = tpDcs.scoreDocs;
if (startIndex + blockSize > tDocs.TotalHits) iterateLast = tDocs.TotalHits;
for (int i = startIndex; i < iterateLast; i++)
{
// Document hitDoc = mSearcher.Doc(hits[i].doc);
Document hitDoc = mSearcher.Doc(i);
result.Add(new IssueDocument() { Id = Int32.Parse(hitDoc.Get("issue_id").ToString())});
}
// close the searcher and indexes
mSearcher.Dispose();
indexDirEs.Dispose();
indexDirEn.Dispose();
indexDirHe.Dispose();
}
catch (Exception ex)
{
T.TraceError("Error MedesSearch, query '{0}'", query.ToString());
T.TraceError(ex);
throw ex;
}
return result;
}
private static Query ParseQuery(SearchFilter searchFilter)
{
if (String.IsNullOrWhiteSpace(searchFilter.SearchTerm))
{
return new MatchAllDocsQuery();
}
var fields = new[] { "Id", "Title", "Tags", "Description", "Author" };
var analyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion);
var queryParser = new MultiFieldQueryParser(LuceneCommon.LuceneVersion, fields, analyzer);
// All terms in the multi-term query appear in at least one of the fields.
var conjuctionQuery = new BooleanQuery();
conjuctionQuery.Boost = 2.0f;
// Some terms in the multi-term query appear in at least one of the fields.
var disjunctionQuery = new BooleanQuery();
disjunctionQuery.Boost = 0.1f;
// Suffix wildcard search e.g. jquer*
var wildCardQuery = new BooleanQuery();
wildCardQuery.Boost = 0.5f;
// Escape the entire term we use for exact searches.
var escapedSearchTerm = Escape(searchFilter.SearchTerm).Replace("id\\:", string.Empty).Replace("author\\:", string.Empty).Replace("tag\\:", string.Empty);
bool searchLimiter = false;
bool onlySearchById = false;
bool onlySearchByAuthor = false;
bool onlySearchByTag = false;
var exactIdQuery = new TermQuery(new Term("Id-Exact", escapedSearchTerm));
exactIdQuery.Boost = 2.5f;
var wildCardIdQuery = new WildcardQuery(new Term("Id-Exact", "*" + escapedSearchTerm + "*"));
foreach (var term in GetSearchTerms(searchFilter.SearchTerm))
{
var localTerm = term.to_lower_invariant();
onlySearchById = localTerm.StartsWith("id\\:");
onlySearchByAuthor = localTerm.StartsWith("author\\:");
onlySearchByTag = localTerm.StartsWith("tag\\:");
if (onlySearchById || onlySearchByAuthor || onlySearchByTag) searchLimiter = true;
localTerm = term.Replace("id\\:", string.Empty).Replace("author\\:", string.Empty).Replace("tag\\:", string.Empty);
var termQuery = queryParser.Parse(localTerm);
conjuctionQuery.Add(termQuery, Occur.MUST);
disjunctionQuery.Add(termQuery, Occur.SHOULD);
foreach (var field in fields)
{
if (onlySearchById && field != "Id") continue;
if (onlySearchByAuthor && field != "Author") continue;
if (onlySearchByTag && field != "Tags") continue;
var wildCardTermQuery = new WildcardQuery(new Term(field, localTerm + "*"));
wildCardTermQuery.Boost = searchLimiter ? 2.5f : 0.7f;
wildCardQuery.Add(wildCardTermQuery, Occur.SHOULD);
}
}
// Create an OR of all the queries that we have
var combinedQuery = conjuctionQuery.Combine(new Query[] { exactIdQuery, wildCardIdQuery, conjuctionQuery, disjunctionQuery, wildCardQuery });
if (onlySearchById)
{
combinedQuery = conjuctionQuery.Combine(new Query[] { exactIdQuery, wildCardIdQuery, wildCardQuery });
} else if (onlySearchByAuthor || onlySearchByTag)
{
combinedQuery = conjuctionQuery.Combine(new Query[] { wildCardQuery });
}
if (searchFilter.SortProperty == SortProperty.Relevance)
{
// If searching by relevance, boost scores by download count.
var downloadCountBooster = new FieldScoreQuery("DownloadCount", FieldScoreQuery.Type.INT);
return new CustomScoreQuery(combinedQuery, downloadCountBooster);
}
return combinedQuery;
}
// Test that values loaded for FieldScoreQuery are cached properly and consumes the proper RAM resources.
private void DoTestCaching(System.String field, FieldScoreQuery.Type tp)
{
// prepare expected array types for comparison
System.Collections.Hashtable expectedArrayTypes = new System.Collections.Hashtable();
expectedArrayTypes[FieldScoreQuery.Type.BYTE] = new sbyte[0];
expectedArrayTypes[FieldScoreQuery.Type.SHORT] = new short[0];
expectedArrayTypes[FieldScoreQuery.Type.INT] = new int[0];
expectedArrayTypes[FieldScoreQuery.Type.FLOAT] = new float[0];
IndexSearcher s = new IndexSearcher(dir, true);
System.Object[] innerArray = new Object[s.IndexReader.GetSequentialSubReaders().Length];
bool warned = false; // print warning once.
for (int i = 0; i < 10; i++)
{
FieldScoreQuery q = new FieldScoreQuery(field, tp);
ScoreDoc[] h = s.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(N_DOCS, h.Length, "All docs should be matched!");
IndexReader[] readers = s.IndexReader.GetSequentialSubReaders();
for (int j = 0; j < readers.Length; j++)
{
IndexReader reader = readers[j];
try
{
if (i == 0)
{
innerArray[j] = q.valSrc.GetValues(reader).InnerArray;
Log(i + ". compare: " + innerArray[j].GetType() + " to " + expectedArrayTypes[tp].GetType());
Assert.AreEqual(innerArray[j].GetType(), expectedArrayTypes[tp].GetType(), "field values should be cached in the correct array type!");
}
else
{
Log(i + ". compare: " + innerArray[j] + " to " + q.valSrc.GetValues(reader).InnerArray);
Assert.AreSame(innerArray[j], q.valSrc.GetValues(reader).InnerArray, "field values should be cached and reused!");
}
}
catch (System.NotSupportedException)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q);
warned = true;
}
}
}
}
// verify new values are reloaded (not reused) for a new reader
s = new IndexSearcher(dir, true);
FieldScoreQuery q2 = new FieldScoreQuery(field, tp);
ScoreDoc[] h2 = s.Search(q2, null, 1000).ScoreDocs;
Assert.AreEqual(N_DOCS, h2.Length, "All docs should be matched!");
IndexReader[] readers2 = s.IndexReader.GetSequentialSubReaders();
for (int j = 0; j < readers2.Length; j++)
{
IndexReader reader = readers2[j];
try
{
Log("compare: " + innerArray + " to " + q2.valSrc.GetValues(reader).InnerArray);
Assert.AreNotSame(innerArray, q2.valSrc.GetValues(reader).InnerArray, "cached field values should not be reused if reader as changed!");
}
catch (System.NotSupportedException)
{
if (!warned)
{
System.Console.Error.WriteLine("WARNING: " + TestName() + " cannot fully test values of " + q2);
warned = true;
}
}
}
}
public virtual Query VisitFieldScoreQuery(FieldScoreQuery fieldScoreq) { throw new NotImplementedException(); }
private static Query ParseQuery(SearchFilter searchFilter)
{
if (String.IsNullOrWhiteSpace(searchFilter.SearchTerm))
{
return new MatchAllDocsQuery();
}
var fields = new[] { "Title", "Description", "MediaCateName", "CityName" };
var analyzer = new ChineseAnalyzer();
//var analyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion);
var queryParser = new MultiFieldQueryParser(LuceneCommon.LuceneVersion, fields, analyzer);
var query = queryParser.Parse(searchFilter.SearchTerm);
// All terms in the multi-term query appear in at least one of the fields.
var conjuctionQuery = new BooleanQuery();
conjuctionQuery.Boost = 2.0f;
// Some terms in the multi-term query appear in at least one of the fields.
var disjunctionQuery = new BooleanQuery();
disjunctionQuery.Boost = 0.1f;
// Suffix wildcard search e.g. jquer*
var wildCardQuery = new BooleanQuery();
wildCardQuery.Boost = 0.5f;
//// Escape the entire term we use for exact searches.
var escapedSearchTerm = Escape(searchFilter.SearchTerm);
var exactIdQuery = new TermQuery(new Term("Title", escapedSearchTerm));
exactIdQuery.Boost = 2.5f;
var wildCardIdQuery = new WildcardQuery(new Term("Title", "*" + escapedSearchTerm + "*"));
foreach (var term in GetSearchTerms(searchFilter.SearchTerm))
{
var termQuery = queryParser.Parse(term);
conjuctionQuery.Add(termQuery, Occur.MUST);
disjunctionQuery.Add(termQuery, Occur.SHOULD);
foreach (var field in fields)
{
var wildCardTermQuery = new WildcardQuery(new Term(field, term + "*"));
wildCardTermQuery.Boost = 0.7f;
wildCardQuery.Add(wildCardTermQuery, Occur.SHOULD);
}
}
//var combinedQuery =
// conjuctionQuery.Combine(new Query[] { exactIdQuery, conjuctionQuery });
//// Create an OR of all the queries that we have
var combinedQuery =
conjuctionQuery.Combine(new Query[] { exactIdQuery, wildCardIdQuery, conjuctionQuery, disjunctionQuery, wildCardQuery });
if (searchFilter.SortProperty == SortProperty.Hit)
{
// If searching by relevance, boost scores by download count.
var downloadCountBooster = new FieldScoreQuery("Hit", FieldScoreQuery.Type.INT);
return new CustomScoreQuery(combinedQuery, downloadCountBooster);
}
return combinedQuery;
}
