private static void WriteData(
JsonWriter jsonWriter,
NuGetIndexSearcher searcher,
TopDocs topDocs,
int skip,
int take,
Uri baseAddress,
bool includePrerelease,
bool includeExplanation,
NuGetVersion semVerLevel,
Query query)
{
jsonWriter.WritePropertyName("data");
jsonWriter.WriteStartArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
jsonWriter.WriteStartObject();
string id = document.Get("Id");
var relativeAddress = UriFormatter.MakeRegistrationRelativeAddress(id);
var absoluteAddress = new Uri(baseAddress, relativeAddress).AbsoluteUri;
WriteProperty(jsonWriter, "@id", absoluteAddress);
WriteProperty(jsonWriter, "@type", "Package");
WriteProperty(jsonWriter, "registration", absoluteAddress);
WriteProperty(jsonWriter, "id", id);
WriteDocumentValue(jsonWriter, "version", document, LuceneMetadataConstants.FullVersionPropertyName);
WriteDocumentValue(jsonWriter, "description", document, "Description");
WriteDocumentValue(jsonWriter, "summary", document, "Summary");
WriteDocumentValue(jsonWriter, "title", document, "Title");
WriteDocumentValue(jsonWriter, "iconUrl", document, "IconUrl");
WriteDocumentValue(jsonWriter, "licenseUrl", document, "LicenseUrl");
WriteDocumentValue(jsonWriter, "projectUrl", document, "ProjectUrl");
WriteDocumentValueAsArray(jsonWriter, "tags", document, "Tags");
WriteDocumentValueAsArray(jsonWriter, "authors", document, "Authors", true);
WriteProperty(jsonWriter, "totalDownloads", searcher.Versions[scoreDoc.Doc].AllVersionDetails.Select(item => item.Downloads).Sum());
WriteProperty(jsonWriter, "verified", searcher.VerifiedPackages.Contains(id));
WriteVersions(jsonWriter, baseAddress, id, includePrerelease, semVerLevel, searcher.Versions[scoreDoc.Doc]);
if (includeExplanation)
{
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
WriteProperty(jsonWriter, "explanation", explanation.ToString());
WriteProperty(jsonWriter, "score", scoreDoc.Score);
}
jsonWriter.WriteEndObject();
}
jsonWriter.WriteEndArray();
}
/// <summary>
/// Converts Lucene document to index model.
/// </summary>
/// <param name="query">The query.</param>
/// <param name="hit">The scoredoc.</param>
/// <returns>Index model</returns>
private IndexModelBase LuceneDocToIndexModel(Query query, ScoreDoc hit)
{
var doc = _indexSearcher.Doc(hit.Doc);
IndexModelBase document = new IndexModelBase();
try
{
var hitJsonField = doc.GetField("JSON");
if (hitJsonField != null)
{
string hitJson = hitJsonField.GetStringValue();
JObject jObject = JObject.Parse(hitJson);
Type indexModelType = Type.GetType($"{ jObject["IndexModelType"].ToStringSafe() }, { jObject["IndexModelAssembly"].ToStringSafe() }");
if (indexModelType != null)
{
document = (IndexModelBase)jObject.ToObject(indexModelType); // return the source document as the derived type
}
else
{
document = jObject.ToObject <IndexModelBase>(); // return the source document as the base type
}
}
Explanation explanation = _indexSearcher.Explain(query, hit.Doc);
document["Explain"] = explanation.ToString();
document.Score = hit.Score;
return(document);
}
catch { } // ignore if the result if an exception resulted (most likely cause is getting a result from a non-rock index)
return(null);
}
public IEnumerable <SampleDataFileRow> _search(string searchTerm, int precision, global::Lucene.Net.Store.Directory indexDirectory)
{
Debug.Assert(!String.IsNullOrEmpty(searchTerm));
List <SampleDataFileRow> results = new List <SampleDataFileRow>();
if (String.IsNullOrEmpty(searchTerm))
{
return(results);
}
using (IndexSearcher searcher = new IndexSearcher(indexDirectory))
{
var analyzer = new StandardAnalyzer(Version.LUCENE_30, ListStopWords);
QueryParser parser = new QueryParser(Version.LUCENE_30, "LineText", analyzer);
if (precision > 0 && precision < 100)
{
parser.FuzzyMinSim = ((float)precision) * 0.01f;
}
else if (precision == 100)
{
parser.FuzzyMinSim = 0.99f;
}
else
{
parser.FuzzyMinSim = 0.8f;
}
//parser.PhraseSlop = 5;
var query = ParseQuery(searchTerm, parser);
//var query = fparseQuery(searchTerm);
ScoreDoc[] hitsFound = searcher.Search(query, null, CountSearchResults).ScoreDocs;
foreach (var t in hitsFound)
{
var sampleDataFileRow = new SampleDataFileRow();
int docId = t.Doc;
float score = t.Score;
Explanation explanation = searcher.Explain(query, t.Doc);
Document doc = searcher.Doc(docId);
sampleDataFileRow.LineNumber = int.Parse(doc.Get("LineNumber"));
sampleDataFileRow.LineText = doc.Get("LineText");
sampleDataFileRow.Score = score;
_explanationResult = explanation.ToString();
results.Add(sampleDataFileRow);
}
analyzer.Close();
searcher.Dispose();
}
return(results.OrderByDescending(x => x.Score).ToList());
}
private static void WriteExplanations(JsonWriter jsonWriter, NuGetIndexSearcher searcher, TopDocs topDocs, int skip, int take, Query query)
{
jsonWriter.WritePropertyName("explanations");
jsonWriter.WriteStartArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
jsonWriter.WriteValue(explanation.ToString());
}
jsonWriter.WriteEndArray();
}
private static string AddExplanation(IndexSearcher searcher, string data, Query query, ScoreDoc scoreDoc, IDictionary <string, int> rankings)
{
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
JObject diagnostics = new JObject();
int rankVal;
string id = searcher.Doc(scoreDoc.Doc).Get("Id");
if (rankings.TryGetValue(id, out rankVal))
{
float rankingScore = RankingScoreQuery.GetRankingScore(rankings, id);
diagnostics.Add("Rank", rankVal);
diagnostics.Add("RankScore", rankingScore);
diagnostics.Add("LuceneScore", scoreDoc.Score / rankingScore);
}
diagnostics.Add("Score", scoreDoc.Score.ToString());
diagnostics.Add("Explanation", explanation.ToString());
diagnostics.Add("IdTerms", GetTerms(searcher, scoreDoc.Doc, "Id"));
diagnostics.Add("TokenizedIdTerms", GetTerms(searcher, scoreDoc.Doc, "TokenizedId"));
diagnostics.Add("ShingledIdTerms", GetTerms(searcher, scoreDoc.Doc, "ShingledId"));
diagnostics.Add("VersionTerms", GetTerms(searcher, scoreDoc.Doc, "Version"));
diagnostics.Add("TitleTerms", GetTerms(searcher, scoreDoc.Doc, "Title"));
diagnostics.Add("TagsTerms", GetTerms(searcher, scoreDoc.Doc, "Tags"));
diagnostics.Add("DescriptionTerms", GetTerms(searcher, scoreDoc.Doc, "Description"));
diagnostics.Add("AuthorsTerms", GetTerms(searcher, scoreDoc.Doc, "Authors"));
diagnostics.Add("OwnersTerms", GetTerms(searcher, scoreDoc.Doc, "Owners"));
diagnostics.Add("PublishedDate", GetInt(searcher, scoreDoc.Doc, "PublishedDate"));
diagnostics.Add("EditedDate", GetInt(searcher, scoreDoc.Doc, "EditedDate"));
diagnostics.Add("CuratedFeed", GetMultiValue(searcher, scoreDoc.Doc, "CuratedFeed"));
diagnostics.Add("Key", GetInt(searcher, scoreDoc.Doc, "Key"));
diagnostics.Add("Checksum", GetInt(searcher, scoreDoc.Doc, "Checksum"));
diagnostics.Add("ProjectGuidRankings", GetProjectGuidRankings(searcher, scoreDoc.Doc));
JObject obj = JObject.Parse(data);
obj.Add("diagnostics", diagnostics);
data = obj.ToString();
return(data);
}
public static JToken MakeResultData(IndexSearcher searcher, string scheme, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
{
Uri registrationBaseAddress = searcherManager.RegistrationBaseAddress[scheme];
JArray array = new JArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
string url = document.Get("Url");
string id = document.Get("Id");
string version = document.Get("Version");
JObject obj = new JObject();
obj["@id"] = new Uri(registrationBaseAddress, url).AbsoluteUri;
obj["@type"] = "Package";
obj["registration"] = new Uri(registrationBaseAddress, string.Format("{0}/index.json", id.ToLowerInvariant())).AbsoluteUri;
obj["id"] = id;
AddField(obj, document, "domain", "Domain");
AddField(obj, document, "description", "Description");
AddField(obj, document, "summary", "Summary");
AddField(obj, document, "title", "Title");
AddField(obj, document, "iconUrl", "IconUrl");
AddFieldAsArray(obj, document, "tags", "Tags");
AddFieldAsArray(obj, document, "authors", "Authors");
obj["version"] = version;
obj["versions"] = searcherManager.GetVersions(scheme, scoreDoc.Doc);
if (includeExplanation)
{
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
obj["explanation"] = explanation.ToString();
}
array.Add(obj);
}
return(array);
}
/// <summary>
/// Searches the index for the querytext
/// </summary>
/// <param name="querytext">The text to search the index</param>
public void SearchText(string querytext)
{
System.Console.WriteLine("Searching for " + querytext);
querytext = querytext.ToLower();
Query query = parser.Parse(querytext);
TopDocs results = searcher.Search(query, 100);
System.Console.WriteLine("Number of results is " + results.TotalHits);
int rank = 0;
foreach (ScoreDoc scoreDoc in results.ScoreDocs)
{
rank++;
Lucene.Net.Documents.Document doc = searcher.Doc(scoreDoc.Doc);
string myFieldValue = doc.Get(TEXT_FN).ToString();
Console.WriteLine("Rank " + rank + " Score" + scoreDoc.Score + " text " + myFieldValue);
Explanation ex = searcher.Explain(query, scoreDoc.Doc);
Console.WriteLine(ex.ToString());
}
}
public override void Collect(int doc)
{
Explanation exp = null;
doc = doc + @base;
try
{
exp = s.Explain(q, doc);
}
catch (IOException e)
{
throw new Exception
("exception in hitcollector of [[" + d + "]] for #" + doc, e);
}
assertNotNull("Explanation of [[" + d + "]] for #" + doc + " is null", exp);
verifyExplanation(d, doc, scorer.Score(), deep, exp);
assertTrue("Explanation of [[" + d + "]] for #" + doc +
" does not indicate match: " + exp.ToString(), exp.IsMatch);
}
public virtual void TestMinFunction()
{
PayloadNearQuery query;
TopDocs hits;
query = NewPhraseQuery("field", "twenty two", true, new MinPayloadFunction());
QueryUtils.Check(query);
// all 10 hits should have score = 2 (min payload value)
hits = Searcher.Search(query, null, 100);
Assert.IsTrue(hits != null, "hits is null and it shouldn't be");
Assert.AreEqual(10, hits.TotalHits, "should be 10 hits");
for (int j = 0; j < hits.ScoreDocs.Length; j++)
{
ScoreDoc doc = hits.ScoreDocs[j];
Assert.AreEqual(2, doc.Score, doc.Score + " does not equal: " + 2);
Explanation explain = Searcher.Explain(query, hits.ScoreDocs[j].Doc);
string exp = explain.ToString();
Assert.IsTrue(exp.IndexOf("MinPayloadFunction") > -1, exp);
Assert.AreEqual(2f, explain.Value, hits.ScoreDocs[j].Score + " explain value does not equal: " + 2);
}
}
public virtual void TestAverageFunction()
{
PayloadNearQuery query;
TopDocs hits;
query = NewPhraseQuery("field", "twenty two", true, new AveragePayloadFunction());
QueryUtils.Check(query);
// all 10 hits should have score = 3 because adjacent terms have payloads of 2,4
// and all the similarity factors are set to 1
hits = Searcher.Search(query, null, 100);
Assert.IsTrue(hits != null, "hits is null and it shouldn't be");
Assert.AreEqual(10, hits.TotalHits, "should be 10 hits");
for (int j = 0; j < hits.ScoreDocs.Length; j++)
{
ScoreDoc doc = hits.ScoreDocs[j];
Assert.AreEqual(3, doc.Score, doc.Score + " does not equal: " + 3);
Explanation explain = Searcher.Explain(query, hits.ScoreDocs[j].Doc);
string exp = explain.ToString();
Assert.IsTrue(exp.IndexOf("AveragePayloadFunction") > -1, exp);
Assert.AreEqual(3f, explain.Value, hits.ScoreDocs[j].Score + " explain value does not equal: " + 3);
}
}
public static JToken AutoCompleteMakeResult(IndexSearcher searcher, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
{
JArray array = new JArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
string id = document.Get("Id");
array.Add(id);
}
JObject result = new JObject();
result.Add("@context", new JObject {
{ "@vocab", "http://schema.nuget.org/schema#" }
});
result.Add("totalHits", topDocs.TotalHits);
result.Add("indexName", searcherManager.IndexName);
result.Add("data", array);
if (includeExplanation)
{
JArray explanations = new JArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
explanations.Add(explanation.ToString());
}
result.Add("explanations", explanations);
}
return(result);
}
public virtual void TestSpanNearScorerExplain()
{
SpanNearQuery q = MakeQuery();
Explanation e = Searcher.Explain(q, 1);
Assert.IsTrue(0.0f < e.Value, "Scorer explanation value for doc#1 isn't positive: " + e.ToString());
}
public static string Explain(IndexSearcher searcher, Query query, ScoreDoc match)
{
Explanation explanation = searcher.Explain(query, match.Doc);
return(explanation.ToString());
}
public static void AddExplanation(this ModelStateDictionary modelState, Explanation explanation)
{
modelState.AddModelError(string.Empty, explanation.ToString());
}
private void trace(Explanation explanation)
{
trace(explanation.ToString());
}
public static JToken MakeResultData(IndexSearcher searcher, string currentOwner, string scheme, TopDocs topDocs, int skip, int take, SecureSearcherManager searcherManager, bool includeExplanation, Query query)
{
Uri registrationBaseAddress = searcherManager.RegistrationBaseAddress[scheme];
JArray array = new JArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
string url = document.Get("Url");
string id = document.Get("Id");
string version = document.Get("Version");
string owner = document.Get("Owner");
string ns = document.Get("Namespace");
if (ns != null)
{
id = string.Format("{0}.{1}", ns, id);
}
JObject obj = new JObject();
obj["@id"] = new Uri(registrationBaseAddress, url).AbsoluteUri;
obj["@type"] = document.Get("@type");
obj["registration"] = new Uri(registrationBaseAddress, string.Format("{0}/index.json", id.ToLowerInvariant())).AbsoluteUri;
obj["id"] = id;
obj["isOwner"] = (owner == currentOwner);
ServiceHelpers.AddField(obj, document, "packageContent", "PackageContent");
ServiceHelpers.AddField(obj, document, "catalogEntry", "CatalogEntry");
ServiceHelpers.AddFieldBool(obj, document, "listed", "Listed");
ServiceHelpers.AddField(obj, document, "tenantId", "TenantId");
ServiceHelpers.AddField(obj, document, "namespace", "Namespace");
ServiceHelpers.AddField(obj, document, "visibility", "Visibility");
ServiceHelpers.AddField(obj, document, "description", "Description");
ServiceHelpers.AddField(obj, document, "summary", "Summary");
ServiceHelpers.AddField(obj, document, "title", "Title");
ServiceHelpers.AddField(obj, document, "iconUrl", "IconUrl");
ServiceHelpers.AddFieldAsObject(obj, document, "owner", "OwnerDetails");
ServiceHelpers.AddFieldAsArray(obj, document, "tags", "Tags");
ServiceHelpers.AddFieldAsArray(obj, document, "authors", "Authors");
obj["version"] = version;
obj["versions"] = searcherManager.GetVersions(scheme, scoreDoc.Doc);
if (includeExplanation)
{
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
obj["explanation"] = explanation.ToString();
}
array.Add(obj);
}
return(array);
}
public DomainException(Exception innerException, Explanation explanation)
: base(explanation.ToString(), innerException)
{
this.Explanation = explanation;
}