// V2 search implementation - called from the NuGet Gallery
public static void WriteV2Result(JsonWriter jsonWriter, NuGetIndexSearcher searcher, TopDocs topDocs, int skip, int take, NuGetVersion semVerLevel)
{
jsonWriter.WriteStartObject();
WriteInfoV2(jsonWriter, searcher, topDocs);
WriteDataV2(jsonWriter, searcher, topDocs, skip, take, semVerLevel);
jsonWriter.WriteEndObject();
}
public static void WriteAutoCompleteVersionResult(JsonWriter jsonWriter, NuGetIndexSearcher searcher, bool includePrerelease, NuGetVersion semVerLevel, TopDocs topDocs)
{
jsonWriter.WriteStartObject();
WriteInfo(jsonWriter, null, searcher, topDocs);
WriteVersions(jsonWriter, searcher, includePrerelease, semVerLevel, topDocs);
jsonWriter.WriteEndObject();
}
private static void WriteInfoV2(JsonWriter jsonWriter, NuGetIndexSearcher searcher, TopDocs topDocs)
{
WriteProperty(jsonWriter, "totalHits", topDocs.TotalHits);
string timestamp;
DateTime dt;
if (searcher.CommitUserData.TryGetValue("commitTimeStamp", out timestamp) &&
DateTime.TryParse(timestamp, out dt))
{
timestamp = dt.ToUniversalTime().ToString("G");
}
else
{
timestamp = DateTime.MinValue.ToUniversalTime().ToString("G");
}
// TODO: can we find this value?
// WriteProperty(jsonWriter, "timeTakenInMs", 0);
WriteProperty(jsonWriter, "index", searcher.Manager.IndexName);
// CommittimeStamp format: 2015-10-12T18:39:39.6830871Z
// Time format in V2: 10/22/2015 4:53:25 PM
WriteProperty(jsonWriter, "indexTimestamp", timestamp);
}
// Diagnostic responses
public static void WriteStatsResult(JsonWriter jsonWriter, NuGetIndexSearcher searcher)
{
jsonWriter.WriteStartObject();
WriteProperty(jsonWriter, "numDocs", searcher.IndexReader.NumDocs());
WriteProperty(jsonWriter, "indexName", searcher.Manager.IndexName);
WriteProperty(jsonWriter, "machineName", searcher.Manager.MachineName);
WriteProperty(jsonWriter, "lastReopen", searcher.LastReopen);
WriteProperty(jsonWriter, "lastIndexReloadTime", searcher.Manager.LastIndexReloadTime);
WriteProperty(jsonWriter, "lastIndexReloadDurationInMilliseconds", searcher.Manager.LastIndexReloadDurationInMilliseconds);
WriteProperty(jsonWriter, "lastAuxiliaryDataLoadTime", searcher.Manager.LastAuxiliaryDataLoadTime);
jsonWriter.WritePropertyName("lastAuxiliaryDataUpdateTime");
jsonWriter.WriteStartObject();
foreach (var userData in searcher.Manager.AuxiliaryFiles?.LastModifiedTimeForFiles ?? new Dictionary <string, DateTime?>())
{
WriteProperty(jsonWriter, userData.Key, userData.Value);
}
jsonWriter.WriteEndObject();
jsonWriter.WritePropertyName("CommitUserData");
jsonWriter.WriteStartObject();
foreach (var userData in searcher.CommitUserData)
{
WriteProperty(jsonWriter, userData.Key, userData.Value);
}
jsonWriter.WriteEndObject();
jsonWriter.WriteEndObject();
}
private static void WriteInfo(JsonWriter jsonWriter, Uri baseAddress, NuGetIndexSearcher searcher, TopDocs topDocs)
{
WriteContext(jsonWriter, baseAddress);
WriteProperty(jsonWriter, "totalHits", topDocs.TotalHits);
WriteProperty(jsonWriter, "lastReopen", searcher.LastReopen.ToString("o"));
WriteProperty(jsonWriter, "index", searcher.Manager.IndexName);
}
// V3 implementation - called directly from the integrated Visual Studio client
public static void WriteSearchResult(
JsonWriter jsonWriter,
NuGetIndexSearcher searcher,
string scheme,
TopDocs topDocs,
int skip,
int take,
bool includePrerelease,
bool includeExplanation,
NuGetVersion semVerLevel,
Query query)
{
Uri baseAddress;
var useSemVer2Registration = SemVerHelpers.ShouldIncludeSemVer2Results(semVerLevel);
RegistrationBaseAddresses registrationAddresses = searcher.Manager.RegistrationBaseAddresses;
switch (scheme)
{
case "https":
baseAddress = (useSemVer2Registration ? registrationAddresses.SemVer2Https : registrationAddresses.LegacyHttps);
break;
case "http":
default:
// if scheme specified is invalid, fall back to using as much information as we can
baseAddress = (useSemVer2Registration ? registrationAddresses.SemVer2Http : registrationAddresses.LegacyHttp);
break;
}
jsonWriter.WriteStartObject();
WriteInfo(jsonWriter, baseAddress, searcher, topDocs);
WriteData(jsonWriter, searcher, topDocs, skip, take, baseAddress, includePrerelease, includeExplanation, semVerLevel, query);
jsonWriter.WriteEndObject();
}
private static void WriteData(
JsonWriter jsonWriter,
NuGetIndexSearcher searcher,
TopDocs topDocs,
int skip,
int take,
Uri baseAddress,
bool includePrerelease,
bool includeExplanation,
NuGetVersion semVerLevel,
Query query)
{
jsonWriter.WritePropertyName("data");
jsonWriter.WriteStartArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
jsonWriter.WriteStartObject();
string id = document.Get("Id");
var relativeAddress = UriFormatter.MakeRegistrationRelativeAddress(id);
var absoluteAddress = new Uri(baseAddress, relativeAddress).AbsoluteUri;
WriteProperty(jsonWriter, "@id", absoluteAddress);
WriteProperty(jsonWriter, "@type", "Package");
WriteProperty(jsonWriter, "registration", absoluteAddress);
WriteProperty(jsonWriter, "id", id);
WriteDocumentValue(jsonWriter, "version", document, LuceneMetadataConstants.FullVersionPropertyName);
WriteDocumentValue(jsonWriter, "description", document, "Description");
WriteDocumentValue(jsonWriter, "summary", document, "Summary");
WriteDocumentValue(jsonWriter, "title", document, "Title");
WriteDocumentValue(jsonWriter, "iconUrl", document, "IconUrl");
WriteDocumentValue(jsonWriter, "licenseUrl", document, "LicenseUrl");
WriteDocumentValue(jsonWriter, "projectUrl", document, "ProjectUrl");
WriteDocumentValueAsArray(jsonWriter, "tags", document, "Tags");
WriteDocumentValueAsArray(jsonWriter, "authors", document, "Authors", true);
WriteProperty(jsonWriter, "totalDownloads", searcher.Versions[scoreDoc.Doc].AllVersionDetails.Select(item => item.Downloads).Sum());
WriteProperty(jsonWriter, "verified", searcher.VerifiedPackages.Contains(id));
WriteVersions(jsonWriter, baseAddress, id, includePrerelease, semVerLevel, searcher.Versions[scoreDoc.Doc]);
if (includeExplanation)
{
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
WriteProperty(jsonWriter, "explanation", explanation.ToString());
WriteProperty(jsonWriter, "score", scoreDoc.Score);
}
jsonWriter.WriteEndObject();
}
jsonWriter.WriteEndArray();
}
public static IEnumerable <string> GetOwners(NuGetIndexSearcher searcher, string id)
{
HashSet <string> owners;
if (searcher.Owners.PackagesWithOwners.TryGetValue(id, out owners))
{
return(owners);
}
return(Enumerable.Empty <string>());
}
private static void WriteExplanations(JsonWriter jsonWriter, NuGetIndexSearcher searcher, TopDocs topDocs, int skip, int take, Query query)
{
jsonWriter.WritePropertyName("explanations");
jsonWriter.WriteStartArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
jsonWriter.WriteValue(explanation.ToString());
}
jsonWriter.WriteEndArray();
}
// V3 auto-complete implementation - called from the Visual Studio "project.json editor"
public static void WriteAutoCompleteResult(JsonWriter jsonWriter, NuGetIndexSearcher searcher, TopDocs topDocs, int skip, int take, bool includeExplanation, Query query)
{
jsonWriter.WriteStartObject();
WriteInfo(jsonWriter, null, searcher, topDocs);
WriteIds(jsonWriter, searcher, topDocs, skip, take);
if (includeExplanation)
{
WriteExplanations(jsonWriter, searcher, topDocs, skip, take, query);
}
jsonWriter.WriteEndObject();
}
private static void WriteIds(JsonWriter jsonWriter, NuGetIndexSearcher searcher, TopDocs topDocs, int skip, int take)
{
jsonWriter.WritePropertyName("data");
jsonWriter.WriteStartArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
string id = document.Get("Id");
jsonWriter.WriteValue(id);
}
jsonWriter.WriteEndArray();
}
private static Query MakeSearchQuery(string q, NuGetIndexSearcher searcher)
{
try
{
Query query = NuGetQuery.MakeQuery(q, searcher.Owners);
Query boostedQuery = new DownloadsBoostedQuery(query,
searcher.DocIdMapping,
searcher.Downloads,
searcher.Rankings,
searcher.QueryBoostingContext);
return(boostedQuery);
}
catch (ParseException)
{
throw new ClientException(HttpStatusCode.BadRequest, "Invalid query format");
}
}
private static void ListDocumentsImpl(JsonWriter jsonWriter,
NuGetIndexSearcher searcher,
Query query,
string sortBy,
int skip,
int take)
{
Query boostedQuery = new DownloadsBoostedQuery(query,
searcher.DocIdMapping,
searcher.Downloads,
searcher.Rankings,
searcher.QueryBoostingContext);
int nDocs = skip + take;
Sort sort = GetSort(sortBy);
TopDocs topDocs = (sort == null)
? searcher.Search(boostedQuery, nDocs)
: searcher.Search(boostedQuery, null, nDocs, sort);
ResponseFormatter.WriteV2Result(jsonWriter, searcher, topDocs, skip, take);
}
private static void ListDocumentsImpl(JsonWriter jsonWriter,
NuGetIndexSearcher searcher,
Query query,
string sortBy,
int skip,
int take,
NuGetVersion semVerLevel)
{
Query boostedQuery = new DownloadsBoostedQuery(query,
searcher.DocIdMapping,
searcher.Downloads,
searcher.Rankings,
searcher.QueryBoostingContext);
int nDocs = skip + take;
Sort sort = GetSort(sortBy);
TopDocs topDocs = (sort == null)
? searcher.Search(boostedQuery, nDocs)
: searcher.Search(boostedQuery, null, nDocs, sort);
ResponseFormatter.WriteV2Result(jsonWriter, searcher, topDocs, skip, take, semVerLevel);
}
private static void WriteVersions(JsonWriter jsonWriter, NuGetIndexSearcher searcher, bool includePrerelease, NuGetVersion semVerLevel, TopDocs topDocs)
{
var includeSemVer2 = SemVerHelpers.ShouldIncludeSemVer2Results(semVerLevel);
jsonWriter.WritePropertyName("data");
jsonWriter.WriteStartArray();
if (topDocs.TotalHits > 0)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[0];
var versions = includePrerelease
? searcher.Versions[scoreDoc.Doc].GetVersions(onlyListed: true, includeSemVer2: includeSemVer2)
: searcher.Versions[scoreDoc.Doc].GetStableVersions(onlyListed: true, includeSemVer2: includeSemVer2);
foreach (var version in versions)
{
jsonWriter.WriteValue(version);
}
}
jsonWriter.WriteEndArray();
}
private static JToken ListDocumentsImpl(NuGetIndexSearcher searcher, Query query, IDictionary<string, int> rankings, Filter filter, string sortBy, int skip, int take, NuGetSearcherManager manager)
{
Query boostedQuery = new RankingScoreQuery(query, rankings);
int nDocs = skip + take;
Sort sort = GetSort(sortBy);
TopDocs topDocs = (sort == null) ?
searcher.Search(boostedQuery, filter, nDocs) :
searcher.Search(boostedQuery, filter, nDocs, sort);
return MakeResults(searcher, topDocs, skip, take, boostedQuery, rankings, manager);
}
private static Query MakeSearchQuery(string q, NuGetIndexSearcher searcher)
{
try
{
Query query = NuGetQuery.MakeQuery(q, searcher.Owners);
Query boostedQuery = new DownloadsBoostedQuery(query,
searcher.DocIdMapping,
searcher.Downloads,
searcher.Rankings,
searcher.QueryBoostingContext);
return boostedQuery;
}
catch (ParseException)
{
throw new ClientException(HttpStatusCode.BadRequest, "Invalid query format");
}
}
private static void WriteDataV2(JsonWriter jsonWriter, NuGetIndexSearcher searcher, TopDocs topDocs, int skip, int take, NuGetVersion semVerLevel)
{
jsonWriter.WritePropertyName("data");
jsonWriter.WriteStartArray();
var includeSemVer2 = SemVerHelpers.ShouldIncludeSemVer2Results(semVerLevel);
var isLatestBitSet = includeSemVer2 ? searcher.LatestSemVer2BitSet : searcher.LatestBitSet;
var isLatestStableBitSet = includeSemVer2 ? searcher.LatestStableSemVer2BitSet : searcher.LatestStableBitSet;
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
string id = document.Get("Id");
string normalizedVersion = document.Get(LuceneMetadataConstants.NormalizedVersionPropertyName);
string fullVersion = document.Get(LuceneMetadataConstants.FullVersionPropertyName);
Tuple <int, int> downloadCounts = NuGetIndexSearcher.DownloadCounts(searcher.Versions[scoreDoc.Doc], normalizedVersion);
bool isVerified = searcher.VerifiedPackages.Contains(id);
bool isLatest = isLatestBitSet.Get(scoreDoc.Doc);
bool isLatestStable = isLatestStableBitSet.Get(scoreDoc.Doc);
jsonWriter.WriteStartObject();
WriteRegistrationV2(jsonWriter, id, downloadCounts.Item1, NuGetIndexSearcher.GetOwners(searcher, id), isVerified);
WriteDocumentValue(jsonWriter, "Version", document, LuceneMetadataConstants.VerbatimVersionPropertyName);
WriteProperty(jsonWriter, "NormalizedVersion", normalizedVersion);
WriteDocumentValue(jsonWriter, "Title", document, "Title");
WriteDocumentValue(jsonWriter, "Description", document, "Description");
WriteDocumentValue(jsonWriter, "Summary", document, "Summary");
WriteDocumentValue(jsonWriter, "Authors", document, "Authors");
WriteDocumentValue(jsonWriter, "Copyright", document, "Copyright");
WriteDocumentValue(jsonWriter, "Language", document, "Language");
WriteDocumentValue(jsonWriter, "Tags", document, "Tags");
WriteDocumentValue(jsonWriter, "ReleaseNotes", document, "ReleaseNotes");
WriteDocumentValue(jsonWriter, "ProjectUrl", document, "ProjectUrl");
WriteDocumentValue(jsonWriter, "IconUrl", document, "IconUrl");
WriteProperty(jsonWriter, "IsLatestStable", isLatestStable);
WriteProperty(jsonWriter, "IsLatest", isLatest);
WriteProperty(jsonWriter, "Listed", bool.Parse(document.Get("Listed") ?? "true"));
WriteDocumentValue(jsonWriter, "Created", document, "OriginalCreated");
WriteDocumentValue(jsonWriter, "Published", document, "OriginalPublished");
WriteDocumentValue(jsonWriter, "LastUpdated", document, "OriginalPublished");
WriteDocumentValue(jsonWriter, "LastEdited", document, "OriginalLastEdited");
WriteProperty(jsonWriter, "DownloadCount", downloadCounts.Item2);
WriteDocumentValue(jsonWriter, "FlattenedDependencies", document, "FlattenedDependencies");
jsonWriter.WritePropertyName("Dependencies");
jsonWriter.WriteRawValue(document.Get("Dependencies") ?? "[]");
jsonWriter.WritePropertyName("SupportedFrameworks");
jsonWriter.WriteRawValue(document.Get("SupportedFrameworks") ?? "[]");
WriteDocumentValue(jsonWriter, "MinClientVersion", document, "MinClientVersion");
WriteDocumentValue(jsonWriter, "Hash", document, "PackageHash");
WriteDocumentValue(jsonWriter, "HashAlgorithm", document, "PackageHashAlgorithm");
WriteProperty(jsonWriter, "PackageFileSize", int.Parse(document.Get("PackageSize") ?? "0"));
WriteDocumentValue(jsonWriter, "LicenseUrl", document, "LicenseUrl");
WriteProperty(jsonWriter, "RequiresLicenseAcceptance", bool.Parse(document.Get("RequiresLicenseAcceptance") ?? "false"));
jsonWriter.WriteEndObject();
}
jsonWriter.WriteEndArray();
}
public static IEnumerable<string> GetOwners(NuGetIndexSearcher searcher, string id)
{
HashSet<string> owners;
if (searcher.Owners.PackagesWithOwners.TryGetValue(id, out owners))
{
return owners;
}
return Enumerable.Empty<string>();
}
public static JToken MakeResultData(NuGetIndexSearcher searcher, string scheme, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
{
Uri registrationBaseAddress = searcherManager.RegistrationBaseAddress[scheme];
JArray array = new JArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
string url = document.Get("Url");
string id = document.Get("Id");
string version = document.Get("Version");
JObject obj = new JObject();
obj["@id"] = new Uri(registrationBaseAddress, url).AbsoluteUri;
obj["@type"] = "Package";
obj["registration"] = new Uri(registrationBaseAddress, string.Format("{0}/index.json", id.ToLowerInvariant())).AbsoluteUri;
obj["id"] = id;
AddField(obj, document, "domain", "Domain");
AddField(obj, document, "description", "Description");
AddField(obj, document, "summary", "Summary");
AddField(obj, document, "title", "Title");
AddField(obj, document, "iconUrl", "IconUrl");
AddField(obj, document, "licenseUrl", "LicenseUrl");
AddField(obj, document, "projectUrl", "ProjectUrl");
AddFieldAsArray(obj, document, "tags", "Tags");
AddFieldAsArray(obj, document, "authors", "Authors");
obj["version"] = version;
obj["versions"] = searcherManager.GetVersions(searcher, scheme, scoreDoc.Doc);
if (includeExplanation)
{
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
obj["explanation"] = explanation.ToString();
}
array.Add(obj);
}
return array;
}
private static JToken MakeResults(NuGetIndexSearcher searcher, TopDocs topDocs, int skip, int take, Query boostedQuery, IDictionary<string, int> rankings, NuGetSearcherManager manager)
{
JObject obj = new JObject();
obj.Add("TotalHits", topDocs.TotalHits);
string timestamp;
if (!searcher.IndexReader.CommitUserData.TryGetValue("commitTimeStamp", out timestamp))
{
timestamp = DateTime.MinValue.ToString();
}
obj.Add("IndexTimestampUtc",timestamp);
JArray array = new JArray();
Tuple<OpenBitSet, OpenBitSet> latestBitSets = manager.GetBitSets(searcher, null);
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
Tuple<int, int> downloadCounts = manager.GetDownloadCounts(document.Get("Id"), document.Get("Version"));
JObject packageObj = new JObject();
JObject registrationObj = new JObject();
registrationObj.Add("Id", document.Get("Id"));
registrationObj.Add("DownloadCount", downloadCounts.Item1);
registrationObj.Add("Owners", JArray.Parse(document.Get("Owners")));
packageObj.Add("PackageRegistration", registrationObj);
packageObj.Add("Version", document.Get("OriginalVersion"));
packageObj.Add("NormalizedVersion", document.Get("Version"));
packageObj.Add("Title", document.Get("Title"));
packageObj.Add("Description", document.Get("Description"));
packageObj.Add("Summary", document.Get("Summary"));
packageObj.Add("Authors", document.Get("Authors"));
packageObj.Add("Copyright", document.Get("Copyright"));
packageObj.Add("Language", document.Get("Language"));
packageObj.Add("Tags", document.Get("Tags"));
packageObj.Add("ReleaseNotes", document.Get("ReleaseNotes"));
packageObj.Add("ProjectUrl", document.Get("ProjectUrl"));
packageObj.Add("IconUrl", document.Get("IconUrl"));
packageObj.Add("IsLatestStable", latestBitSets.Item1.Get(scoreDoc.Doc));
packageObj.Add("IsLatest", latestBitSets.Item2.Get(scoreDoc.Doc));
packageObj.Add("Listed", bool.Parse(document.Get("Listed") ?? "true"));
packageObj.Add("Created", document.Get("OriginalCreated"));
packageObj.Add("Published", document.Get("OriginalPublished"));
packageObj.Add("LastUpdated", document.Get("OriginalPublished"));
packageObj.Add("LastEdited", document.Get("OriginalEditedDate"));
packageObj.Add("DownloadCount", downloadCounts.Item2);
packageObj.Add("FlattenedDependencies", document.Get("FlattenedDependencies"));
packageObj.Add("Dependencies", JArray.Parse(document.Get("Dependencies")));
packageObj.Add("SupportedFrameworks", JArray.Parse(document.Get("SupportedFrameworks")));
packageObj.Add("MinClientVersion", document.Get("MinClientVersion"));
packageObj.Add("Hash", document.Get("PackageHash"));
packageObj.Add("HashAlgorithm", document.Get("PackageHashAlgorithm"));
packageObj.Add("PackageFileSize", int.Parse(document.Get("PackageSize") ?? "0"));
packageObj.Add("LicenseUrl", document.Get("LicenseUrl"));
packageObj.Add("RequiresLicenseAcceptance", bool.Parse(document.Get("RequiresLicenseAcceptance") ?? "true"));
packageObj.Add("LicenseNames", document.Get("LicenseNames"));
packageObj.Add("LicenseReportUrl", document.Get("LicenseReportUrl"));
packageObj.Add("HideLicenseReport", bool.Parse(document.Get("HideLicenseReport") ?? "true")); //TODO: data is missing from index
array.Add(packageObj);
}
obj.Add("Data", array);
return obj;
}
static JToken MakeResult(NuGetIndexSearcher searcher, string scheme, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
{
JToken data = MakeResultData(searcher, scheme, topDocs, skip, take, searcherManager, includeExplanation, query);
JObject result = new JObject();
result.Add("@context", new JObject { { "@vocab", "http://schema.nuget.org/schema#" } });
result.Add("totalHits", topDocs.TotalHits);
result.Add("lastReopen", searcherManager.LastReopen.ToString("o"));
result.Add("index", searcherManager.IndexName);
result.Add("data", data);
return result;
}
