static JToken FindSearch(NuGetSearcherManager searcherManager, string id, string scheme)
{
if (id == null)
{
return(null);
}
IndexSearcher searcher = searcherManager.Get();
try
{
string analyzedId = id.ToLowerInvariant();
Query query = new TermQuery(new Term("Id", analyzedId));
TopDocs topDocs = searcher.Search(query, 1);
if (topDocs.TotalHits > 0)
{
Uri registrationBaseAddress = searcherManager.RegistrationBaseAddress[scheme];
JObject obj = new JObject();
obj["registration"] = new Uri(registrationBaseAddress, string.Format("{0}/index.json", id.ToLowerInvariant())).AbsoluteUri;
return(obj);
}
else
{
return(null);
}
}
finally
{
searcherManager.Release(searcher);
}
}
public static JToken QuerySearch(NuGetSearcherManager searcherManager, string scheme, string q, bool countOnly, string projectType, string supportedFramework, bool includePrerelease, int skip, int take, bool includeExplanation, bool applyFilter)
{
IndexSearcher searcher = searcherManager.Get();
try
{
Query query = MakeQuery(q, searcherManager);
TopDocs topDocs;
if (applyFilter)
{
Filter filter = searcherManager.GetFilter(includePrerelease, supportedFramework);
//TODO: uncomment these lines when we have an index that contains the appropriate @type field in every document
//Filter typeFilter = new CachingWrapperFilter(new TypeFilter("http://schema.nuget.org/schema#NuGetClassicPackage"));
//filter = new ChainedFilter(new Filter[] { filter, typeFilter }, ChainedFilter.Logic.AND);
topDocs = searcher.Search(query, filter, skip + take);
}
else
{
topDocs = searcher.Search(query, skip + take);
}
return(MakeResult(searcher, scheme, topDocs, skip, take, searcherManager, includeExplanation, query));
}
finally
{
searcherManager.Release(searcher);
}
}
public static Query MakeQuery(string q, NuGetSearcherManager searcherManager)
{
Query query = LuceneQueryCreator.Parse(q, false);
Query boostedQuery = new RankingScoreQuery(query, searcherManager.GetRankings());
return(boostedQuery);
}
public static void Search(JsonWriter jsonWriter,
NuGetSearcherManager searcherManager,
string scheme,
string q,
bool includePrerelease,
NuGetVersion semVerLevel,
int skip,
int take,
string feed,
bool includeExplanation)
{
var searcher = searcherManager.Get();
try
{
Query query = MakeSearchQuery(q, searcher);
Filter filter = null;
if (searcher.TryGetFilter(false, includePrerelease, semVerLevel, feed, out filter))
{
// Filter before running the query (make the search set smaller)
query = new FilteredQuery(query, filter);
}
TopDocs topDocs = searcher.Search(query, skip + take);
ResponseFormatter.WriteSearchResult(jsonWriter, searcher, scheme, topDocs, skip, take, includePrerelease, includeExplanation, semVerLevel, query);
}
finally
{
searcherManager.Release(searcher);
}
}
static JToken AutoCompleteSearch(NuGetSearcherManager searcherManager, string q, string id, string supportedFramework, bool includePrerelease, int skip, int take, bool includeExplanation)
{
IndexSearcher searcher = searcherManager.Get();
try
{
Filter filter = searcherManager.GetFilter(includePrerelease, supportedFramework);
if (q != null)
{
Query query = AutoCompleteMakeQuery(q, searcherManager);
TopDocs topDocs = searcher.Search(query, filter, skip + take);
return(AutoCompleteMakeResult(searcher, topDocs, skip, take, searcherManager, includeExplanation, query));
}
else
{
Query query = new TermQuery(new Term("Id", id.ToLowerInvariant()));
TopDocs topDocs = searcher.Search(query, filter, 1);
return(AutoCompleteMakeVersionResult(searcherManager, includePrerelease, topDocs));
}
}
finally
{
searcherManager.Release(searcher);
}
}
public static async Task TargetFrameworks(IOwinContext context, NuGetSearcherManager searcherManager)
{
IndexSearcher searcher = searcherManager.Get();
try
{
HashSet <string> targetFrameworks = new HashSet <string>();
IndexReader reader = searcher.IndexReader;
for (int i = 0; i < reader.MaxDoc; i++)
{
Document document = reader[i];
Field[] frameworks = document.GetFields("TargetFramework");
foreach (Field framework in frameworks)
{
targetFrameworks.Add(framework.StringValue);
}
}
JArray result = new JArray();
foreach (string targetFramework in targetFrameworks)
{
result.Add(targetFramework);
}
await ServiceHelpers.WriteResponse(context, System.Net.HttpStatusCode.OK, result);
}
finally
{
searcherManager.Release(searcher);
}
}
public static JToken Find(IOwinContext context, NuGetSearcherManager searcherManager)
{
string id = context.Request.Query["id"];
string scheme = context.Request.Uri.Scheme;
return(FindSearch(searcherManager, id, scheme));
}
public static JToken Query(IOwinContext context, NuGetSearcherManager searcherManager)
{
int skip;
if (!int.TryParse(context.Request.Query["skip"], out skip))
{
skip = 0;
}
int take;
if (!int.TryParse(context.Request.Query["take"], out take))
{
take = 20;
}
bool countOnly;
if (!bool.TryParse(context.Request.Query["countOnly"], out countOnly))
{
countOnly = false;
}
bool includePrerelease;
if (!bool.TryParse(context.Request.Query["prerelease"], out includePrerelease))
{
includePrerelease = false;
}
bool includeExplanation;
if (!bool.TryParse(context.Request.Query["explanation"], out includeExplanation))
{
includeExplanation = false;
}
bool applyFilter;
if (!bool.TryParse(context.Request.Query["filter"], out applyFilter))
{
applyFilter = true;
}
string projectType = context.Request.Query["projectType"] ?? string.Empty;
string supportedFramework = context.Request.Query["supportedFramework"];
string q = context.Request.Query["q"] ?? string.Empty;
string scheme = context.Request.Uri.Scheme;
return(QuerySearch(searcherManager, scheme, q, countOnly, projectType, supportedFramework, includePrerelease, skip, take, includeExplanation, applyFilter));
}
public static void AutoComplete(
JsonWriter jsonWriter,
NuGetSearcherManager searcherManager,
string q,
string id,
bool includePrerelease,
NuGetVersion semVerLevel,
int skip,
int take,
bool includeExplanation)
{
var searcher = searcherManager.Get();
try
{
Filter filter = null;
if (q != null)
{
Query query = MakeAutoCompleteQuery(q,
searcher.DocIdMapping,
searcher.Downloads,
searcher.Rankings,
searcher.QueryBoostingContext);
if (searcher.TryGetFilter(false, includePrerelease, semVerLevel, null, out filter))
{
// Filter before running the query (make the search set smaller)
query = new FilteredQuery(query, filter);
}
TopDocs topDocs = searcher.Search(query, skip + take);
ResponseFormatter.WriteAutoCompleteResult(jsonWriter, searcher, topDocs, skip, take, includeExplanation, query);
}
else
{
Query query = MakeAutoCompleteVersionQuery(id);
if (searcher.TryGetFilter(false, includePrerelease, semVerLevel, null, out filter))
{
// Filter before running the query (make the search set smaller)
query = new FilteredQuery(query, filter);
}
TopDocs topDocs = searcher.Search(query, 1);
ResponseFormatter.WriteAutoCompleteVersionResult(jsonWriter, searcher, includePrerelease, semVerLevel, topDocs);
}
}
finally
{
searcherManager.Release(searcher);
}
}
/// <summary>Initializes a <see cref="NuGetSearcherManager"/> instance.</summary>
/// <param name="directory">
/// Optionally, the Lucene directory to read the index from. If <c>null</c> is provided, the directory
/// implementation is determined based off of the configuration (<see cref="config"/>).
/// </param>
/// <param name="loader">
/// Optionally, the loader used to read the JSON data files. If <c>null</c> is provided, the loader
/// implementation is determined based off of the configuration (<see cref="config"/>).
/// </param>
/// <param name="config">
/// The configuration to read which primarily determines whether the resulting instance will read from the local
/// disk or from blob storage.
/// </param>
/// <param name="loggerFactory">
/// Optionally, the logger factory defined by the consuming application.
/// </param>
/// <returns>The resulting <see cref="NuGetSearcherManager"/> instance.</returns>
public static NuGetSearcherManager Create(IndexingConfiguration config,
ILoggerFactory loggerFactory,
Directory directory = null,
ILoader loader = null)
{
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
LoggerFactory = loggerFactory;
var logger = loggerFactory.CreateLogger <NuGetSearcherManager>();
// If a local Lucene directory has been specified, create a directory and loader for the specified directory.
var luceneDirectory = config.LocalLuceneDirectory;
if (!string.IsNullOrEmpty(luceneDirectory))
{
directory = directory ?? new SimpleFSDirectory(new DirectoryInfo(luceneDirectory));
loader = loader ?? new FileLoader(config.LocalDataDirectory);
}
IIndexDirectoryProvider indexProvider;
if (directory == null)
{
// If no directory has been provided, create a CloudIndexDirectoryProvider from the configuration.
indexProvider = new CloudIndexDirectoryProvider(config, logger);
}
else
{
// Use the specified directory to create a FixedIndexDirectoryProvider.
var indexContainerName = luceneDirectory ?? config.IndexContainer;
indexProvider = new LocalIndexDirectoryProvider(directory, indexContainerName);
}
// If a loader has been specified, use it.
// Otherwise, create a StorageLoader from the configuration.
loader = loader ?? new StorageLoader(config, logger);
var searcherManager = new NuGetSearcherManager(logger, indexProvider, loader, config.AuxiliaryDataRefreshRateSec, config.IndexReloadRateSec);
var registrationBaseAddress = config.RegistrationBaseAddress;
var semVer2RegistrationBaseAddress = config.SemVer2RegistrationBaseAddress;
searcherManager.RegistrationBaseAddresses.LegacyHttp = MakeRegistrationBaseAddress("http", registrationBaseAddress);
searcherManager.RegistrationBaseAddresses.LegacyHttps = MakeRegistrationBaseAddress("https", registrationBaseAddress);
searcherManager.RegistrationBaseAddresses.SemVer2Http = MakeRegistrationBaseAddress("http", semVer2RegistrationBaseAddress);
searcherManager.RegistrationBaseAddresses.SemVer2Https = MakeRegistrationBaseAddress("https", semVer2RegistrationBaseAddress);
return(searcherManager);
}
public static void Stats(JsonWriter jsonWriter, NuGetSearcherManager searcherManager)
{
var searcher = searcherManager.Get();
try
{
ResponseFormatter.WriteStatsResult(jsonWriter, searcher);
}
finally
{
searcherManager.Release(searcher);
}
}
public static JToken AutoComplete(IOwinContext context, NuGetSearcherManager searcherManager)
{
IndexSearcher searcher = searcherManager.Get();
try
{
int skip;
if (!int.TryParse(context.Request.Query["skip"], out skip))
{
skip = 0;
}
int take;
if (!int.TryParse(context.Request.Query["take"], out take))
{
take = 20;
}
bool includePrerelease;
if (!bool.TryParse(context.Request.Query["prerelease"], out includePrerelease))
{
includePrerelease = false;
}
bool includeExplanation = false;
if (!bool.TryParse(context.Request.Query["explanation"], out includeExplanation))
{
includeExplanation = false;
}
string supportedFramework = context.Request.Query["supportedFramework"];
string q = context.Request.Query["q"];
string id = context.Request.Query["id"];
if (q == null && id == null)
{
q = string.Empty;
}
return(AutoCompleteSearch(searcherManager, q, id, supportedFramework, includePrerelease, skip, take, includeExplanation));
}
finally
{
searcherManager.Release(searcher);
}
}
public NuGetIndexSearcher(
NuGetSearcherManager manager,
IndexReader reader,
IDictionary <string, string> commitUserData,
IDictionary <string, Filter> curatedFeeds,
Dictionary <LatestListedMask, Filter> latest,
IReadOnlyDictionary <string, int[]> docIdMapping,
Downloads downloads,
VersionResult[] versions,
RankingResult rankings,
QueryBoostingContext context,
OpenBitSet latestBitSet,
OpenBitSet latestStableBitSet,
OpenBitSet latestSemVer2BitSet,
OpenBitSet latestStableSemVer2BitSet,
OwnersResult owners,
HashSet <string> verifiedPackages)
: base(reader)
{
Manager = manager;
CommitUserData = commitUserData;
_curatedFeeds = new Dictionary <string, Filter>(curatedFeeds.Count);
foreach (var curatedFeedsFilter in curatedFeeds)
{
_curatedFeeds.Add(curatedFeedsFilter.Key, new CachingWrapperFilter(curatedFeedsFilter.Value));
}
_latest = latest;
DocIdMapping = docIdMapping;
Downloads = downloads;
Versions = versions;
Rankings = rankings;
LatestBitSet = latestBitSet;
LatestStableBitSet = latestStableBitSet;
LatestSemVer2BitSet = latestSemVer2BitSet;
LatestStableSemVer2BitSet = latestStableSemVer2BitSet;
Owners = owners;
VerifiedPackages = verifiedPackages;
QueryBoostingContext = context;
LastReopen = DateTime.UtcNow;
}
static Query AutoCompleteMakeQuery(string q, NuGetSearcherManager searcherManager)
{
if (string.IsNullOrEmpty(q))
{
return(new MatchAllDocsQuery());
}
QueryParser queryParser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, "IdAutocomplete", new PackageAnalyzer());
//TODO: we should be doing phrase queries to get the ordering right
//const int MAX_NGRAM_LENGTH = 8;
//q = (q.Length < MAX_NGRAM_LENGTH) ? q : q.Substring(0, MAX_NGRAM_LENGTH);
//string phraseQuery = string.Format("IdAutocompletePhrase:\"/ {0}\"~20", q);
//Query query = queryParser.Parse(phraseQuery);
Query query = queryParser.Parse(q);
Query boostedQuery = new RankingScoreQuery(query, searcherManager.GetRankings());
return(boostedQuery);
}
static JObject AutoCompleteMakeVersionResult(NuGetSearcherManager searcherManager, bool includePrerelease, TopDocs topDocs)
{
JObject result = new JObject();
result.Add("@context", new JObject {
{ "@vocab", "http://schema.nuget.org/schema#" }
});
result.Add("indexName", searcherManager.IndexName);
if (topDocs.TotalHits > 0)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[0];
JArray versions = searcherManager.GetVersionLists(scoreDoc.Doc);
result.Add("totalHits", versions.Count());
result["data"] = versions;
}
else
{
result.Add("totalHits", 0);
result["data"] = new JArray();
}
return(result);
}
static JToken MakeResult(IndexSearcher searcher, string scheme, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
{
JToken data = MakeResultData(searcher, scheme, topDocs, skip, take, searcherManager, includeExplanation, query);
JObject result = new JObject();
result.Add("@context", new JObject {
{ "@vocab", "http://schema.nuget.org/schema#" }
});
result.Add("totalHits", topDocs.TotalHits);
result.Add("lastReopen", searcherManager.LastReopen.ToString("o"));
result.Add("index", searcherManager.IndexName);
result.Add("data", data);
return(result);
}
public static JToken MakeResultData(IndexSearcher searcher, string scheme, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
{
Uri registrationBaseAddress = searcherManager.RegistrationBaseAddress[scheme];
JArray array = new JArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
string url = document.Get("Url");
string id = document.Get("Id");
string version = document.Get("Version");
JObject obj = new JObject();
obj["@id"] = new Uri(registrationBaseAddress, url).AbsoluteUri;
obj["@type"] = "Package";
obj["registration"] = new Uri(registrationBaseAddress, string.Format("{0}/index.json", id.ToLowerInvariant())).AbsoluteUri;
obj["id"] = id;
AddField(obj, document, "domain", "Domain");
AddField(obj, document, "description", "Description");
AddField(obj, document, "summary", "Summary");
AddField(obj, document, "title", "Title");
AddField(obj, document, "iconUrl", "IconUrl");
AddFieldAsArray(obj, document, "tags", "Tags");
AddFieldAsArray(obj, document, "authors", "Authors");
obj["version"] = version;
obj["versions"] = searcherManager.GetVersions(scheme, scoreDoc.Doc);
if (includeExplanation)
{
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
obj["explanation"] = explanation.ToString();
}
array.Add(obj);
}
return(array);
}
public static void Search(JsonWriter jsonWriter,
NuGetSearcherManager searcherManager,
string q,
bool countOnly,
bool includePrerelease,
NuGetVersion semVerLevel,
string sortBy,
int skip,
int take,
string feed,
bool ignoreFilter,
bool luceneQuery)
{
if (jsonWriter == null)
{
throw new ArgumentNullException(nameof(jsonWriter));
}
if (searcherManager == null)
{
throw new ArgumentNullException(nameof(searcherManager));
}
var searcher = searcherManager.Get();
try
{
// The old V2 search service would treat "id:" queries (~match) in the same way as it did "packageid:" (==match).
// If "id:" is in the query, replace it.
if (luceneQuery && !string.IsNullOrEmpty(q) && q.StartsWith("id:", StringComparison.OrdinalIgnoreCase))
{
q = "packageid:" + q.Substring(3);
}
// Build the query
Query query = NuGetQuery.MakeQuery(q, searcher.Owners);
// Build filter
bool includeUnlisted = ignoreFilter;
includePrerelease = ignoreFilter || includePrerelease;
feed = ignoreFilter ? null : feed;
var combinedQuery = new BooleanQuery();
combinedQuery.Add(query, Occur.SHOULD);
// Add this clause to the query here so we still respect semVerLevel that is passed when ignoring filters.
if (!SemVerHelpers.ShouldIncludeSemVer2Results(semVerLevel))
{
combinedQuery.Add(
NuGetQuery.ConstructClauseQuery(
new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30),
MetadataConstants.LuceneMetadata.SemVerLevelPropertyName,
new List <string> {
SemVerHelpers.SemVerLevelKeySemVer2
}),
Occur.MUST_NOT);
}
query = combinedQuery;
Filter filter = null;
if (!ignoreFilter && searcher.TryGetFilter(includeUnlisted, includePrerelease, semVerLevel, feed, out filter))
{
// Filter before running the query (make the search set smaller)
query = new FilteredQuery(query, filter);
}
if (countOnly)
{
DocumentCountImpl(jsonWriter, searcher, query);
}
else
{
ListDocumentsImpl(jsonWriter, searcher, query, sortBy, skip, take, semVerLevel);
}
}
finally
{
searcherManager.Release(searcher);
}
}
public static JToken AutoCompleteMakeResult(IndexSearcher searcher, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
{
JArray array = new JArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
string id = document.Get("Id");
array.Add(id);
}
JObject result = new JObject();
result.Add("@context", new JObject {
{ "@vocab", "http://schema.nuget.org/schema#" }
});
result.Add("totalHits", topDocs.TotalHits);
result.Add("indexName", searcherManager.IndexName);
result.Add("data", array);
if (includeExplanation)
{
JArray explanations = new JArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
explanations.Add(explanation.ToString());
}
result.Add("explanations", explanations);
}
return(result);
}