public static JToken QuerySearch(NuGetSearcherManager searcherManager, string q, bool countOnly, string projectType, string supportedFramework, bool includePrerelease, string sortBy, int skip, int take, bool ignoreFilter)
{
var searcher = searcherManager.Get();
try
{
Filter filter = ignoreFilter ? null : searcherManager.GetFilter(searcher, includePrerelease, null);
Query query = LuceneQueryCreator.Parse(q, false);
if (countOnly)
{
return DocumentCountImpl(searcher, query, filter);
}
else
{
IDictionary<string, int> rankings = searcherManager.GetRankings(projectType);
return ListDocumentsImpl(searcher, query, rankings, filter, sortBy, skip, take, searcherManager);
}
}
finally
{
searcherManager.Release(searcher);
}
}
private void TrackIndexMetrics(NuGetSearcherManager searcherManager, SearchTelemetryClient searchTelemetryClient)
{
var searcher = searcherManager.Get();
try
{
// Track number of documents in index
searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneNumDocs, searcher.IndexReader.NumDocs());
// Track time between Lucene commit and reopen
string temp;
if (searcher.CommitUserData.TryGetValue("commitTimeStamp", out temp))
{
var commitTimestamp = DateTimeOffset.Parse(temp, null, DateTimeStyles.AssumeUniversal);
searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneLoadLag,
(searcher.LastReopen - commitTimestamp.UtcDateTime).TotalSeconds,
new Dictionary <string, string>()
{
{ SearchTelemetryClient.MetricName.LuceneLastReopen, searcher.LastReopen.ToString("o") },
{ SearchTelemetryClient.MetricName.LuceneCommitTimestamp, commitTimestamp.UtcDateTime.ToString("o") }
});
}
}
finally
{
searcherManager.Release(searcher);
}
}
public static async Task RankingsAsync(IOwinContext context, NuGetSearcherManager searcherManager, ResponseWriter responseWriter)
{
await responseWriter.WriteResponseAsync(
context,
HttpStatusCode.OK,
jsonWriter => ServiceInfoImpl.Rankings(jsonWriter, searcherManager));
}
public static JToken QuerySearch(NuGetSearcherManager searcherManager, string scheme, string q, bool countOnly, string projectType, string supportedFramework, bool includePrerelease, int skip, int take, bool includeExplanation, bool applyFilter)
{
var searcher = searcherManager.Get();
try
{
Query query = MakeQuery(q, searcherManager);
TopDocs topDocs;
if (applyFilter)
{
Filter filter = searcherManager.GetFilter(searcher, includePrerelease, supportedFramework);
//TODO: uncomment these lines when we have an index that contains the appropriate @type field in every document
//Filter typeFilter = new CachingWrapperFilter(new TypeFilter("http://schema.nuget.org/schema#NuGetClassicPackage"));
//filter = new ChainedFilter(new Filter[] { filter, typeFilter }, ChainedFilter.Logic.AND);
topDocs = searcher.Search(query, filter, skip + take);
}
else
{
topDocs = searcher.Search(query, skip + take);
}
return MakeResult(searcher, scheme, topDocs, skip, take, searcherManager, includeExplanation, query);
}
finally
{
searcherManager.Release(searcher);
}
}
public static async Task Stats(IOwinContext context, NuGetSearcherManager searcherManager, ResponseWriter responseWriter)
{
await responseWriter.WriteResponseAsync(
context,
HttpStatusCode.OK,
jsonWriter => ServiceInfoImpl.Stats(jsonWriter, searcherManager));
}
public static void Search(JsonWriter jsonWriter,
NuGetSearcherManager searcherManager,
string scheme,
string q,
bool includePrerelease,
int skip,
int take,
string feed,
bool includeExplanation)
{
var searcher = searcherManager.Get();
try
{
Query query = MakeSearchQuery(q, searcher);
Filter filter = null;
if (searcher.TryGetFilter(false, includePrerelease, feed, out filter))
{
// Filter before running the query (make the search set smaller)
query = new FilteredQuery(query, filter);
}
TopDocs topDocs = searcher.Search(query, skip + take);
ResponseFormatter.WriteSearchResult(jsonWriter, searcher, scheme, topDocs, skip, take, includePrerelease, includeExplanation, query);
}
finally
{
searcherManager.Release(searcher);
}
}
public static async Task RankingsAsync(IOwinContext context, NuGetSearcherManager searcherManager)
{
await ResponseHelpers.WriteResponseAsync(
context,
HttpStatusCode.OK,
jsonWriter => ServiceInfoImpl.Rankings(jsonWriter, searcherManager));
}
public NuGetSearcherManager CreateSearcherManager()
{
NuGetSearcherManager searcherManager;
string luceneDirectory = System.Configuration.ConfigurationManager.AppSettings.Get("Local.Lucene.Directory");
if (!string.IsNullOrEmpty(luceneDirectory))
{
string dataDirectory = System.Configuration.ConfigurationManager.AppSettings.Get("Local.Data.Directory");
searcherManager = NuGetSearcherManager.CreateLocal(luceneDirectory, dataDirectory);
}
else
{
string storagePrimary = System.Configuration.ConfigurationManager.AppSettings.Get("Storage.Primary");
string searchIndexContainer = System.Configuration.ConfigurationManager.AppSettings.Get("Search.IndexContainer");
string searchDataContainer = System.Configuration.ConfigurationManager.AppSettings.Get("Search.DataContainer");
searcherManager = NuGetSearcherManager.CreateAzure(storagePrimary, searchIndexContainer, searchDataContainer);
}
string registrationBaseAddress = System.Configuration.ConfigurationManager.AppSettings.Get("Search.RegistrationBaseAddress");
searcherManager.RegistrationBaseAddress["http"] = MakeRegistrationBaseAddress("http", registrationBaseAddress);
searcherManager.RegistrationBaseAddress["https"] = MakeRegistrationBaseAddress("https", registrationBaseAddress);
return(searcherManager);
}
private static NuGetSearcherManager InitNuGetSearcherManager(string indexName)
{
var searcherManager = new NuGetSearcherManager(indexName, new Mock<ILogger>().Object, directory: null, loader: null, azureDirectorySynchronizer: null);
searcherManager.RegistrationBaseAddress[Constants.SchemeName] = new Uri(Constants.BaseUri);
return searcherManager;
}
public static JToken Query(IOwinContext context, NuGetSearcherManager searcherManager)
{
Trace.TraceInformation("Search: {0}", context.Request.QueryString);
string q = context.Request.Query["q"] ?? string.Empty;
string projectType = context.Request.Query["projectType"] ?? string.Empty;
string sortBy = context.Request.Query["sortBy"] ?? string.Empty;
bool includePrerelease;
if (!bool.TryParse(context.Request.Query["prerelease"], out includePrerelease))
{
includePrerelease = false;
}
bool countOnly;
if (!bool.TryParse(context.Request.Query["countOnly"], out countOnly))
{
countOnly = false;
}
string feed = context.Request.Query["feed"] ?? "none";
int skip;
if (!int.TryParse(context.Request.Query["skip"], out skip))
{
skip = 0;
}
int take;
if (!int.TryParse(context.Request.Query["take"], out take))
{
take = 20;
}
bool ignoreFilter;
if (!bool.TryParse(context.Request.Query["ignoreFilter"], out ignoreFilter))
{
ignoreFilter = false;
}
//string fxName = context.Request.Query["supportedFramework"];
//FrameworkName supportedFramework = null;
//if (!String.IsNullOrEmpty(fxName))
//{
// supportedFramework = VersionUtility.ParseFrameworkName(fxName);
//}
//if (supportedFramework == null || !SearcherManager.GetFrameworks().Contains(supportedFramework))
//{
// supportedFramework = FrameworksList.AnyFramework;
//}
string supportedFramework = string.Empty;
return QuerySearch(searcherManager, q, countOnly, projectType, supportedFramework, includePrerelease, sortBy, skip, take, ignoreFilter);
}
public static async Task FindAsync(IOwinContext context, NuGetSearcherManager searcherManager, ResponseWriter responseWriter)
{
var id = context.Request.Query["id"] ?? string.Empty;
var scheme = context.Request.Uri.Scheme;
await responseWriter.WriteResponseAsync(
context,
HttpStatusCode.OK,
jsonWriter => ServiceImpl.Find(jsonWriter, searcherManager, id, scheme));
}
public static async Task FindAsync(IOwinContext context, NuGetSearcherManager searcherManager)
{
var id = context.Request.Query["id"] ?? string.Empty;
var scheme = context.Request.Uri.Scheme;
await ResponseHelpers.WriteResponseAsync(
context,
HttpStatusCode.OK,
jsonWriter => ServiceImpl.Find(jsonWriter, searcherManager, id, scheme));
}
public static void Stats(JsonWriter jsonWriter, NuGetSearcherManager searcherManager)
{
var searcher = searcherManager.Get();
try
{
ResponseFormatter.WriteStatsResult(jsonWriter, searcher);
}
finally
{
searcherManager.Release(searcher);
}
}
public static void Search(JsonWriter jsonWriter, NuGetSearcherManager searcherManager, string q, bool countOnly, bool includePrerelease, string sortBy, int skip, int take, string feed, bool ignoreFilter, bool luceneQuery)
{
if (jsonWriter == null)
{
throw new ArgumentNullException(nameof(jsonWriter));
}
if (searcherManager == null)
{
throw new ArgumentNullException(nameof(searcherManager));
}
var searcher = searcherManager.Get();
try
{
// The old V2 search service would treat "id:" queries (~match) in the same way as it did "packageid:" (==match).
// If "id:" is in the query, replace it.
if (luceneQuery && !string.IsNullOrEmpty(q) && q.StartsWith("id:", StringComparison.OrdinalIgnoreCase))
{
q = "packageid:" + q.Substring(3);
}
// Build the query
Query query = NuGetQuery.MakeQuery(q, searcher.Owners);
// Build filter
bool includeUnlisted = ignoreFilter;
includePrerelease = ignoreFilter || includePrerelease;
feed = ignoreFilter ? null : feed;
Filter filter = null;
if (!ignoreFilter && searcher.TryGetFilter(includeUnlisted, includePrerelease, feed, out filter))
{
// Filter before running the query (make the search set smaller)
query = new FilteredQuery(query, filter);
}
if (countOnly)
{
DocumentCountImpl(jsonWriter, searcher, query);
}
else
{
ListDocumentsImpl(jsonWriter, searcher, query, sortBy, skip, take);
}
}
finally
{
searcherManager.Release(searcher);
}
}
public static JToken Query(IOwinContext context, NuGetSearcherManager searcherManager)
{
int skip;
if (!int.TryParse(context.Request.Query["skip"], out skip))
{
skip = 0;
}
int take;
if (!int.TryParse(context.Request.Query["take"], out take))
{
take = 20;
}
bool countOnly;
if (!bool.TryParse(context.Request.Query["countOnly"], out countOnly))
{
countOnly = false;
}
bool includePrerelease;
if (!bool.TryParse(context.Request.Query["prerelease"], out includePrerelease))
{
includePrerelease = false;
}
bool includeExplanation;
if (!bool.TryParse(context.Request.Query["explanation"], out includeExplanation))
{
includeExplanation = false;
}
bool applyFilter;
if (!bool.TryParse(context.Request.Query["filter"], out applyFilter))
{
applyFilter = true;
}
string projectType = context.Request.Query["projectType"] ?? string.Empty;
string supportedFramework = context.Request.Query["supportedFramework"];
string q = context.Request.Query["q"] ?? string.Empty;
string scheme = context.Request.Uri.Scheme;
return QuerySearch(searcherManager, scheme, q, countOnly, projectType, supportedFramework, includePrerelease, skip, take, includeExplanation, applyFilter);
}
public static async Task V3SearchAsync(IOwinContext context, NuGetSearcherManager searcherManager)
{
var skip = GetSkip(context);
var take = GetTake(context);
var includePrerelease = GetIncludePrerelease(context);
var includeExplanation = GetIncludeExplanation(context);
var q = context.Request.Query["q"] ?? string.Empty;
var feed = context.Request.Query["feed"];
var scheme = context.Request.Uri.Scheme;
await ResponseHelpers.WriteResponseAsync(
context,
HttpStatusCode.OK,
jsonWriter => ServiceImpl.Search(jsonWriter, searcherManager, scheme, q, includePrerelease, skip, take, feed, includeExplanation));
}
public static async Task V3SearchAsync(IOwinContext context, NuGetSearcherManager searcherManager, ResponseWriter responseWriter)
{
var skip = GetSkip(context);
var take = GetTake(context);
var includePrerelease = GetIncludePrerelease(context);
bool includeExplanation = GetBool(context, nameof(includeExplanation));
var q = context.Request.Query["q"] ?? string.Empty;
var feed = context.Request.Query["feed"];
var scheme = context.Request.Uri.Scheme;
await responseWriter.WriteResponseAsync(
context,
HttpStatusCode.OK,
jsonWriter => ServiceImpl.Search(jsonWriter, searcherManager, scheme, q, includePrerelease, skip, take, feed, includeExplanation));
}
public static async Task V2SearchAsync(IOwinContext context, NuGetSearcherManager searcherManager)
{
var skip = GetSkip(context);
var take = GetTake(context);
var ignoreFilter = GetIgnoreFilter(context);
var countOnly = GetCountOnly(context);
var includePrerelease = GetIncludePrerelease(context);
var q = context.Request.Query["q"] ?? string.Empty;
var sortBy = context.Request.Query["sortBy"] ?? string.Empty;
var feed = context.Request.Query["feed"];
await ResponseHelpers.WriteResponseAsync(
context,
HttpStatusCode.OK,
jsonWriter => GalleryServiceImpl.Search(jsonWriter, searcherManager, q, countOnly, includePrerelease, sortBy, skip, take, feed, ignoreFilter));
}
public static async Task V2SearchAsync(IOwinContext context, NuGetSearcherManager searcherManager)
{
var skip = GetSkip(context);
var take = GetTake(context);
var ignoreFilter = GetIgnoreFilter(context);
var countOnly = GetCountOnly(context);
var includePrerelease = GetIncludePrerelease(context);
var q = context.Request.Query["q"] ?? string.Empty;
var sortBy = context.Request.Query["sortBy"] ?? string.Empty;
var feed = context.Request.Query["feed"];
await ResponseHelpers.WriteResponseAsync(
context,
HttpStatusCode.OK,
jsonWriter => GalleryServiceImpl.Search(jsonWriter, searcherManager, q, countOnly, includePrerelease, sortBy, skip, take, feed, ignoreFilter));
}
public void Configuration(IAppBuilder app)
{
//app.UseErrorPage();
// search test console
app.Use(async (context, next) =>
{
if (String.Equals(context.Request.Path.Value, "/console", StringComparison.OrdinalIgnoreCase))
{
// Redirect to trailing slash to maintain relative links
context.Response.Redirect(context.Request.PathBase + context.Request.Path + "/");
context.Response.StatusCode = 301;
return;
}
else if (String.Equals(context.Request.Path.Value, "/console/", StringComparison.OrdinalIgnoreCase))
{
context.Request.Path = new PathString("/console/Index.html");
}
await next();
});
app.UseStaticFiles(new StaticFileOptions(new SharedOptions
{
RequestPath = new PathString("/console"),
FileSystem = new EmbeddedResourceFileSystem(typeof(Startup).Assembly, "NuGet.Services.BasicSearch.Console")
}));
// start the service running - the Lucene index needs to be reopened regularly on a background thread
string searchIndexRefresh = System.Configuration.ConfigurationManager.AppSettings.Get("Search.IndexRefresh") ?? "15";
int seconds;
if (!int.TryParse(searchIndexRefresh, out seconds))
{
seconds = 15;
}
_searcherManager = CreateSearcherManager();
_searcherManager.Open();
_gate = 0;
_timer = new Timer(new TimerCallback(ReopenCallback), 0, 0, seconds * 1000);
app.Run(Invoke);
}
public static async Task AutoCompleteAsync(IOwinContext context, NuGetSearcherManager searcherManager, ResponseWriter responseWriter)
{
var skip = GetSkip(context);
var take = GetTake(context);
var includePrerelease = GetIncludePrerelease(context);
bool explanation = GetBool(context, nameof(explanation));
var q = context.Request.Query["q"];
var id = context.Request.Query["id"];
if (q == null && id == null)
{
q = string.Empty;
}
await responseWriter.WriteResponseAsync(
context,
HttpStatusCode.OK,
jsonWriter => ServiceImpl.AutoComplete(jsonWriter, searcherManager, q, id, includePrerelease, skip, take, explanation));
}
private bool InitializeSearcherManager(IndexingConfiguration config, Directory directory, ILoader loader, ILoggerFactory loggerFactory)
{
const int maxRetries = 10;
try
{
Retry.Incremental(
() =>
{
var stopwatch = Stopwatch.StartNew();
_searcherManager = NuGetSearcherManager.Create(config, loggerFactory, directory, loader);
_searcherManager.Open();
stopwatch.Stop();
_searchTelemetryClient.TrackMetric(
SearchTelemetryClient.MetricName.SearchIndexReopenDuration, stopwatch.Elapsed.TotalSeconds);
TrackIndexMetrics(_searcherManager, _searchTelemetryClient);
},
shouldRetry: e =>
{
// Retry on any exception (but log it)
_logger.LogError("Startup: An error occurred initializing searcher manager. Going to retry... Exception: {Exception}",
e);
_searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.SearchIndexReopenFailed, 1);
return(true);
},
maxRetries: maxRetries,
waitIncrement: TimeSpan.FromSeconds(1));
return(true);
}
catch (Exception e)
{
_logger.LogCritical("Startup: A critical error occurred initializing searcher manager. Number of retries exhausted. Exception: {Exception}", e);
_searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.SearchIndexReopenFailed, maxRetries);
return(false);
}
}
public static void AutoComplete(JsonWriter jsonWriter, NuGetSearcherManager searcherManager, string q, string id, bool includePrerelease, int skip, int take, bool includeExplanation)
{
var searcher = searcherManager.Get();
try
{
Filter filter = null;
if (q != null)
{
Query query = MakeAutoCompleteQuery(q,
searcher.DocIdMapping,
searcher.Downloads,
searcher.Rankings,
searcher.QueryBoostingContext);
if (searcher.TryGetFilter(false, includePrerelease, null, out filter))
{
// Filter before running the query (make the search set smaller)
query = new FilteredQuery(query, filter);
}
TopDocs topDocs = searcher.Search(query, skip + take);
ResponseFormatter.WriteAutoCompleteResult(jsonWriter, searcher, topDocs, skip, take, includeExplanation, query);
}
else
{
Query query = MakeAutoCompleteVersionQuery(id);
if (searcher.TryGetFilter(false, includePrerelease, null, out filter))
{
// Filter before running the query (make the search set smaller)
query = new FilteredQuery(query, filter);
}
TopDocs topDocs = searcher.Search(query, 1);
ResponseFormatter.WriteAutoCompleteVersionResult(jsonWriter, searcher, includePrerelease, topDocs);
}
}
finally
{
searcherManager.Release(searcher);
}
}
public static async Task AutoCompleteAsync(IOwinContext context, NuGetSearcherManager searcherManager)
{
var skip = GetSkip(context);
var take = GetTake(context);
var includePrerelease = GetIncludePrerelease(context);
var includeExplanation = GetIncludeExplanation(context);
var q = context.Request.Query["q"];
var id = context.Request.Query["id"];
if (q == null && id == null)
{
q = string.Empty;
}
await ResponseHelpers.WriteResponseAsync(
context,
HttpStatusCode.OK,
jsonWriter => ServiceImpl.AutoComplete(jsonWriter, searcherManager, q, id, includePrerelease, skip, take, includeExplanation));
}
public void Configuration(IAppBuilder app)
{
app.UseErrorPage();
// search test console
app.Use(async(context, next) =>
{
if (String.Equals(context.Request.Path.Value, "/console", StringComparison.OrdinalIgnoreCase))
{
// Redirect to trailing slash to maintain relative links
context.Response.Redirect(context.Request.PathBase + context.Request.Path + "/");
context.Response.StatusCode = 301;
return;
}
else if (String.Equals(context.Request.Path.Value, "/console/", StringComparison.OrdinalIgnoreCase))
{
context.Request.Path = new PathString("/console/Index.html");
}
await next();
});
app.UseStaticFiles(new StaticFileOptions(new SharedOptions
{
RequestPath = new PathString("/console"),
FileSystem = new EmbeddedResourceFileSystem(typeof(Startup).Assembly, "NuGet.Services.BasicSearch.Console")
}));
// start the service running - the Lucene index needs to be reopened regularly on a background thread
_searcherManager = CreateSearcherManager();
_searcherManager.Open();
_gate = 0;
_timer = new Timer(new TimerCallback(ReopenCallback), 0, 0, 180 * 1000);
app.Run(Invoke);
}
async Task CreateLuceneIndex(Uri catalogIndex, Func <StorageHttpMessageHandler> handlerFunc, CancellationToken cancellationToken)
{
Lucene.Net.Store.Directory luceneDirectory = new RAMDirectory();
var collector = new SearchIndexFromCatalogCollector(
catalogIndex,
luceneDirectory,
null,
handlerFunc);
await collector.Run(
new MemoryCursor(DateTime.MinValue.ToUniversalTime()),
new MemoryCursor(DateTime.MaxValue.ToUniversalTime()),
cancellationToken);
ILogger logger = new DebugLogger("Lucene");
ILoader loader = new AuxillaryIndexLoader();
SearcherManager = new NuGetSearcherManager("memory", logger, luceneDirectory, loader);
SearcherManager.RegistrationBaseAddress["http"] = new Uri(_baseAddress, Registration);
SearcherManager.RegistrationBaseAddress["https"] = new Uri(_baseAddress, Registration);
SearcherManager.Open();
}
public NuGetIndexSearcher(
NuGetSearcherManager manager,
IndexReader reader,
IDictionary<string, string> commitUserData,
IDictionary<string, Filter> curatedFeeds,
Filter[][] latest,
IReadOnlyDictionary<string, int[]> docIdMapping,
Downloads downloads,
VersionResult[] versions,
RankingResult rankings,
QueryBoostingContext context,
OpenBitSet latestBitSet,
OpenBitSet latestStableBitSet,
OwnersResult owners)
: base(reader)
{
Manager = manager;
CommitUserData = commitUserData;
_curatedFeeds = new Dictionary<string, Filter>(curatedFeeds.Count);
foreach (var curatedFeedsFilter in curatedFeeds)
{
_curatedFeeds.Add(curatedFeedsFilter.Key, new CachingWrapperFilter(curatedFeedsFilter.Value));
}
_latest = latest;
DocIdMapping = docIdMapping;
Downloads = downloads;
Versions = versions;
Rankings = rankings;
LatestBitSet = latestBitSet;
LatestStableBitSet = latestStableBitSet;
Owners = owners;
QueryBoostingContext = context;
LastReopen = DateTime.UtcNow;
}
public static JToken AutoCompleteMakeResult(IndexSearcher searcher, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
{
JArray array = new JArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
string id = document.Get("Id");
array.Add(id);
}
JObject result = new JObject();
result.Add("@context", new JObject { { "@vocab", "http://schema.nuget.org/schema#" } });
result.Add("totalHits", topDocs.TotalHits);
result.Add("indexName", searcherManager.IndexName);
result.Add("data", array);
if (includeExplanation)
{
JArray explanations = new JArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
explanations.Add(explanation.ToString());
}
result.Add("explanations", explanations);
}
return result;
}
public static void Find(JsonWriter jsonWriter, NuGetSearcherManager searcherManager, string id, string scheme)
{
var searcher = searcherManager.Get();
try
{
Query query = MakeFindQuery(id);
TopDocs topDocs = searcher.Search(query, 1);
ResponseFormatter.WriteFindResult(jsonWriter, searcher, scheme, topDocs);
}
finally
{
searcherManager.Release(searcher);
}
}
public static JToken MakeResultData(NuGetIndexSearcher searcher, string scheme, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
{
Uri registrationBaseAddress = searcherManager.RegistrationBaseAddress[scheme];
JArray array = new JArray();
for (int i = skip; i < Math.Min(skip + take, topDocs.ScoreDocs.Length); i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document document = searcher.Doc(scoreDoc.Doc);
string url = document.Get("Url");
string id = document.Get("Id");
string version = document.Get("Version");
JObject obj = new JObject();
obj["@id"] = new Uri(registrationBaseAddress, url).AbsoluteUri;
obj["@type"] = "Package";
obj["registration"] = new Uri(registrationBaseAddress, string.Format("{0}/index.json", id.ToLowerInvariant())).AbsoluteUri;
obj["id"] = id;
AddField(obj, document, "domain", "Domain");
AddField(obj, document, "description", "Description");
AddField(obj, document, "summary", "Summary");
AddField(obj, document, "title", "Title");
AddField(obj, document, "iconUrl", "IconUrl");
AddField(obj, document, "licenseUrl", "LicenseUrl");
AddField(obj, document, "projectUrl", "ProjectUrl");
AddFieldAsArray(obj, document, "tags", "Tags");
AddFieldAsArray(obj, document, "authors", "Authors");
obj["version"] = version;
obj["versions"] = searcherManager.GetVersions(searcher, scheme, scoreDoc.Doc);
if (includeExplanation)
{
Explanation explanation = searcher.Explain(query, scoreDoc.Doc);
obj["explanation"] = explanation.ToString();
}
array.Add(obj);
}
return array;
}
static JToken MakeResult(NuGetIndexSearcher searcher, string scheme, TopDocs topDocs, int skip, int take, NuGetSearcherManager searcherManager, bool includeExplanation, Query query)
{
JToken data = MakeResultData(searcher, scheme, topDocs, skip, take, searcherManager, includeExplanation, query);
JObject result = new JObject();
result.Add("@context", new JObject { { "@vocab", "http://schema.nuget.org/schema#" } });
result.Add("totalHits", topDocs.TotalHits);
result.Add("lastReopen", searcherManager.LastReopen.ToString("o"));
result.Add("index", searcherManager.IndexName);
result.Add("data", data);
return result;
}
private void TrackIndexMetrics(NuGetSearcherManager searcherManager, SearchTelemetryClient searchTelemetryClient)
{
var searcher = searcherManager.Get();
try
{
// Track number of documents in index
searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneNumDocs, searcher.IndexReader.NumDocs());
// Track time between Lucene commit and reopen
string temp;
if (searcher.CommitUserData.TryGetValue("commitTimeStamp", out temp))
{
var commitTimestamp = DateTimeOffset.Parse(temp, null, DateTimeStyles.AssumeUniversal);
searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneLoadLag,
(searcher.LastReopen - commitTimestamp.UtcDateTime).TotalSeconds,
new Dictionary<string, string>()
{
{ SearchTelemetryClient.MetricName.LuceneLastReopen, searcher.LastReopen.ToString("o") },
{ SearchTelemetryClient.MetricName.LuceneCommitTimestamp, commitTimestamp.UtcDateTime.ToString("o") }
});
}
}
finally
{
searcherManager.Release(searcher);
}
}
private bool InitializeSearcherManager(IConfiguration configuration, Directory directory, ILoader loader, ILoggerFactory loggerFactory)
{
const int maxRetries = 10;
try
{
Retry.Incremental(
() =>
{
var stopwatch = Stopwatch.StartNew();
_searcherManager = NuGetSearcherManager.Create(configuration, loggerFactory, directory, loader);
_searcherManager.Open();
stopwatch.Stop();
_searchTelemetryClient.TrackMetric(
SearchTelemetryClient.MetricName.SearchIndexReopenDuration, stopwatch.Elapsed.TotalSeconds);
TrackIndexMetrics(_searcherManager, _searchTelemetryClient);
},
shouldRetry: e =>
{
// Retry on any exception (but log it)
_logger.LogError("Startup: An error occurred initializing searcher manager. Going to retry...", e);
_searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.SearchIndexReopenFailed, 1);
return true;
},
maxRetries: maxRetries,
waitIncrement: TimeSpan.FromSeconds(1));
return true;
}
catch (Exception e)
{
_logger.LogCritical("Startup: A critical error occurred initializing searcher manager. Number of retries exhausted.", e);
_searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.SearchIndexReopenFailed, maxRetries);
return false;
}
}
public static Query MakeQuery(string q, NuGetSearcherManager searcherManager)
{
Query query = LuceneQueryCreator.Parse(q, false);
Query boostedQuery = new RankingScoreQuery(query, searcherManager.GetRankings());
return boostedQuery;
}
static JToken FindSearch(NuGetSearcherManager searcherManager, string id, string scheme)
{
if (id == null)
{
return null;
}
var searcher = searcherManager.Get();
try
{
string analyzedId = id.ToLowerInvariant();
Query query = new TermQuery(new Term("Id", analyzedId));
TopDocs topDocs = searcher.Search(query, 1);
if (topDocs.TotalHits > 0)
{
Uri registrationBaseAddress = searcherManager.RegistrationBaseAddress[scheme];
JObject obj = new JObject();
obj["registration"] = new Uri(registrationBaseAddress, string.Format("{0}/index.json", id.ToLowerInvariant())).AbsoluteUri;
return obj;
}
else
{
return null;
}
}
finally
{
searcherManager.Release(searcher);
}
}
public static JToken AutoComplete(IOwinContext context, NuGetSearcherManager searcherManager)
{
var searcher = searcherManager.Get();
try
{
int skip;
if (!int.TryParse(context.Request.Query["skip"], out skip))
{
skip = 0;
}
int take;
if (!int.TryParse(context.Request.Query["take"], out take))
{
take = 20;
}
bool includePrerelease;
if (!bool.TryParse(context.Request.Query["prerelease"], out includePrerelease))
{
includePrerelease = false;
}
bool includeExplanation;
if (!bool.TryParse(context.Request.Query["explanation"], out includeExplanation))
{
includeExplanation = false;
}
string supportedFramework = context.Request.Query["supportedFramework"];
string q = context.Request.Query["q"];
string id = context.Request.Query["id"];
if (q == null && id == null)
{
q = string.Empty;
}
return AutoCompleteSearch(searcherManager, q, id, supportedFramework, includePrerelease, skip, take, includeExplanation);
}
finally
{
searcherManager.Release(searcher);
}
}
public static JToken Find(IOwinContext context, NuGetSearcherManager searcherManager)
{
string id = context.Request.Query["id"];
string scheme = context.Request.Uri.Scheme;
return FindSearch(searcherManager, id, scheme);
}
public static async Task Stats(IOwinContext context, NuGetSearcherManager searcherManager)
{
await ResponseHelpers.WriteResponseAsync(
context,
HttpStatusCode.OK,
jsonWriter => ServiceInfoImpl.Stats(jsonWriter, searcherManager));
}
static Query AutoCompleteMakeQuery(string q, NuGetSearcherManager searcherManager)
{
if (string.IsNullOrEmpty(q))
{
return new MatchAllDocsQuery();
}
QueryParser queryParser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, "IdAutocomplete", new PackageAnalyzer());
//TODO: we should be doing phrase queries to get the ordering right
//const int MAX_NGRAM_LENGTH = 8;
//q = (q.Length < MAX_NGRAM_LENGTH) ? q : q.Substring(0, MAX_NGRAM_LENGTH);
//string phraseQuery = string.Format("IdAutocompletePhrase:\"/ {0}\"~20", q);
//Query query = queryParser.Parse(phraseQuery);
Query query = queryParser.Parse(q);
Query boostedQuery = new RankingScoreQuery(query, searcherManager.GetRankings());
return boostedQuery;
}
static JObject AutoCompleteMakeVersionResult(NuGetSearcherManager searcherManager, bool includePrerelease, TopDocs topDocs)
{
var searcher = searcherManager.Get();
try
{
JObject result = new JObject();
result.Add("@context", new JObject { { "@vocab", "http://schema.nuget.org/schema#" } });
result.Add("indexName", searcherManager.IndexName);
if (topDocs.TotalHits > 0)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[0];
JArray versions = searcherManager.GetVersionLists(searcher, scoreDoc.Doc);
result.Add("totalHits", versions.Count());
result["data"] = versions;
}
else
{
result.Add("totalHits", 0);
result["data"] = new JArray();
}
return result;
}
finally
{
searcherManager.Release(searcher);
}
}
static JToken AutoCompleteSearch(NuGetSearcherManager searcherManager, string q, string id, string supportedFramework, bool includePrerelease, int skip, int take, bool includeExplanation)
{
var searcher = searcherManager.Get();
try
{
Filter filter = searcherManager.GetFilter(searcher, includePrerelease, supportedFramework);
if (q != null)
{
Query query = AutoCompleteMakeQuery(q, searcherManager);
TopDocs topDocs = searcher.Search(query, filter, skip + take);
return AutoCompleteMakeResult(searcher, topDocs, skip, take, searcherManager, includeExplanation, query);
}
else
{
Query query = new TermQuery(new Term("Id", id.ToLowerInvariant()));
TopDocs topDocs = searcher.Search(query, filter, 1);
return AutoCompleteMakeVersionResult(searcherManager, includePrerelease, topDocs);
}
}
finally
{
searcherManager.Release(searcher);
}
}
