public static async Task Execute(IOwinContext context, PackageSearcherManager searcherManager)
{
Trace.TraceInformation("Range: {0}", context.Request.QueryString);
string min = context.Request.Query["min"];
string max = context.Request.Query["max"];
string content = "[]";
int minKey;
int maxKey;
if (min != null && max != null && int.TryParse(min, out minKey) && int.TryParse(max, out maxKey))
{
Trace.TraceInformation("Searcher.KeyRangeQuery(..., {0}, {1})", minKey, maxKey);
content = Searcher.KeyRangeQuery(searcherManager, minKey, maxKey);
}
context.Response.Headers.Add("Pragma", new[] { "no-cache" });
context.Response.Headers.Add("Cache-Control", new[] { "no-cache" });
context.Response.Headers.Add("Expires", new[] { "0" });
context.Response.ContentType = "application/json";
await context.Response.WriteAsync(content);
}
public static string GetSegments(PackageSearcherManager searcherManager)
{
searcherManager.MaybeReopen();
IndexSearcher searcher = searcherManager.Get();
try
{
IndexReader indexReader = searcher.IndexReader;
JArray segments = new JArray();
foreach (ReadOnlySegmentReader segmentReader in indexReader.GetSequentialSubReaders())
{
JObject segmentInfo = new JObject();
segmentInfo.Add("segment", segmentReader.SegmentName);
segmentInfo.Add("documents", segmentReader.NumDocs());
segments.Add(segmentInfo);
}
return segments.ToString();
}
finally
{
searcherManager.Release(searcher);
}
}
public static string KeyRangeQuery(PackageSearcherManager searcherManager, int minKey, int maxKey)
{
// for range queries we always want the IndexReader to be absolutely up to date
searcherManager.MaybeReopen();
IndexSearcher searcher = searcherManager.Get();
try
{
NumericRangeQuery<int> numericRangeQuery = NumericRangeQuery.NewIntRange("Key", minKey, maxKey, true, true);
List<DocumentKey> pairs = new List<DocumentKey>();
searcher.Search(numericRangeQuery, new KeyCollector(pairs));
// Group by key
IEnumerable<IGrouping<int, DocumentKey>> groups = pairs.GroupBy(p => p.PackageKey);
// De-duplicate
IEnumerable<DocumentKey> deduped = groups.Select(g => g.First());
JObject keys = new JObject();
keys.Add(deduped.Select(p => new JProperty(p.PackageKey.ToString(), p.Checksum)));
return keys.ToString();
}
finally
{
searcherManager.Release(searcher);
}
}
public void Configuration(IAppBuilder app)
{
var instrumentationKey = System.Configuration.ConfigurationManager.AppSettings.Get("Telemetry.InstrumentationKey");
if (!string.IsNullOrEmpty(instrumentationKey))
{
// set it as early as possible to avoid losing telemetry
TelemetryConfiguration.Active.InstrumentationKey = instrumentationKey;
}
_searcherManager = CreateSearcherManager();
//test console
app.Use(async (context, next) =>
{
if (string.Equals(context.Request.Path.Value, "/console", StringComparison.OrdinalIgnoreCase))
{
context.Response.Redirect(context.Request.PathBase + context.Request.Path + "/");
context.Response.StatusCode = 301;
return;
}
else if (string.Equals(context.Request.Path.Value, "/console/", StringComparison.OrdinalIgnoreCase))
{
context.Request.Path = new PathString("/console/Index.html");
}
await next();
});
app.UseStaticFiles(new StaticFileOptions(new SharedOptions
{
RequestPath = new PathString("/console"),
FileSystem = new EmbeddedResourceFileSystem(typeof(QueryMiddleware).Assembly, "NuGet.Services.Search.Console")
}));
app.Run(Invoke);
}
public void ReloadIndex()
{
SearchServiceEventSource.Log.ReloadingIndex();
PackageSearcherManager newIndex = SearcherManagerBuilder();
Interlocked.Exchange(ref _searcherManager, newIndex);
SearchServiceEventSource.Log.ReloadedIndex();
}
public static async Task Execute(IOwinContext context, PackageSearcherManager SearcherManager)
{
Trace.TraceInformation("Diag");
context.Response.Headers.Add("Pragma", new[] { "no-cache" });
context.Response.Headers.Add("Cache-Control", new[] { "no-cache" });
context.Response.Headers.Add("Expires", new[] { "0" });
context.Response.ContentType = "application/json";
await context.Response.WriteAsync(IndexAnalyzer.Analyze(SearcherManager));
}
public static string Analyze(PackageSearcherManager searcherManager)
{
searcherManager.MaybeReopen();
IndexSearcher searcher = searcherManager.Get();
try
{
IndexReader indexReader = searcher.IndexReader;
JObject report = new JObject();
report.Add("NumDocs", indexReader.NumDocs());
report.Add("SearcherManagerIdentity", searcherManager.Id.ToString());
AzureDirectory azDir = indexReader.Directory() as AzureDirectory;
if (azDir != null)
{
report.Add("Index", azDir.BlobContainer.Name);
}
else
{
SimpleFSDirectory fsDir = indexReader.Directory() as SimpleFSDirectory;
if (fsDir != null)
{
report.Add("Index", fsDir.Directory.Name);
}
}
report.Add("RankingsUpdated", searcherManager.RankingsUpdatedUtc);
report.Add("DownloadCountsUpdated", searcherManager.DownloadCountsUpdatedUtc);
if (indexReader.CommitUserData != null)
{
JObject commitUserdata = new JObject();
foreach (KeyValuePair<string, string> userData in indexReader.CommitUserData)
{
commitUserdata.Add(userData.Key, userData.Value);
}
report.Add("CommitUserData", commitUserdata);
}
// Moved segments to their own command since they can take a while to calculate in Azure
return report.ToString();
}
finally
{
searcherManager.Release(searcher);
}
}
private PackageSearcherManager GetSearcherManager(SearchConfiguration config)
{
if (!String.IsNullOrEmpty(config.IndexPath))
{
return(PackageSearcherManager.CreateLocal(config.IndexPath));
}
else
{
return(PackageSearcherManager.CreateAzure(
Configuration.Storage.Primary,
config.IndexContainer,
config.DataContainer));
}
}
public static string Search(PackageSearcherManager searcherManager, string q, bool countOnly, string projectType, bool includePrerelease, string feed, string sortBy, int skip, int take, bool includeExplanation, bool ignoreFilter)
{
return Search(
searcherManager,
LuceneQueryCreator.Parse(q, true),
countOnly,
projectType,
includePrerelease,
feed,
sortBy,
skip,
take,
includeExplanation,
ignoreFilter);
}
private static PackageSearcherManager GetSearcherManager()
{
var searchIndexPath = System.Configuration.ConfigurationManager.AppSettings.Get("Search.IndexPath");
if (!string.IsNullOrEmpty(searchIndexPath))
{
return(PackageSearcherManager.CreateLocal(searchIndexPath));
}
else
{
return(PackageSearcherManager.CreateAzure(
System.Configuration.ConfigurationManager.AppSettings.Get("Storage.Primary"),
System.Configuration.ConfigurationManager.AppSettings.Get("Search.IndexContainer"),
System.Configuration.ConfigurationManager.AppSettings.Get("Search.DataContainer")));
}
}
public static async Task Execute(IOwinContext context, PackageSearcherManager searcherManager)
{
Trace.TraceInformation("Range: {0}", context.Request.QueryString);
string min = context.Request.Query["min"];
string max = context.Request.Query["max"];
string content = "[]";
int minKey;
int maxKey;
if (min != null && max != null && int.TryParse(min, out minKey) && int.TryParse(max, out maxKey))
{
Trace.TraceInformation("Searcher.KeyRangeQuery(..., {0}, {1})", minKey, maxKey);
content = Searcher.KeyRangeQuery(searcherManager, minKey, maxKey);
}
context.Response.Headers.Add("Pragma", new[] { "no-cache" });
context.Response.Headers.Add("Cache-Control", new[] { "no-cache" });
context.Response.Headers.Add("Expires", new[] { "0" });
context.Response.ContentType = "application/json";
await context.Response.WriteAsync(content);
}
public void Configuration(IAppBuilder app)
{
var instrumentationKey = System.Configuration.ConfigurationManager.AppSettings.Get("Telemetry.InstrumentationKey");
if (!string.IsNullOrEmpty(instrumentationKey))
{
// set it as early as possible to avoid losing telemetry
TelemetryConfiguration.Active.InstrumentationKey = instrumentationKey;
}
_searcherManager = CreateSearcherManager();
//test console
app.Use(async(context, next) =>
{
if (string.Equals(context.Request.Path.Value, "/console", StringComparison.OrdinalIgnoreCase))
{
context.Response.Redirect(context.Request.PathBase + context.Request.Path + "/");
context.Response.StatusCode = 301;
return;
}
else if (string.Equals(context.Request.Path.Value, "/console/", StringComparison.OrdinalIgnoreCase))
{
context.Request.Path = new PathString("/console/Index.html");
}
await next();
});
app.UseStaticFiles(new StaticFileOptions(new SharedOptions
{
RequestPath = new PathString("/console"),
FileSystem = new EmbeddedResourceFileSystem(typeof(QueryMiddleware).Assembly, "NuGet.Services.Search.Console")
}));
app.Run(Invoke);
}
private static string ListDocumentsImpl(IndexSearcher searcher, Query query, IDictionary<string, int> rankings, Filter filter, string sortBy, int skip, int take, bool includeExplanation, PackageSearcherManager manager)
{
Query boostedQuery = new RankingScoreQuery(query, rankings);
int nDocs = GetDocsCount(skip, take);
Sort sort = GetSort(sortBy);
Stopwatch sw = new Stopwatch();
sw.Start();
TopDocs topDocs = (sort == null) ?
searcher.Search(boostedQuery, filter, nDocs) :
searcher.Search(boostedQuery, filter, nDocs, sort);
sw.Stop();
sw.Stop();
return MakeResults(searcher, topDocs, skip, take, includeExplanation, boostedQuery, sw.ElapsedMilliseconds, rankings, manager);
}
public static string GetDistinctStoredFieldNames(PackageSearcherManager searcherManager)
{
searcherManager.MaybeReopen();
IndexSearcher searcher = searcherManager.Get();
try
{
IndexReader indexReader = searcher.IndexReader;
HashSet<string> distinctFieldNames = new HashSet<string>();
for (int i = 0; i < indexReader.MaxDoc; i++)
{
if (!indexReader.IsDeleted(i))
{
Document document = indexReader.Document(i);
IList<IFieldable> fields = document.GetFields();
foreach (IFieldable field in fields)
{
distinctFieldNames.Add(field.Name);
}
}
}
JArray array = new JArray();
foreach (string fieldName in distinctFieldNames)
{
array.Add(fieldName);
}
return array.ToString();
}
finally
{
searcherManager.Release(searcher);
}
}
public static async Task Execute(IOwinContext context, PackageSearcherManager searcherManager)
{
Trace.TraceInformation("Search: {0}", context.Request.QueryString);
string q = context.Request.Query["q"] ?? string.Empty;
string projectType = context.Request.Query["projectType"] ?? string.Empty;
string sortBy = context.Request.Query["sortBy"] ?? string.Empty;
bool luceneQuery;
if (!bool.TryParse(context.Request.Query["luceneQuery"], out luceneQuery))
{
luceneQuery = true;
}
bool includePrerelease;
if (!bool.TryParse(context.Request.Query["prerelease"], out includePrerelease))
{
includePrerelease = false;
}
bool countOnly;
if (!bool.TryParse(context.Request.Query["countOnly"], out countOnly))
{
countOnly = false;
}
string feed = context.Request.Query["feed"] ?? "none";
int skip;
if (!int.TryParse(context.Request.Query["skip"], out skip))
{
skip = 0;
}
int take;
if (!int.TryParse(context.Request.Query["take"], out take))
{
take = 20;
}
bool includeExplanation;
if (!bool.TryParse(context.Request.Query["explanation"], out includeExplanation))
{
includeExplanation = false;
}
bool ignoreFilter;
if (!bool.TryParse(context.Request.Query["ignoreFilter"], out ignoreFilter))
{
ignoreFilter = false;
}
string fxName = context.Request.Query["supportedFramework"];
FrameworkName supportedFramework = null;
if (!string.IsNullOrEmpty(fxName))
{
supportedFramework = VersionUtility.ParseFrameworkName(fxName);
}
if (supportedFramework == null || !searcherManager.GetFrameworks().Contains(supportedFramework))
{
supportedFramework = FrameworksList.AnyFramework;
}
Query query;
string content;
int maxAgeSeconds = 0;
try
{
query = LuceneQueryCreator.Parse(q, luceneQuery);
if (query == null)
{
query = new MatchAllDocsQuery();
maxAgeSeconds = 3600;
}
if (!ignoreFilter && !luceneQuery)
{
string facet = includePrerelease ?
Facets.LatestPrereleaseVersion(supportedFramework) :
Facets.LatestStableVersion(supportedFramework);
var newQuery = new BooleanQuery();
newQuery.Add(query, Occur.MUST);
newQuery.Add(new TermQuery(new Term("Facet", facet)), Occur.MUST);
query = newQuery;
}
string args = string.Format("Searcher.Search(..., {0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10})", q, countOnly, projectType, includePrerelease, feed, sortBy, skip, take, includeExplanation, ignoreFilter, luceneQuery);
Trace.TraceInformation(args);
content = Searcher.Search(
searcherManager,
query,
countOnly,
projectType,
includePrerelease,
feed,
sortBy,
skip,
take,
includeExplanation,
ignoreFilter);
JObject result = JObject.Parse(content);
result["answeredBy"] = "Search";
}
catch (Lucene.Net.QueryParsers.ParseException)
{
context.Response.StatusCode = 400;
content = "Invalid query";
}
finally
{
context.Response.Headers.Add("Cache-Control", new[] { string.Format("private, max-age={0}", maxAgeSeconds) });
context.Response.ContentType = "application/json";
}
await context.Response.WriteAsync(content);
}
public static async Task Execute(IOwinContext context, PackageSearcherManager searcherManager)
{
Trace.TraceInformation("Search: {0}", context.Request.QueryString);
string q = context.Request.Query["q"] ?? string.Empty;
string projectType = context.Request.Query["projectType"] ?? string.Empty;
string sortBy = context.Request.Query["sortBy"] ?? string.Empty;
bool luceneQuery;
if (!bool.TryParse(context.Request.Query["luceneQuery"], out luceneQuery))
{
luceneQuery = true;
}
bool includePrerelease;
if (!bool.TryParse(context.Request.Query["prerelease"], out includePrerelease))
{
includePrerelease = false;
}
bool countOnly;
if (!bool.TryParse(context.Request.Query["countOnly"], out countOnly))
{
countOnly = false;
}
string feed = context.Request.Query["feed"] ?? "none";
int skip;
if (!int.TryParse(context.Request.Query["skip"], out skip))
{
skip = 0;
}
int take;
if (!int.TryParse(context.Request.Query["take"], out take))
{
take = 20;
}
bool includeExplanation;
if (!bool.TryParse(context.Request.Query["explanation"], out includeExplanation))
{
includeExplanation = false;
}
bool ignoreFilter;
if (!bool.TryParse(context.Request.Query["ignoreFilter"], out ignoreFilter))
{
ignoreFilter = false;
}
string fxName = context.Request.Query["supportedFramework"];
FrameworkName supportedFramework = null;
if (!string.IsNullOrEmpty(fxName))
{
supportedFramework = VersionUtility.ParseFrameworkName(fxName);
}
if (supportedFramework == null || !searcherManager.GetFrameworks().Contains(supportedFramework))
{
supportedFramework = FrameworksList.AnyFramework;
}
Query query;
string content;
int maxAgeSeconds = 0;
try
{
query = LuceneQueryCreator.Parse(q, luceneQuery);
if (query == null)
{
query = new MatchAllDocsQuery();
maxAgeSeconds = 3600;
}
if (!ignoreFilter && !luceneQuery)
{
string facet = includePrerelease ?
Facets.LatestPrereleaseVersion(supportedFramework) :
Facets.LatestStableVersion(supportedFramework);
var newQuery = new BooleanQuery();
newQuery.Add(query, Occur.MUST);
newQuery.Add(new TermQuery(new Term("Facet", facet)), Occur.MUST);
query = newQuery;
}
string args = string.Format("Searcher.Search(..., {0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10})", q, countOnly, projectType, includePrerelease, feed, sortBy, skip, take, includeExplanation, ignoreFilter, luceneQuery);
Trace.TraceInformation(args);
content = Searcher.Search(
searcherManager,
query,
countOnly,
projectType,
includePrerelease,
feed,
sortBy,
skip,
take,
includeExplanation,
ignoreFilter);
JObject result = JObject.Parse(content);
result["answeredBy"] = "Search";
}
catch (Lucene.Net.QueryParsers.ParseException)
{
context.Response.StatusCode = 400;
content = "Invalid query";
}
finally
{
context.Response.Headers.Add("Cache-Control", new[] { string.Format("private, max-age={0}", maxAgeSeconds) });
context.Response.ContentType = "application/json";
}
await context.Response.WriteAsync(content);
}
public static string Search(PackageSearcherManager searcherManager, Query q, bool countOnly, string projectType, bool includePrerelease, string feed, string sortBy, int skip, int take, bool includeExplanation, bool ignoreFilter)
{
IndexSearcher searcher;
try
{
if ((DateTime.UtcNow - WarmTimeStampUtc) > TimeSpan.FromMinutes(1))
{
WarmTimeStampUtc = DateTime.UtcNow;
// Re-open on a background thread. We can safely continue to use the old reader while this happens.
Task.Factory
.StartNew(() => searcherManager.MaybeReopen())
.ContinueWith(t =>
{
// Log and suppress the exception to prevent taking down the whole process
if (t.IsFaulted)
{
Trace.WriteLine("Exception reopening searcher: {0}", t.Exception.ToString());
// Return a completed task indicating everything is A-OK :)
return Task.FromResult(0);
}
return t;
});
}
// Get the current reader. If a re-open is in progress but not yet complete, this will return the current reader.
searcher = searcherManager.Get();
}
catch (Exception e)
{
throw new CorruptIndexException("Exception on (re)opening", e);
}
try
{
// ignoreFilter = true, don't filter by framework, feed or latest(stable)
Filter filter = null;
if (!ignoreFilter)
{
// So if false, set up the filter and adjust the query for the framework if needed
filter = GetFilter(feed);
}
if (countOnly)
{
return DocumentCountImpl(searcher, q, filter);
}
else
{
IDictionary<string, int> rankings = searcherManager.GetRankings(projectType);
return ListDocumentsImpl(searcher, q, rankings, filter, sortBy, skip, take, includeExplanation, searcherManager);
}
}
finally
{
searcherManager.Release(searcher);
}
}
// Doesn't return JSON because consumers will want to make monitoring decisions based on this data as well as saving it/returning it from APIs
public static IndexConsistencyReport GetIndexConsistency(PackageSearcherManager searcherManager, int databasePackageCount)
{
searcherManager.MaybeReopen();
IndexSearcher searcher = searcherManager.Get();
try
{
IndexReader indexReader = searcher.IndexReader;
// Get the number of documents
int numDocs = indexReader.NumDocs();
// Build the report
return new IndexConsistencyReport(numDocs, databasePackageCount);
}
finally
{
searcherManager.Release(searcher);
}
}
private static string MakeResults(IndexSearcher searcher, TopDocs topDocs, int skip, int take, bool includeExplanation, Query query, long elapsed, IDictionary<string, int> rankings, PackageSearcherManager manager)
{
// note the use of a StringBuilder because we have the response data already formatted as JSON in the fields in the index
StringBuilder strBldr = new StringBuilder();
string timestamp;
if (!searcher.IndexReader.CommitUserData.TryGetValue("commit-time-stamp", out timestamp))
{
timestamp = null;
}
strBldr.AppendFormat("{{\"totalHits\":{0},\"timeTakenInMs\":{1},\"index\":\"{2}\"", topDocs.TotalHits, elapsed, manager.IndexName);
if (!String.IsNullOrEmpty(timestamp))
{
strBldr.AppendFormat(",\"indexTimestamp\":\"{0}\"", timestamp);
}
if (includeExplanation)
{
// JsonConvert.Serialize does escaping and quoting.
strBldr.AppendFormat(",\"executedQuery\":{0}", Newtonsoft.Json.JsonConvert.SerializeObject(query.ToString()));
}
strBldr.Append(",\"data\":[");
bool hasResult = false;
for (int i = skip; i < topDocs.ScoreDocs.Length; i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document doc = searcher.Doc(scoreDoc.Doc);
string data = doc.Get("Data");
string id = doc.Get("Id");
NuGet.Versioning.NuGetVersion ngVersion = new Versioning.NuGetVersion(doc.Get("Version"));
if (!String.IsNullOrEmpty(id) && ngVersion != null)
{
Tuple<int,int> countRecord = manager.GetDownloadCount(id,ngVersion.ToNormalizedString());
if (countRecord != null)
{
// Patch the data in to the JSON
JObject parsed = JObject.Parse(data);
parsed["DownloadCount"] = countRecord.Item1;
parsed["PackageRegistration"]["DownloadCount"] = countRecord.Item2;
data = parsed.ToString(Formatting.None);
}
}
if (includeExplanation)
{
data = AddExplanation(searcher, data, query, scoreDoc, rankings);
}
strBldr.Append(data);
strBldr.Append(",");
hasResult = true;
}
if (hasResult)
{
strBldr.Remove(strBldr.Length - 1, 1);
}
strBldr.Append("]}");
string result = strBldr.ToString();
return result;
}