protected PackageSearcherManager GetSearcherManager()
{
PackageSearcherManager manager;
if (!string.IsNullOrEmpty(Container))
{
manager = PackageSearcherManager.CreateAzure(
StorageAccount,
Container,
DataContainer,
requireDownloadCounts: false);
}
else if (!string.IsNullOrEmpty(Folder))
{
manager = PackageSearcherManager.CreateLocal(
Folder,
FrameworksFile);
}
else
{
throw new Exception("You must specify either a folder or container");
}
manager.Open();
return(manager);
}
public static string GetSegments(PackageSearcherManager searcherManager)
{
searcherManager.MaybeReopen();
IndexSearcher searcher = searcherManager.Get();
try
{
IndexReader indexReader = searcher.IndexReader;
JArray segments = new JArray();
foreach (ReadOnlySegmentReader segmentReader in indexReader.GetSequentialSubReaders())
{
JObject segmentInfo = new JObject();
segmentInfo.Add("segment", segmentReader.SegmentName);
segmentInfo.Add("documents", segmentReader.NumDocs());
segments.Add(segmentInfo);
}
return(segments.ToString());
}
finally
{
searcherManager.Release(searcher);
}
}
public static string KeyRangeQuery(PackageSearcherManager searcherManager, int minKey, int maxKey)
{
// for range queries we always want the IndexReader to be absolutely up to date
searcherManager.MaybeReopen();
IndexSearcher searcher = searcherManager.Get();
try
{
NumericRangeQuery <int> numericRangeQuery = NumericRangeQuery.NewIntRange("Key", minKey, maxKey, true, true);
List <DocumentKey> pairs = new List <DocumentKey>();
searcher.Search(numericRangeQuery, new KeyCollector(pairs));
// Group by key
IEnumerable <IGrouping <int, DocumentKey> > groups = pairs.GroupBy(p => p.PackageKey);
// De-duplicate
IEnumerable <DocumentKey> deduped = groups.Select(g => g.First());
JObject keys = new JObject();
keys.Add(deduped.Select(p => new JProperty(p.PackageKey.ToString(), p.Checksum)));
return(keys.ToString());
}
finally
{
searcherManager.Release(searcher);
}
}
public static string Analyze(PackageSearcherManager searcherManager)
{
searcherManager.MaybeReopen();
IndexSearcher searcher = searcherManager.Get();
try
{
IndexReader indexReader = searcher.IndexReader;
JObject report = new JObject();
report.Add("NumDocs", indexReader.NumDocs());
report.Add("SearcherManagerIdentity", searcherManager.Id.ToString());
AzureDirectory azDir = indexReader.Directory() as AzureDirectory;
if (azDir != null)
{
report.Add("Index", azDir.BlobContainer.Name);
}
else
{
SimpleFSDirectory fsDir = indexReader.Directory() as SimpleFSDirectory;
if (fsDir != null)
{
report.Add("Index", fsDir.Directory.Name);
}
}
report.Add("RankingsUpdated", searcherManager.RankingsUpdatedUtc);
report.Add("DownloadCountsUpdated", searcherManager.DownloadCountsUpdatedUtc);
if (indexReader.CommitUserData != null)
{
JObject commitUserdata = new JObject();
foreach (KeyValuePair <string, string> userData in indexReader.CommitUserData)
{
commitUserdata.Add(userData.Key, userData.Value);
}
report.Add("CommitUserData", commitUserdata);
}
// Moved segments to their own command since they can take a while to calculate in Azure
return(report.ToString());
}
finally
{
searcherManager.Release(searcher);
}
}
public static string Search(PackageSearcherManager searcherManager, string q, bool countOnly, string projectType, bool includePrerelease, string feed, string sortBy, int skip, int take, bool includeExplanation, bool ignoreFilter)
{
return(Search(
searcherManager,
LuceneQueryCreator.Parse(q, true),
countOnly,
projectType,
includePrerelease,
feed,
sortBy,
skip,
take,
includeExplanation,
ignoreFilter));
}
// Doesn't return JSON because consumers will want to make monitoring decisions based on this data as well as saving it/returning it from APIs
public static IndexConsistencyReport GetIndexConsistency(PackageSearcherManager searcherManager, int databasePackageCount)
{
searcherManager.MaybeReopen();
IndexSearcher searcher = searcherManager.Get();
try
{
IndexReader indexReader = searcher.IndexReader;
// Get the number of documents
int numDocs = indexReader.NumDocs();
// Build the report
return(new IndexConsistencyReport(numDocs, databasePackageCount));
}
finally
{
searcherManager.Release(searcher);
}
}
public static string GetDistinctStoredFieldNames(PackageSearcherManager searcherManager)
{
searcherManager.MaybeReopen();
IndexSearcher searcher = searcherManager.Get();
try
{
IndexReader indexReader = searcher.IndexReader;
HashSet <string> distinctFieldNames = new HashSet <string>();
for (int i = 0; i < indexReader.MaxDoc; i++)
{
if (!indexReader.IsDeleted(i))
{
Document document = indexReader.Document(i);
IList <IFieldable> fields = document.GetFields();
foreach (IFieldable field in fields)
{
distinctFieldNames.Add(field.Name);
}
}
}
JArray array = new JArray();
foreach (string fieldName in distinctFieldNames)
{
array.Add(fieldName);
}
return(array.ToString());
}
finally
{
searcherManager.Release(searcher);
}
}
public static string Search(PackageSearcherManager searcherManager, Query q, bool countOnly, string projectType, bool includePrerelease, string feed, string sortBy, int skip, int take, bool includeExplanation, bool ignoreFilter)
{
IndexSearcher searcher;
try
{
if ((DateTime.UtcNow - WarmTimeStampUtc) > TimeSpan.FromMinutes(1))
{
WarmTimeStampUtc = DateTime.UtcNow;
// Re-open on a background thread. We can safely continue to use the old reader while this happens.
Task.Factory
.StartNew(() => searcherManager.MaybeReopen())
.ContinueWith(t =>
{
// Log and suppress the exception to prevent taking down the whole process
if (t.IsFaulted)
{
Trace.WriteLine("Exception reopening searcher: {0}", t.Exception.ToString());
// Return a completed task indicating everything is A-OK :)
return(Task.FromResult(0));
}
return(t);
});
}
// Get the current reader. If a re-open is in progress but not yet complete, this will return the current reader.
searcher = searcherManager.Get();
}
catch (Exception e)
{
throw new CorruptIndexException("Exception on (re)opening", e);
}
try
{
// ignoreFilter = true, don't filter by framework, feed or latest(stable)
Filter filter = null;
if (!ignoreFilter)
{
// So if false, set up the filter and adjust the query for the framework if needed
filter = GetFilter(feed);
}
if (countOnly)
{
return(DocumentCountImpl(searcher, q, filter));
}
else
{
IDictionary <string, int> rankings = searcherManager.GetRankings(projectType);
return(ListDocumentsImpl(searcher, q, rankings, filter, sortBy, skip, take, includeExplanation, searcherManager));
}
}
finally
{
searcherManager.Release(searcher);
}
}
private static string MakeResults(IndexSearcher searcher, TopDocs topDocs, int skip, int take, bool includeExplanation, Query query, long elapsed, IDictionary <string, int> rankings, PackageSearcherManager manager)
{
// note the use of a StringBuilder because we have the response data already formatted as JSON in the fields in the index
StringBuilder strBldr = new StringBuilder();
string timestamp;
if (!searcher.IndexReader.CommitUserData.TryGetValue("commit-time-stamp", out timestamp))
{
timestamp = null;
}
strBldr.AppendFormat("{{\"totalHits\":{0},\"timeTakenInMs\":{1},\"index\":\"{2}\"", topDocs.TotalHits, elapsed, manager.IndexName);
if (!String.IsNullOrEmpty(timestamp))
{
strBldr.AppendFormat(",\"indexTimestamp\":\"{0}\"", timestamp);
}
if (includeExplanation)
{
// JsonConvert.Serialize does escaping and quoting.
strBldr.AppendFormat(",\"executedQuery\":{0}", Newtonsoft.Json.JsonConvert.SerializeObject(query.ToString()));
}
strBldr.Append(",\"data\":[");
bool hasResult = false;
for (int i = skip; i < topDocs.ScoreDocs.Length; i++)
{
ScoreDoc scoreDoc = topDocs.ScoreDocs[i];
Document doc = searcher.Doc(scoreDoc.Doc);
string data = doc.Get("Data");
string key = doc.Get("Key");
int keyVal;
if (!String.IsNullOrEmpty(key) && Int32.TryParse(key, out keyVal))
{
DownloadCountRecord countRecord = manager.GetDownloadCounts(keyVal);
if (countRecord != null)
{
// Patch the data in to the JSON
JObject parsed = JObject.Parse(data);
parsed["DownloadCount"] = countRecord.Downloads;
parsed["PackageRegistration"]["DownloadCount"] = countRecord.RegistrationDownloads;
parsed.Add("Installs", countRecord.Installs);
parsed.Add("Updates", countRecord.Updates);
data = parsed.ToString(Formatting.None);
}
}
if (includeExplanation)
{
data = AddExplanation(searcher, data, query, scoreDoc, rankings);
}
strBldr.Append(data);
strBldr.Append(",");
hasResult = true;
}
if (hasResult)
{
strBldr.Remove(strBldr.Length - 1, 1);
}
strBldr.Append("]}");
string result = strBldr.ToString();
return(result);
}
private static string ListDocumentsImpl(IndexSearcher searcher, Query query, IDictionary <string, int> rankings, Filter filter, string sortBy, int skip, int take, bool includeExplanation, PackageSearcherManager manager)
{
Query boostedQuery = new RankingScoreQuery(query, rankings);
int nDocs = GetDocsCount(skip, take);
Sort sort = GetSort(sortBy);
Stopwatch sw = new Stopwatch();
sw.Start();
TopDocs topDocs = (sort == null) ?
searcher.Search(boostedQuery, filter, nDocs) :
searcher.Search(boostedQuery, filter, nDocs, sort);
sw.Stop();
sw.Stop();
return(MakeResults(searcher, topDocs, skip, take, includeExplanation, boostedQuery, sw.ElapsedMilliseconds, rankings, manager));
}