// Helper function
// 1) fix cases of field names: ID -> Id
// 2) null out field names that we don't understand (so we will search them as non-field-specific terms)
// 3) For ID search, split search terms such as Id:"Foo.Bar" and "Foo-Bar" into a phrase "Foo Bar" which will work better for analyzed field search
private static NuGetSearchTerm StandardizeSearchTerms(NuGetSearchTerm input)
{
var fieldName = FieldAliases
.FirstOrDefault(f => f.Equals(input.Field, StringComparison.OrdinalIgnoreCase));
if (string.Equals(fieldName, "Author", StringComparison.OrdinalIgnoreCase))
{
fieldName = "Authors";
}
else if (string.Equals(fieldName, "Owner", StringComparison.OrdinalIgnoreCase))
{
fieldName = "Owners";
}
else if (string.Equals(fieldName, "Tag", StringComparison.OrdinalIgnoreCase))
{
fieldName = "Tags";
}
var searchTerm = new NuGetSearchTerm {
Field = fieldName, TermOrPhrase = input.TermOrPhrase
};
if (fieldName == "Id")
{
searchTerm.TermOrPhrase = PackageIndexEntity.SplitId(searchTerm.TermOrPhrase);
}
return(searchTerm);
}
public void UpdatePackage(Package package)
{
if (_getShouldAutoUpdate())
{
var packageRegistrationKey = package.PackageRegistrationKey;
var updateTerm = new Term("PackageRegistrationKey", packageRegistrationKey.ToString(CultureInfo.InvariantCulture));
if (!package.IsLatest || !package.IsLatestStable)
{
// Someone passed us in a version which was e.g. just unlisted? Or just not the latest version which is what we want to index. Doesn't really matter. We'll find one to index.
package = _packageRepository.GetAll()
.Where(p => (p.IsLatest || p.IsLatestStable) && p.PackageRegistrationKey == packageRegistrationKey)
.Include(p => p.PackageRegistration)
.Include(p => p.PackageRegistration.Owners)
.Include(p => p.SupportedFrameworks)
.FirstOrDefault();
}
// Just update the provided package
EnsureIndexWriter(creatingIndex: false);
if (package != null)
{
var indexEntity = new PackageIndexEntity(package);
_indexWriter.UpdateDocument(updateTerm, indexEntity.ToDocument());
}
else
{
_indexWriter.DeleteDocuments(updateTerm);
}
_indexWriter.Commit();
}
}
public void UpdatePackage(Package package)
{
var packageRegistrationKey = package.PackageRegistrationKey;
var updateTerm = new Term("PackageRegistrationKey", packageRegistrationKey.ToString(CultureInfo.InvariantCulture));
if (!package.IsLatest || package.IsLatestStable)
{
// Someone passed us in a version which was e.g. just unlisted? Or just not the latest version which is what we want to index. Doesn't really matter. We'll find one to index.
package = _packageRepository.GetAll()
.Where(p => (p.IsLatest || p.IsLatestStable) && p.PackageRegistrationKey == packageRegistrationKey)
.Include(p => p.PackageRegistration)
.Include(p => p.PackageRegistration.Owners)
.Include(p => p.SupportedFrameworks)
.FirstOrDefault();
}
// Just update the provided package
using (Trace.Activity(String.Format(CultureInfo.CurrentCulture, "Updating Lucene Index for: {0} {1} [PackageKey:{2}]", package.PackageRegistration.Id, package.Version, package.Key)))
{
EnsureIndexWriter(creatingIndex: false);
if (package != null)
{
var indexEntity = new PackageIndexEntity(package);
Trace.Information(String.Format(CultureInfo.CurrentCulture, "Updating Document: {0}", updateTerm.ToString()));
_indexWriter.UpdateDocument(updateTerm, indexEntity.ToDocument());
}
else
{
Trace.Information(String.Format(CultureInfo.CurrentCulture, "Deleting Document: {0}", updateTerm.ToString()));
_indexWriter.DeleteDocuments(updateTerm);
}
_indexWriter.Commit();
}
}
private void AddPackage(PackageIndexEntity packageInfo)
{
EnsureIndexWriter(creatingIndex: false);
DoAndRetryOnOutOfMemory(
() => _indexWriter.AddDocument(packageInfo.ToDocument()),
handleException: () => DisposeAndEnsureIndexWriter(creatingIndex: false));
}
private static void AddPackage(PackageIndexEntity package)
{
var document = new Document();
var field = new Field("Id-Exact", package.Id.ToLowerInvariant(), Field.Store.NO, Field.Index.NOT_ANALYZED);
field.SetBoost(2.5f);
document.Add(field);
field = new Field("Description", package.Description, Field.Store.NO, Field.Index.ANALYZED);
field.SetBoost(0.1f);
document.Add(field);
var tokenizedId = TokenizeId(package.Id);
foreach (var idToken in tokenizedId)
{
field = new Field("Id", idToken, Field.Store.NO, Field.Index.ANALYZED);
document.Add(field);
}
// If an element does not have a Title, then add all the tokenized Id components as Title.
// Lucene's StandardTokenizer does not tokenize items of the format a.b.c which does not play well with things like "xunit.net".
// We will feed it values that are already tokenized.
var titleTokens = String.IsNullOrEmpty(package.Title) ? tokenizedId : package.Title.Split(idSeparators, StringSplitOptions.RemoveEmptyEntries);
foreach (var idToken in titleTokens)
{
field = new Field("Title", idToken, Field.Store.NO, Field.Index.ANALYZED);
field.SetBoost(0.9f);
document.Add(field);
}
if (!String.IsNullOrEmpty(package.Tags))
{
field = new Field("Tags", package.Tags, Field.Store.NO, Field.Index.ANALYZED);
field.SetBoost(0.8f);
document.Add(field);
}
document.Add(new Field("Author", package.Authors, Field.Store.NO, Field.Index.ANALYZED));
// Fields meant for filtering and sorting
document.Add(new Field("Key", package.Key.ToString(CultureInfo.InvariantCulture), Field.Store.YES, Field.Index.NO));
document.Add(new Field("PackageRegistrationKey", package.PackageRegistrationKey.ToString(CultureInfo.InvariantCulture), Field.Store.NO, Field.Index.NOT_ANALYZED));
document.Add(new Field("IsLatest", package.IsLatest.ToString(), Field.Store.NO, Field.Index.NOT_ANALYZED));
document.Add(new Field("IsLatestStable", package.IsLatestStable.ToString(), Field.Store.NO, Field.Index.NOT_ANALYZED));
document.Add(new Field("PublishedDate", package.Published.Ticks.ToString(), Field.Store.NO, Field.Index.NOT_ANALYZED));
document.Add(new Field("DownloadCount", package.DownloadCount.ToString(CultureInfo.InvariantCulture), Field.Store.NO, Field.Index.NOT_ANALYZED));
string displayName = String.IsNullOrEmpty(package.Title) ? package.Id : package.Title;
document.Add(new Field("DisplayName", displayName.ToLower(CultureInfo.CurrentCulture), Field.Store.NO, Field.Index.NOT_ANALYZED));
indexWriter.AddDocument(document);
}
public void UpdatePackage(Package package)
{
if (_indexContainsAllVersions)
{
//just update everything since the last write time
UpdateIndex(forceRefresh: false);
return;
}
// when we only store the latest, we can run the rest of this
if (_getShouldAutoUpdate())
{
var packageRegistrationKey = package.PackageRegistrationKey;
var updateTerm = new Term("PackageRegistrationKey", packageRegistrationKey.ToString(CultureInfo.InvariantCulture));
if (!package.IsLatest || !package.IsLatestStable)
{
// Someone passed us in a version which was e.g. just unlisted? Or just not the latest version which is what we want to index. Doesn't really matter. We'll find one to index.
using (var context = _contextThunk())
{
var packageRepo = new EntityRepository <Package>(context);
package = packageRepo.GetAll()
.Where(p => (p.IsLatest || p.IsLatestStable) && p.PackageRegistrationKey == packageRegistrationKey)
.Include(p => p.PackageRegistration)
.Include(p => p.PackageRegistration.Owners)
.Include(p => p.SupportedFrameworks)
.FirstOrDefault();
}
}
void handleOutOfMemory() => DisposeAndEnsureIndexWriter(creatingIndex: false);
EnsureIndexWriter(creatingIndex: false);
// Just update the provided package
if (package != null)
{
var document = new PackageIndexEntity(package).ToDocument();
DoAndRetryOnOutOfMemory(
() => _indexWriter.UpdateDocument(updateTerm, document),
handleOutOfMemory,
waitMilliseconds: 50);
}
else
{
DoAndRetryOnOutOfMemory(() => _indexWriter.DeleteDocuments(new Term[] { updateTerm }), handleOutOfMemory, waitMilliseconds: 50);
}
DoAndRetryOnOutOfMemory(() => _indexWriter.Commit(), handleOutOfMemory);
}
}
public void UpdatePackage(Package package)
{
// Just update the provided package
using (Trace.Activity(String.Format(CultureInfo.CurrentCulture, "Updating Lucene Index for: {0} {1} [PackageKey:{2}]", package.PackageRegistration.Id, package.Version, package.Key)))
{
EnsureIndexWriter(creatingIndex: false);
var indexEntity = new PackageIndexEntity(package);
var updateTerm = new Term("PackageRegistrationKey", package.PackageRegistrationKey.ToString(CultureInfo.InvariantCulture));
if (package.Listed)
{
Trace.Information(String.Format(CultureInfo.CurrentCulture, "Updating Document: {0}", updateTerm.ToString()));
_indexWriter.UpdateDocument(updateTerm, indexEntity.ToDocument());
}
else
{
Trace.Information(String.Format(CultureInfo.CurrentCulture, "Deleting Document: {0}", updateTerm.ToString()));
_indexWriter.DeleteDocuments(updateTerm);
}
_indexWriter.Commit();
}
}
public void UpdatePackage(Package package)
{
// Just update the provided package
using (Trace.Activity(String.Format(CultureInfo.CurrentCulture, "Updating Lucene Index for: {0} {1} [PackageKey:{2}]", package.PackageRegistration.Id, package.Version, package.Key)))
{
EnsureIndexWriter(creatingIndex: false);
var indexEntity = new PackageIndexEntity(package);
var updateTerm = new Term("PackageRegistrationKey", package.PackageRegistrationKey.ToString(CultureInfo.InvariantCulture));
if (package.Listed)
{
Trace.Information(String.Format(CultureInfo.CurrentCulture, "Updating Document: {0}", updateTerm.ToString()));
_indexWriter.UpdateDocument(updateTerm, indexEntity.ToDocument());
}
else
{
Trace.Information(String.Format(CultureInfo.CurrentCulture, "Deleting Document: {0}", updateTerm.ToString()));
_indexWriter.DeleteDocuments(updateTerm);
}
_indexWriter.Commit();
}
}
private void AddPackage(PackageIndexEntity packageInfo)
{
_indexWriter.AddDocument(packageInfo.ToDocument());
}
public void UpdatePackage(Package package)
{
if (_getShouldAutoUpdate())
{
var packageRegistrationKey = package.PackageRegistrationKey;
var updateTerm = new Term("PackageRegistrationKey", packageRegistrationKey.ToString(CultureInfo.InvariantCulture));
if (!package.IsLatest || !package.IsLatestStable)
{
// Someone passed us in a version which was e.g. just unlisted? Or just not the latest version which is what we want to index. Doesn't really matter. We'll find one to index.
package = _packageRepository.GetAll()
.Where(p => (p.IsLatest || p.IsLatestStable) && p.PackageRegistrationKey == packageRegistrationKey)
.Include(p => p.PackageRegistration)
.Include(p => p.PackageRegistration.Owners)
.Include(p => p.SupportedFrameworks)
.FirstOrDefault();
}
// Just update the provided package
using (Trace.Activity(String.Format(CultureInfo.CurrentCulture, "Updating Document: {0}", updateTerm.ToString())))
{
EnsureIndexWriter(creatingIndex: false);
if (package != null)
{
var indexEntity = new PackageIndexEntity(package);
Trace.Information(String.Format(CultureInfo.CurrentCulture, "Updating Lucene Index for: {0} {1} [PackageKey:{2}]", package.PackageRegistration.Id, package.Version, package.Key));
_indexWriter.UpdateDocument(updateTerm, indexEntity.ToDocument());
}
else
{
Trace.Information(String.Format(CultureInfo.CurrentCulture, "Deleting Document: {0}", updateTerm.ToString()));
_indexWriter.DeleteDocuments(updateTerm);
}
_indexWriter.Commit();
}
}
}
private void AddPackage(PackageIndexEntity packageInfo)
{
_indexWriter.AddDocument(packageInfo.ToDocument());
}
private static void UpdatePackage(PackageIndexEntity package)
{
string key = package.Key.ToString(CultureInfo.InvariantCulture);
var document = new Document();
var field = new Field("Id-Exact", package.Id.ToLowerInvariant(), Field.Store.NO, Field.Index.NOT_ANALYZED);
field.SetBoost(2.5f);
document.Add(field);
field = new Field("Description", package.Description, Field.Store.NO, Field.Index.ANALYZED);
field.SetBoost(0.1f);
document.Add(field);
var tokenizedId = TokenizeId(package.Id);
foreach (var idToken in tokenizedId)
{
field = new Field("Id", idToken, Field.Store.NO, Field.Index.ANALYZED);
field.SetBoost(1.2f);
document.Add(field);
}
// If an element does not have a Title, then add all the tokenized Id components as Title.
// Lucene's StandardTokenizer does not tokenize items of the format a.b.c which does not play well with things like "xunit.net".
// We will feed it values that are already tokenized.
var titleTokens = String.IsNullOrEmpty(package.Title) ? tokenizedId : package.Title.Split(idSeparators, StringSplitOptions.RemoveEmptyEntries);
foreach (var idToken in titleTokens)
{
document.Add(new Field("Title", idToken, Field.Store.NO, Field.Index.ANALYZED));
}
if (!String.IsNullOrEmpty(package.Tags))
{
field = new Field("Tags", package.Tags, Field.Store.NO, Field.Index.ANALYZED);
field.SetBoost(0.8f);
document.Add(field);
}
document.Add(new Field("Author", package.Authors, Field.Store.NO, Field.Index.ANALYZED));
// Fields meant for filtering and sorting
document.Add(new Field("Key", key, Field.Store.YES, Field.Index.NO));
document.Add(new Field("IsLatestStable", package.IsLatestStable.ToString(), Field.Store.NO, Field.Index.NOT_ANALYZED));
document.Add(new Field("PublishedDate", package.Published.Ticks.ToString(), Field.Store.NO, Field.Index.NOT_ANALYZED));
document.Add(new Field("DownloadCount", package.DownloadCount.ToString(CultureInfo.InvariantCulture), Field.Store.NO, Field.Index.NOT_ANALYZED));
string displayName = String.IsNullOrEmpty(package.Title) ? package.Id : package.Title;
document.Add(new Field("DisplayName", displayName.ToLower(CultureInfo.CurrentCulture), Field.Store.NO, Field.Index.NOT_ANALYZED));
indexWriter.UpdateDocument(new Term("Key", key), document);
}
