/// <summary>
/// Register a versioner to add a versioning system to the VersionManager
/// </summary>
/// <param name="name">The version key</param>
/// <param name="isAddressable">Whether the versioning system is addressable (different version values can be accessed by changing url)</param>
/// <param name="versioner">The Versioner for the versioning system</param>
/// <param name="versions">All the valid version values for the versioning system (null if these are not limited)</param>
public void RegisterVersion(
Versioner versioner)
{
Versioners.Add(versioner);
if (versioner.IsAddressable)
{
AddressableVersionKeys.Add(versioner.VersionKey);
}
else
{
UnaddressableVersionKeys.Add(versioner.VersionKey);
}
VersionLists.Add(versioner.VersionKey, versioner.AllVersionValues);
}
/// <summary>
/// Register a versioner to add a versioning system to the VersionManager
/// </summary>
/// <param name="name">The version key</param>
/// <param name="versioner">The Versioner for the versioning system</param>
public void RegisterVersion(Versioner versioner)
{
Versioners.Add(versioner);
if (versioner.IsAddressable)
{
AddressableVersionKeys.Add(versioner.VersionKey);
}
else
{
UnaddressableVersionKeys.Add(versioner.VersionKey);
}
if (!VersionLists.ContainsKey(versioner.VersionKey))
{
VersionLists.TryAdd(versioner.VersionKey, versioner.AllVersionValues);
}
}
private IndexAction <KeyedDocument> GetSearchIndexAction(
NewPackageRegistration packageRegistration,
IReadOnlyDictionary <NuGetVersion, Package> versionToPackage,
VersionLists versionLists,
SearchFilters searchFilters,
SearchIndexChangeType changeType)
{
if (changeType == SearchIndexChangeType.Delete)
{
return(IndexAction.Delete(_search.Keyed(
packageRegistration.PackageId,
searchFilters)));
}
if (changeType != SearchIndexChangeType.AddFirst)
{
throw new ArgumentException(
$"The only change types supported are {nameof(SearchIndexChangeType.AddFirst)} and " +
$"{nameof(SearchIndexChangeType.Delete)}.",
nameof(changeType));
}
var latestFlags = _search.LatestFlagsOrNull(versionLists, searchFilters);
var package = versionToPackage[latestFlags.LatestVersionInfo.ParsedVersion];
var owners = packageRegistration
.Owners
.OrderBy(u => u, StringComparer.InvariantCultureIgnoreCase)
.ToArray();
VerifyConsistency(packageRegistration.PackageId, package);
return(IndexAction.Upload <KeyedDocument>(_search.FullFromDb(
packageRegistration.PackageId,
searchFilters,
latestFlags.LatestVersionInfo.ListedFullVersions,
latestFlags.IsLatestStable,
latestFlags.IsLatest,
latestFlags.LatestVersionInfo.FullVersion,
package,
owners,
packageRegistration.TotalDownloadCount,
packageRegistration.IsExcludedByDefault)));
}
public IndexActions AddNewPackageRegistration(NewPackageRegistration packageRegistration)
{
var versionProperties = new Dictionary <string, VersionPropertiesData>();
var versionListData = new VersionListData(versionProperties);
var versionLists = new VersionLists(versionListData);
var changes = packageRegistration
.Packages
.Select(GetVersionListChange)
.ToList();
var indexChanges = versionLists.ApplyChanges(changes);
var versionToPackage = packageRegistration
.Packages
.ToDictionary(p => NuGetVersion.Parse(p.Version));
var search = indexChanges
.Search
.Select(p => GetSearchIndexAction(
packageRegistration,
versionToPackage,
versionLists,
p.Key,
p.Value))
.ToList();
var hijack = indexChanges
.Hijack
.Select(p => GetHijackIndexAction(
packageRegistration.PackageId,
versionToPackage[p.Key],
p.Value))
.ToList();
return(new IndexActions(
search,
hijack,
new ResultAndAccessCondition <VersionListData>(
versionLists.GetVersionListData(),
AccessConditionWrapper.GenerateEmptyCondition())));
}
public async Task <DocumentFixUp> TryFixUpAsync(
IReadOnlyList <CatalogCommitItem> itemList,
ConcurrentBag <IdAndValue <IndexActions> > allIndexActions,
InvalidOperationException exception)
{
var innerEx = exception.InnerException as IndexBatchException;
if (innerEx == null || innerEx.IndexingResults == null)
{
return(DocumentFixUp.IsNotApplicable());
}
// There may have been a Case of the Missing Document! We have confirmed with the Azure Search team that
// this is a bug on the Azure Search side. To mitigate the issue, we replace any Merge operation that
// failed with 404 with a MergeOrUpload with the full metadata so that we can replace that missing document.
//
// 1. The first step is to find all of the document keys that failed with a 404 Not Found error.
var notFoundKeys = new HashSet <string>(innerEx
.IndexingResults
.Where(x => x.StatusCode == (int)HttpStatusCode.NotFound)
.Select(x => x.Key));
if (!notFoundKeys.Any())
{
return(DocumentFixUp.IsNotApplicable());
}
_logger.LogWarning("{Count} document action(s) failed with 404 Not Found.", notFoundKeys.Count);
// 2. Find all of the package IDs that were affected, only considering Merge operations against the Search
// index. We ignore the the hijack index for now because we have only ever seen the problem in the Search
// index.
var failedIds = new HashSet <string>();
foreach (var pair in allIndexActions.OrderBy(x => x.Id, StringComparer.OrdinalIgnoreCase))
{
var failedMerges = pair
.Value
.Search
.Where(a => a.ActionType == IndexActionType.Merge)
.Where(a => notFoundKeys.Contains(a.Document.Key));
if (failedMerges.Any() && failedIds.Add(pair.Id))
{
_logger.LogWarning("Package {PackageId} had a Merge operation fail with 404 Not Found.", pair.Id);
}
}
if (!failedIds.Any())
{
_logger.LogInformation("No failed Merge operations against the Search index were found.");
return(DocumentFixUp.IsNotApplicable());
}
// 3. For each affected package ID, get the version list to determine the latest version per search filter
// so we can find the the catalog entry for the version.
var identityToItems = itemList.GroupBy(x => x.PackageIdentity).ToDictionary(x => x.Key, x => x.ToList());
foreach (var packageId in failedIds)
{
var accessConditionAndData = await _versionListClient.ReadAsync(packageId);
var versionLists = new VersionLists(accessConditionAndData.Result);
var latestVersions = DocumentUtilities
.AllSearchFilters
.Select(sf => versionLists.GetLatestVersionInfoOrNull(sf))
.Where(lvi => lvi != null)
.Select(lvi => (IReadOnlyList <NuGetVersion>) new List <NuGetVersion> {
lvi.ParsedVersion
})
.ToList();
var leaves = await _leafFetcher.GetLatestLeavesAsync(packageId, latestVersions);
// We ignore unavailable (deleted) versions for now. We have never had a delete cause this problem. It's
// only ever been discovered when a new version is being added or updated.
//
// For each package details leaf found, create a catalog commit item and add it to the set of items we
// will process. This will force the metadata to be updated on each of the latest versions. Since this
// is the latest metadata, replace any older leaves that may be associated with that package version.
foreach (var pair in leaves.Available)
{
var identity = new PackageIdentity(packageId, pair.Key);
var leaf = pair.Value;
if (identityToItems.TryGetValue(identity, out var existing))
{
if (existing.Count == 1 && existing[0].Uri.AbsoluteUri == leaf.Url)
{
_logger.LogInformation(
"For {PackageId} {PackageVersion}, metadata will remain the same.",
identity.Id,
identity.Version.ToNormalizedString(),
leaf.Url,
existing.Count);
continue;
}
else
{
_logger.LogInformation(
"For {PackageId} {PackageVersion}, metadata from {Url} will be used instead of {Count} catalog commit items.",
identity.Id,
identity.Version.ToNormalizedString(),
leaf.Url,
existing.Count);
}
}
else
{
_logger.LogInformation(
"For {PackageId} {PackageVersion}, metadata from {Url} will be used.",
identity.Id,
identity.Version.ToNormalizedString(),
leaf.Url);
}
identityToItems[identity] = new List <CatalogCommitItem>
{
new CatalogCommitItem(
new Uri(leaf.Url),
leaf.CommitId,
leaf.CommitTimestamp.UtcDateTime,
new string[0],
new[] { Schema.DataTypes.PackageDetails },
identity),
};
}
}
_logger.LogInformation("The catalog commit item list has been modified to fix up the missing document(s).");
var newItemList = identityToItems.SelectMany(x => x.Value).ToList();
return(DocumentFixUp.IsApplicable(newItemList));
}
