/// <summary> /// Register a versioner to add a versioning system to the VersionManager /// </summary> /// <param name="name">The version key</param> /// <param name="isAddressable">Whether the versioning system is addressable (different version values can be accessed by changing url)</param> /// <param name="versioner">The Versioner for the versioning system</param> /// <param name="versions">All the valid version values for the versioning system (null if these are not limited)</param> public void RegisterVersion( Versioner versioner) { Versioners.Add(versioner); if (versioner.IsAddressable) { AddressableVersionKeys.Add(versioner.VersionKey); } else { UnaddressableVersionKeys.Add(versioner.VersionKey); } VersionLists.Add(versioner.VersionKey, versioner.AllVersionValues); }
/// <summary> /// Register a versioner to add a versioning system to the VersionManager /// </summary> /// <param name="name">The version key</param> /// <param name="versioner">The Versioner for the versioning system</param> public void RegisterVersion(Versioner versioner) { Versioners.Add(versioner); if (versioner.IsAddressable) { AddressableVersionKeys.Add(versioner.VersionKey); } else { UnaddressableVersionKeys.Add(versioner.VersionKey); } if (!VersionLists.ContainsKey(versioner.VersionKey)) { VersionLists.TryAdd(versioner.VersionKey, versioner.AllVersionValues); } }
private IndexAction <KeyedDocument> GetSearchIndexAction( NewPackageRegistration packageRegistration, IReadOnlyDictionary <NuGetVersion, Package> versionToPackage, VersionLists versionLists, SearchFilters searchFilters, SearchIndexChangeType changeType) { if (changeType == SearchIndexChangeType.Delete) { return(IndexAction.Delete(_search.Keyed( packageRegistration.PackageId, searchFilters))); } if (changeType != SearchIndexChangeType.AddFirst) { throw new ArgumentException( $"The only change types supported are {nameof(SearchIndexChangeType.AddFirst)} and " + $"{nameof(SearchIndexChangeType.Delete)}.", nameof(changeType)); } var latestFlags = _search.LatestFlagsOrNull(versionLists, searchFilters); var package = versionToPackage[latestFlags.LatestVersionInfo.ParsedVersion]; var owners = packageRegistration .Owners .OrderBy(u => u, StringComparer.InvariantCultureIgnoreCase) .ToArray(); VerifyConsistency(packageRegistration.PackageId, package); return(IndexAction.Upload <KeyedDocument>(_search.FullFromDb( packageRegistration.PackageId, searchFilters, latestFlags.LatestVersionInfo.ListedFullVersions, latestFlags.IsLatestStable, latestFlags.IsLatest, latestFlags.LatestVersionInfo.FullVersion, package, owners, packageRegistration.TotalDownloadCount, packageRegistration.IsExcludedByDefault))); }
public IndexActions AddNewPackageRegistration(NewPackageRegistration packageRegistration) { var versionProperties = new Dictionary <string, VersionPropertiesData>(); var versionListData = new VersionListData(versionProperties); var versionLists = new VersionLists(versionListData); var changes = packageRegistration .Packages .Select(GetVersionListChange) .ToList(); var indexChanges = versionLists.ApplyChanges(changes); var versionToPackage = packageRegistration .Packages .ToDictionary(p => NuGetVersion.Parse(p.Version)); var search = indexChanges .Search .Select(p => GetSearchIndexAction( packageRegistration, versionToPackage, versionLists, p.Key, p.Value)) .ToList(); var hijack = indexChanges .Hijack .Select(p => GetHijackIndexAction( packageRegistration.PackageId, versionToPackage[p.Key], p.Value)) .ToList(); return(new IndexActions( search, hijack, new ResultAndAccessCondition <VersionListData>( versionLists.GetVersionListData(), AccessConditionWrapper.GenerateEmptyCondition()))); }
public async Task <DocumentFixUp> TryFixUpAsync( IReadOnlyList <CatalogCommitItem> itemList, ConcurrentBag <IdAndValue <IndexActions> > allIndexActions, InvalidOperationException exception) { var innerEx = exception.InnerException as IndexBatchException; if (innerEx == null || innerEx.IndexingResults == null) { return(DocumentFixUp.IsNotApplicable()); } // There may have been a Case of the Missing Document! We have confirmed with the Azure Search team that // this is a bug on the Azure Search side. To mitigate the issue, we replace any Merge operation that // failed with 404 with a MergeOrUpload with the full metadata so that we can replace that missing document. // // 1. The first step is to find all of the document keys that failed with a 404 Not Found error. var notFoundKeys = new HashSet <string>(innerEx .IndexingResults .Where(x => x.StatusCode == (int)HttpStatusCode.NotFound) .Select(x => x.Key)); if (!notFoundKeys.Any()) { return(DocumentFixUp.IsNotApplicable()); } _logger.LogWarning("{Count} document action(s) failed with 404 Not Found.", notFoundKeys.Count); // 2. Find all of the package IDs that were affected, only considering Merge operations against the Search // index. We ignore the the hijack index for now because we have only ever seen the problem in the Search // index. var failedIds = new HashSet <string>(); foreach (var pair in allIndexActions.OrderBy(x => x.Id, StringComparer.OrdinalIgnoreCase)) { var failedMerges = pair .Value .Search .Where(a => a.ActionType == IndexActionType.Merge) .Where(a => notFoundKeys.Contains(a.Document.Key)); if (failedMerges.Any() && failedIds.Add(pair.Id)) { _logger.LogWarning("Package {PackageId} had a Merge operation fail with 404 Not Found.", pair.Id); } } if (!failedIds.Any()) { _logger.LogInformation("No failed Merge operations against the Search index were found."); return(DocumentFixUp.IsNotApplicable()); } // 3. For each affected package ID, get the version list to determine the latest version per search filter // so we can find the the catalog entry for the version. var identityToItems = itemList.GroupBy(x => x.PackageIdentity).ToDictionary(x => x.Key, x => x.ToList()); foreach (var packageId in failedIds) { var accessConditionAndData = await _versionListClient.ReadAsync(packageId); var versionLists = new VersionLists(accessConditionAndData.Result); var latestVersions = DocumentUtilities .AllSearchFilters .Select(sf => versionLists.GetLatestVersionInfoOrNull(sf)) .Where(lvi => lvi != null) .Select(lvi => (IReadOnlyList <NuGetVersion>) new List <NuGetVersion> { lvi.ParsedVersion }) .ToList(); var leaves = await _leafFetcher.GetLatestLeavesAsync(packageId, latestVersions); // We ignore unavailable (deleted) versions for now. We have never had a delete cause this problem. It's // only ever been discovered when a new version is being added or updated. // // For each package details leaf found, create a catalog commit item and add it to the set of items we // will process. This will force the metadata to be updated on each of the latest versions. Since this // is the latest metadata, replace any older leaves that may be associated with that package version. foreach (var pair in leaves.Available) { var identity = new PackageIdentity(packageId, pair.Key); var leaf = pair.Value; if (identityToItems.TryGetValue(identity, out var existing)) { if (existing.Count == 1 && existing[0].Uri.AbsoluteUri == leaf.Url) { _logger.LogInformation( "For {PackageId} {PackageVersion}, metadata will remain the same.", identity.Id, identity.Version.ToNormalizedString(), leaf.Url, existing.Count); continue; } else { _logger.LogInformation( "For {PackageId} {PackageVersion}, metadata from {Url} will be used instead of {Count} catalog commit items.", identity.Id, identity.Version.ToNormalizedString(), leaf.Url, existing.Count); } } else { _logger.LogInformation( "For {PackageId} {PackageVersion}, metadata from {Url} will be used.", identity.Id, identity.Version.ToNormalizedString(), leaf.Url); } identityToItems[identity] = new List <CatalogCommitItem> { new CatalogCommitItem( new Uri(leaf.Url), leaf.CommitId, leaf.CommitTimestamp.UtcDateTime, new string[0], new[] { Schema.DataTypes.PackageDetails }, identity), }; } } _logger.LogInformation("The catalog commit item list has been modified to fix up the missing document(s)."); var newItemList = identityToItems.SelectMany(x => x.Value).ToList(); return(DocumentFixUp.IsApplicable(newItemList)); }