private async Task DeleteOrphansAsync( HiveType hive, IReadOnlyList <HiveType> replicaHives, IEnumerable <string> existingPageUrls, IndexInfo indexInfo, HiveMergeResult mergeResult) { // Start with all of the page URLs found in the index prior to the update process. var orphanUrls = new HashSet <string>(existingPageUrls); // Add all of the deleted leaf URLs. orphanUrls.UnionWith(mergeResult.DeletedLeaves.Select(x => x.LeafItem.Url)); // Leave the new page URLs alone. foreach (var pageInfo in indexInfo.Items) { orphanUrls.Remove(pageInfo.PageItem.Url); } // Leave the modified leaf URLs alone. This should not be necessary since deleted leaves and modified // leaves are disjoint sets but is a reasonable precaution. foreach (var leafInfo in mergeResult.ModifiedLeaves) { orphanUrls.Remove(leafInfo.LeafItem.Url); } if (orphanUrls.Count == 0) { _logger.LogInformation("There are no orphan blobs to delete."); return; } _logger.LogInformation("About to delete {Count} orphan blobs.", orphanUrls.Count); var work = new ConcurrentBag <string>(orphanUrls); await ParallelAsync.Repeat( async() => { while (work.TryTake(out var url)) { await _storage.DeleteUrlAsync(hive, replicaHives, url); } }, _options.Value.MaxConcurrentOperationsPerHive); _logger.LogInformation("Done deleting orphan blobs.", orphanUrls.Count); }
private async Task UpdateLeavesAsync( HiveType hive, IReadOnlyList <HiveType> replicaHives, string id, Dictionary <NuGetVersion, PackageDetailsCatalogLeaf> versionToCatalogLeaf, CatalogCommit registrationCommit, HiveMergeResult mergeResult) { if (!mergeResult.ModifiedLeaves.Any()) { _logger.LogInformation("No leaves need to be updated."); return; } _logger.LogInformation( "Updating {Count} registration leaves.", mergeResult.ModifiedLeaves.Count, id, hive); var taskFactories = new ConcurrentBag <Func <Task> >(); foreach (var leafInfo in mergeResult.ModifiedLeaves) { _entityBuilder.UpdateLeafItem(leafInfo.LeafItem, hive, id, versionToCatalogLeaf[leafInfo.Version]); _entityBuilder.UpdateCommit(leafInfo.LeafItem, registrationCommit); var leaf = _entityBuilder.NewLeaf(leafInfo.LeafItem); taskFactories.Add(async() => { _logger.LogInformation("Updating leaf {PackageId} {Version}.", id, leafInfo.Version.ToNormalizedString()); await _storage.WriteLeafAsync(hive, replicaHives, id, leafInfo.Version, leaf); }); } await ParallelAsync.Repeat( async() => { await Task.Yield(); while (taskFactories.TryTake(out var taskFactory)) { await taskFactory(); } }, _options.Value.MaxConcurrentOperationsPerHive); }
private async Task UpdateNonInlinedPagesAsync( HiveType hive, IReadOnlyList <HiveType> replicaHives, string id, IndexInfo indexInfo, CatalogCommit registrationCommit, HiveMergeResult mergeResult) { var taskFactories = new ConcurrentBag <Func <Task> >(); for (var pageIndex = 0; pageIndex < indexInfo.Items.Count; pageIndex++) { var pageInfo = indexInfo.Items[pageIndex]; if (pageInfo.IsInlined) { _logger.LogInformation( "Moving page {PageNumber}/{PageCount} [{Lower}, {Upper}] from being inlined to having its own blob.", pageIndex + 1, indexInfo.Items.Count, pageInfo.Lower.ToNormalizedString(), pageInfo.Upper.ToNormalizedString()); pageInfo = await pageInfo.CloneToNonInlinedAsync(); indexInfo.RemoveAt(pageIndex); indexInfo.Insert(pageIndex, pageInfo); } else if (!mergeResult.ModifiedPages.Contains(pageInfo)) { _logger.LogInformation( "Skipping unmodified page {PageNumber}/{PageCount} [{Lower}, {Upper}].", pageIndex + 1, indexInfo.Items.Count, pageInfo.Lower.ToNormalizedString(), pageInfo.Upper.ToNormalizedString()); continue; } Guard.Assert(!pageInfo.IsInlined, "The page should not be inlined at this point."); var page = await pageInfo.GetPageAsync(); _entityBuilder.UpdateNonInlinedPageItem(pageInfo.PageItem, hive, id, pageInfo.Count, pageInfo.Lower, pageInfo.Upper); _entityBuilder.UpdateCommit(pageInfo.PageItem, registrationCommit); _entityBuilder.UpdatePage(page, hive, id, pageInfo.Count, pageInfo.Lower, pageInfo.Upper); _entityBuilder.UpdateCommit(page, registrationCommit); var pageNumber = pageIndex + 1; taskFactories.Add(async() => { _logger.LogInformation( "Updating page {PageNumber}/{PageCount} [{Lower}, {Upper}].", pageNumber, indexInfo.Items.Count, pageInfo.Lower.ToNormalizedString(), pageInfo.Upper.ToNormalizedString()); await _storage.WritePageAsync(hive, replicaHives, id, pageInfo.Lower, pageInfo.Upper, page); }); } await ParallelAsync.Repeat( async() => { await Task.Yield(); while (taskFactories.TryTake(out var taskFactory)) { await taskFactory(); } }, _options.Value.MaxConcurrentOperationsPerHive); }
public Facts(ITestOutputHelper output) { Storage = new Mock <IHiveStorage>(); Merger = new Mock <IHiveMerger>(); EntityBuilder = new Mock <IEntityBuilder>(); Options = new Mock <IOptionsSnapshot <Catalog2RegistrationConfiguration> >(); Logger = output.GetLogger <HiveUpdater>(); Config = new Catalog2RegistrationConfiguration(); Hive = HiveType.SemVer2; ReplicaHives = new List <HiveType>(); Id = "NuGet.Versioning"; Entries = new List <CatalogCommitItem>(); EntryToCatalogLeaf = new Dictionary <CatalogCommitItem, PackageDetailsCatalogLeaf>( ReferenceEqualityComparer <CatalogCommitItem> .Default); RegistrationIndex = new RegistrationIndex { Items = new List <RegistrationPage> { new RegistrationPage { Lower = "1.0.0", Upper = "3.0.0", Count = 2, Items = new List <RegistrationLeafItem> { new RegistrationLeafItem { Url = $"https://example/reg/{Id.ToLowerInvariant()}/1.0.0.json", CatalogEntry = new RegistrationCatalogEntry { Version = "1.0.0", } }, new RegistrationLeafItem { Url = $"https://example/reg/{Id.ToLowerInvariant()}/3.0.0.json", CatalogEntry = new RegistrationCatalogEntry { Version = "3.0.0", } }, } } } }; MergeResult = new HiveMergeResult( new HashSet <PageInfo>(), new HashSet <LeafInfo>(), new HashSet <LeafInfo>()); RegistrationLeaf = new RegistrationLeaf(); RegistrationCommit = new CatalogCommit( "b580f835-f041-4361-aa46-57e5dc338a63", new DateTimeOffset(2019, 10, 25, 0, 0, 0, TimeSpan.Zero)); Options.Setup(x => x.Value).Returns(() => Config); Storage .Setup(x => x.ReadIndexOrNullAsync(It.IsAny <HiveType>(), It.IsAny <string>())) .ReturnsAsync(() => RegistrationIndex); var concreteHiveMerger = new HiveMerger(Options.Object, output.GetLogger <HiveMerger>()); Merger .Setup(x => x.MergeAsync(It.IsAny <IndexInfo>(), It.IsAny <IReadOnlyList <CatalogCommitItem> >())) .Returns <IndexInfo, IReadOnlyList <CatalogCommitItem> >((i, e) => concreteHiveMerger.MergeAsync(i, e)); EntityBuilder .Setup(x => x.NewLeaf(It.IsAny <RegistrationLeafItem>())) .Returns(() => RegistrationLeaf); EntityBuilder .Setup(x => x.UpdateNonInlinedPageItem( It.IsAny <RegistrationPage>(), It.IsAny <HiveType>(), It.IsAny <string>(), It.IsAny <int>(), It.IsAny <NuGetVersion>(), It.IsAny <NuGetVersion>())) .Callback <RegistrationPage, HiveType, string, int, NuGetVersion, NuGetVersion>((p, h, id, c, l, u) => { p.Url = $"https://example/reg/" + $"{id.ToLowerInvariant()}/" + $"{l.ToNormalizedString().ToLowerInvariant()}/" + $"{u.ToNormalizedString().ToLowerInvariant()}.json"; }); Target = new HiveUpdater( Storage.Object, Merger.Object, EntityBuilder.Object, Options.Object, Logger); }