private async Task DeleteOrphansAsync(
HiveType hive,
IReadOnlyList <HiveType> replicaHives,
IEnumerable <string> existingPageUrls,
IndexInfo indexInfo,
HiveMergeResult mergeResult)
{
// Start with all of the page URLs found in the index prior to the update process.
var orphanUrls = new HashSet <string>(existingPageUrls);
// Add all of the deleted leaf URLs.
orphanUrls.UnionWith(mergeResult.DeletedLeaves.Select(x => x.LeafItem.Url));
// Leave the new page URLs alone.
foreach (var pageInfo in indexInfo.Items)
{
orphanUrls.Remove(pageInfo.PageItem.Url);
}
// Leave the modified leaf URLs alone. This should not be necessary since deleted leaves and modified
// leaves are disjoint sets but is a reasonable precaution.
foreach (var leafInfo in mergeResult.ModifiedLeaves)
{
orphanUrls.Remove(leafInfo.LeafItem.Url);
}
if (orphanUrls.Count == 0)
{
_logger.LogInformation("There are no orphan blobs to delete.");
return;
}
_logger.LogInformation("About to delete {Count} orphan blobs.", orphanUrls.Count);
var work = new ConcurrentBag <string>(orphanUrls);
await ParallelAsync.Repeat(
async() =>
{
while (work.TryTake(out var url))
{
await _storage.DeleteUrlAsync(hive, replicaHives, url);
}
},
_options.Value.MaxConcurrentOperationsPerHive);
_logger.LogInformation("Done deleting orphan blobs.", orphanUrls.Count);
}
private async Task UpdateLeavesAsync(
HiveType hive,
IReadOnlyList <HiveType> replicaHives,
string id,
Dictionary <NuGetVersion, PackageDetailsCatalogLeaf> versionToCatalogLeaf,
CatalogCommit registrationCommit,
HiveMergeResult mergeResult)
{
if (!mergeResult.ModifiedLeaves.Any())
{
_logger.LogInformation("No leaves need to be updated.");
return;
}
_logger.LogInformation(
"Updating {Count} registration leaves.",
mergeResult.ModifiedLeaves.Count,
id,
hive);
var taskFactories = new ConcurrentBag <Func <Task> >();
foreach (var leafInfo in mergeResult.ModifiedLeaves)
{
_entityBuilder.UpdateLeafItem(leafInfo.LeafItem, hive, id, versionToCatalogLeaf[leafInfo.Version]);
_entityBuilder.UpdateCommit(leafInfo.LeafItem, registrationCommit);
var leaf = _entityBuilder.NewLeaf(leafInfo.LeafItem);
taskFactories.Add(async() =>
{
_logger.LogInformation("Updating leaf {PackageId} {Version}.", id, leafInfo.Version.ToNormalizedString());
await _storage.WriteLeafAsync(hive, replicaHives, id, leafInfo.Version, leaf);
});
}
await ParallelAsync.Repeat(
async() =>
{
await Task.Yield();
while (taskFactories.TryTake(out var taskFactory))
{
await taskFactory();
}
},
_options.Value.MaxConcurrentOperationsPerHive);
}
private async Task UpdateNonInlinedPagesAsync(
HiveType hive,
IReadOnlyList <HiveType> replicaHives,
string id,
IndexInfo indexInfo,
CatalogCommit registrationCommit,
HiveMergeResult mergeResult)
{
var taskFactories = new ConcurrentBag <Func <Task> >();
for (var pageIndex = 0; pageIndex < indexInfo.Items.Count; pageIndex++)
{
var pageInfo = indexInfo.Items[pageIndex];
if (pageInfo.IsInlined)
{
_logger.LogInformation(
"Moving page {PageNumber}/{PageCount} [{Lower}, {Upper}] from being inlined to having its own blob.",
pageIndex + 1,
indexInfo.Items.Count,
pageInfo.Lower.ToNormalizedString(),
pageInfo.Upper.ToNormalizedString());
pageInfo = await pageInfo.CloneToNonInlinedAsync();
indexInfo.RemoveAt(pageIndex);
indexInfo.Insert(pageIndex, pageInfo);
}
else if (!mergeResult.ModifiedPages.Contains(pageInfo))
{
_logger.LogInformation(
"Skipping unmodified page {PageNumber}/{PageCount} [{Lower}, {Upper}].",
pageIndex + 1,
indexInfo.Items.Count,
pageInfo.Lower.ToNormalizedString(),
pageInfo.Upper.ToNormalizedString());
continue;
}
Guard.Assert(!pageInfo.IsInlined, "The page should not be inlined at this point.");
var page = await pageInfo.GetPageAsync();
_entityBuilder.UpdateNonInlinedPageItem(pageInfo.PageItem, hive, id, pageInfo.Count, pageInfo.Lower, pageInfo.Upper);
_entityBuilder.UpdateCommit(pageInfo.PageItem, registrationCommit);
_entityBuilder.UpdatePage(page, hive, id, pageInfo.Count, pageInfo.Lower, pageInfo.Upper);
_entityBuilder.UpdateCommit(page, registrationCommit);
var pageNumber = pageIndex + 1;
taskFactories.Add(async() =>
{
_logger.LogInformation(
"Updating page {PageNumber}/{PageCount} [{Lower}, {Upper}].",
pageNumber,
indexInfo.Items.Count,
pageInfo.Lower.ToNormalizedString(),
pageInfo.Upper.ToNormalizedString());
await _storage.WritePageAsync(hive, replicaHives, id, pageInfo.Lower, pageInfo.Upper, page);
});
}
await ParallelAsync.Repeat(
async() =>
{
await Task.Yield();
while (taskFactories.TryTake(out var taskFactory))
{
await taskFactory();
}
},
_options.Value.MaxConcurrentOperationsPerHive);
}
public Facts(ITestOutputHelper output)
{
Storage = new Mock <IHiveStorage>();
Merger = new Mock <IHiveMerger>();
EntityBuilder = new Mock <IEntityBuilder>();
Options = new Mock <IOptionsSnapshot <Catalog2RegistrationConfiguration> >();
Logger = output.GetLogger <HiveUpdater>();
Config = new Catalog2RegistrationConfiguration();
Hive = HiveType.SemVer2;
ReplicaHives = new List <HiveType>();
Id = "NuGet.Versioning";
Entries = new List <CatalogCommitItem>();
EntryToCatalogLeaf = new Dictionary <CatalogCommitItem, PackageDetailsCatalogLeaf>(
ReferenceEqualityComparer <CatalogCommitItem> .Default);
RegistrationIndex = new RegistrationIndex
{
Items = new List <RegistrationPage>
{
new RegistrationPage
{
Lower = "1.0.0",
Upper = "3.0.0",
Count = 2,
Items = new List <RegistrationLeafItem>
{
new RegistrationLeafItem
{
Url = $"https://example/reg/{Id.ToLowerInvariant()}/1.0.0.json",
CatalogEntry = new RegistrationCatalogEntry
{
Version = "1.0.0",
}
},
new RegistrationLeafItem
{
Url = $"https://example/reg/{Id.ToLowerInvariant()}/3.0.0.json",
CatalogEntry = new RegistrationCatalogEntry
{
Version = "3.0.0",
}
},
}
}
}
};
MergeResult = new HiveMergeResult(
new HashSet <PageInfo>(),
new HashSet <LeafInfo>(),
new HashSet <LeafInfo>());
RegistrationLeaf = new RegistrationLeaf();
RegistrationCommit = new CatalogCommit(
"b580f835-f041-4361-aa46-57e5dc338a63",
new DateTimeOffset(2019, 10, 25, 0, 0, 0, TimeSpan.Zero));
Options.Setup(x => x.Value).Returns(() => Config);
Storage
.Setup(x => x.ReadIndexOrNullAsync(It.IsAny <HiveType>(), It.IsAny <string>()))
.ReturnsAsync(() => RegistrationIndex);
var concreteHiveMerger = new HiveMerger(Options.Object, output.GetLogger <HiveMerger>());
Merger
.Setup(x => x.MergeAsync(It.IsAny <IndexInfo>(), It.IsAny <IReadOnlyList <CatalogCommitItem> >()))
.Returns <IndexInfo, IReadOnlyList <CatalogCommitItem> >((i, e) => concreteHiveMerger.MergeAsync(i, e));
EntityBuilder
.Setup(x => x.NewLeaf(It.IsAny <RegistrationLeafItem>()))
.Returns(() => RegistrationLeaf);
EntityBuilder
.Setup(x => x.UpdateNonInlinedPageItem(
It.IsAny <RegistrationPage>(),
It.IsAny <HiveType>(),
It.IsAny <string>(),
It.IsAny <int>(),
It.IsAny <NuGetVersion>(),
It.IsAny <NuGetVersion>()))
.Callback <RegistrationPage, HiveType, string, int, NuGetVersion, NuGetVersion>((p, h, id, c, l, u) =>
{
p.Url = $"https://example/reg/" +
$"{id.ToLowerInvariant()}/" +
$"{l.ToNormalizedString().ToLowerInvariant()}/" +
$"{u.ToNormalizedString().ToLowerInvariant()}.json";
});
Target = new HiveUpdater(
Storage.Object,
Merger.Object,
EntityBuilder.Object,
Options.Object,
Logger);
}
