private async Task ProcessHiveAsync(
HiveType hive,
string id,
IReadOnlyList <CatalogCommitItem> entries,
IReadOnlyDictionary <CatalogCommitItem, PackageDetailsCatalogLeaf> entryToLeaf,
DateTimeOffset registrationCommitTimestamp)
{
var registrationCommitId = Guid.NewGuid().ToString();
var registrationCommit = new CatalogCommit(registrationCommitId, registrationCommitTimestamp);
using (_logger.BeginScope(
"Processing package {PackageId}, " +
"hive {Hive}, " +
"replica hives {ReplicaHives}, " +
"registration commit ID {RegistrationCommitId}, " +
"registration commit timestamp {RegistrationCommitTimestamp:O}.",
id,
hive,
HiveToReplicaHives[hive],
registrationCommitId,
registrationCommitTimestamp))
{
_logger.LogInformation("Processing {Count} catalog commit items.", entries.Count);
await _hiveUpdater.UpdateAsync(hive, HiveToReplicaHives[hive], id, entries, entryToLeaf, registrationCommit);
}
}
public Facts()
{
Options = new Mock <IOptionsSnapshot <Catalog2RegistrationConfiguration> >();
Config = new Catalog2RegistrationConfiguration
{
LegacyBaseUrl = "https://example/reg/",
GzippedBaseUrl = "https://example/reg-gz/",
SemVer2BaseUrl = "https://example/reg-gz-semver2/",
GalleryBaseUrl = "https://example-gallery/",
FlatContainerBaseUrl = "https://example/fc/",
};
Options.Setup(x => x.Value).Returns(() => Config);
LeafItem = new RegistrationLeafItem();
Page = new RegistrationPage();
Index = new RegistrationIndex();
Hive = HiveType.Legacy;
Id = V3Data.PackageId;
PackageDetails = new PackageDetailsCatalogLeaf
{
PackageVersion = V3Data.NormalizedVersion,
};
Commit = new CatalogCommit(V3Data.CommitId, V3Data.CommitTimestamp);
SerializerSettings = NuGetJsonSerialization.Settings;
SerializerSettings.Formatting = Formatting.Indented;
}
private async Task UpdateInlinedPagesAsync(
HiveType hive,
string id,
IndexInfo indexInfo,
CatalogCommit registrationCommit)
{
for (var pageIndex = 0; pageIndex < indexInfo.Items.Count; pageIndex++)
{
var pageInfo = indexInfo.Items[pageIndex];
if (!pageInfo.IsInlined)
{
_logger.LogInformation(
"Moving page {PageNumber}/{PageCount} [{Lower}, {Upper}] from having its own blob to being inlined.",
pageIndex + 1,
indexInfo.Items.Count,
pageInfo.Lower.ToNormalizedString(),
pageInfo.Upper.ToNormalizedString());
pageInfo = await pageInfo.CloneToInlinedAsync();
indexInfo.RemoveAt(pageIndex);
indexInfo.Insert(pageIndex, pageInfo);
}
Guard.Assert(pageInfo.IsInlined, "The page should be inlined at this point.");
_entityBuilder.UpdateInlinedPageItem(pageInfo.PageItem, hive, id, pageInfo.Count, pageInfo.Lower, pageInfo.Upper);
_entityBuilder.UpdateCommit(pageInfo.PageItem, registrationCommit);
}
}
private async Task UpdateLeavesAsync(
HiveType hive,
IReadOnlyList <HiveType> replicaHives,
string id,
Dictionary <NuGetVersion, PackageDetailsCatalogLeaf> versionToCatalogLeaf,
CatalogCommit registrationCommit,
HiveMergeResult mergeResult)
{
if (!mergeResult.ModifiedLeaves.Any())
{
_logger.LogInformation("No leaves need to be updated.");
return;
}
_logger.LogInformation(
"Updating {Count} registration leaves.",
mergeResult.ModifiedLeaves.Count,
id,
hive);
var taskFactories = new ConcurrentBag <Func <Task> >();
foreach (var leafInfo in mergeResult.ModifiedLeaves)
{
_entityBuilder.UpdateLeafItem(leafInfo.LeafItem, hive, id, versionToCatalogLeaf[leafInfo.Version]);
_entityBuilder.UpdateCommit(leafInfo.LeafItem, registrationCommit);
var leaf = _entityBuilder.NewLeaf(leafInfo.LeafItem);
taskFactories.Add(async() =>
{
_logger.LogInformation("Updating leaf {PackageId} {Version}.", id, leafInfo.Version.ToNormalizedString());
await _storage.WriteLeafAsync(hive, replicaHives, id, leafInfo.Version, leaf);
});
}
await ParallelAsync.Repeat(
async() =>
{
await Task.Yield();
while (taskFactories.TryTake(out var taskFactory))
{
await taskFactory();
}
},
_options.Value.MaxConcurrentOperationsPerHive);
}
public void UpdateCommit(ICommitted committed, CatalogCommit commit)
{
committed.CommitId = commit.Id;
committed.CommitTimestamp = commit.Timestamp;
}
public async Task UpdateAsync(
HiveType hive,
IReadOnlyList <HiveType> replicaHives,
string id,
IReadOnlyList <CatalogCommitItem> entries,
IReadOnlyDictionary <CatalogCommitItem, PackageDetailsCatalogLeaf> entryToCatalogLeaf,
CatalogCommit registrationCommit)
{
// Validate the input and put it in more convenient forms.
if (!entries.Any())
{
return;
}
GuardInput(entries, entryToCatalogLeaf);
var sortedCatalog = entries.OrderBy(x => x.PackageIdentity.Version).ToList();
var versionToCatalogLeaf = entryToCatalogLeaf.ToDictionary(x => x.Key.PackageIdentity.Version, x => x.Value);
// Remove SemVer 2.0.0 versions if this hive should only have SemVer 1.0.0 versions.
if (ShouldExcludeSemVer2(hive))
{
Guard.Assert(
replicaHives.All(ShouldExcludeSemVer2),
"A replica hive of a non-SemVer 2.0.0 hive must also exclude SemVer 2.0.0.");
ExcludeSemVer2(hive, sortedCatalog, versionToCatalogLeaf);
}
else
{
Guard.Assert(
replicaHives.All(x => !ShouldExcludeSemVer2(x)),
"A replica hive of a SemVer 2.0.0 hive must also include SemVer 2.0.0.");
}
_logger.LogInformation(
"Starting to update the {PackageId} registration index in the {Hive} hive and {ReplicaHives} replica hives with {UpsertCount} " +
"package details and {DeleteCount} package deletes.",
id,
hive,
replicaHives,
entryToCatalogLeaf.Count,
entries.Count - entryToCatalogLeaf.Count);
// Read the existing registration index if it exists. If it does not exist, initialize a new index.
var index = await _storage.ReadIndexOrNullAsync(hive, id);
IndexInfo indexInfo;
if (index == null)
{
indexInfo = IndexInfo.New();
}
else
{
indexInfo = IndexInfo.Existing(_storage, hive, index);
}
// Find all of the existing page URLs. This will be used later to find orphan pages.
var existingPageUrls = GetPageUrls(indexInfo);
// Read all of the obviously relevant pages in parallel. This simply evaluates some work that would
// otherwise be done lazily.
await LoadRelevantPagesAsync(sortedCatalog, indexInfo);
// Merge the incoming catalog entries in memory.
var mergeResult = await _merger.MergeAsync(indexInfo, sortedCatalog);
// Write the modified leaves.
await UpdateLeavesAsync(hive, replicaHives, id, versionToCatalogLeaf, registrationCommit, mergeResult);
// Write the pages and handle the inline vs. non-inlined cases.
if (indexInfo.Items.Count == 0)
{
_logger.LogInformation("There are no pages to update.");
}
else
{
var itemCount = indexInfo.Items.Sum(x => x.Count);
if (itemCount <= _options.Value.MaxInlinedLeafItems)
{
_logger.LogInformation(
"There are {Count} total leaf items so the leaf items will be inlined.",
itemCount);
await UpdateInlinedPagesAsync(hive, id, indexInfo, registrationCommit);
}
else
{
_logger.LogInformation(
"There are {Count} total leaf items so the leaf items will not be inlined.",
itemCount);
await UpdateNonInlinedPagesAsync(hive, replicaHives, id, indexInfo, registrationCommit, mergeResult);
}
}
// Write the index, if there were any changes.
if (mergeResult.ModifiedPages.Any() || mergeResult.ModifiedLeaves.Any())
{
_logger.LogInformation("Updating the index.");
_entityBuilder.UpdateIndex(indexInfo.Index, hive, id, indexInfo.Items.Count);
_entityBuilder.UpdateCommit(indexInfo.Index, registrationCommit);
await _storage.WriteIndexAsync(hive, replicaHives, id, indexInfo.Index);
}
if (!indexInfo.Items.Any())
{
_logger.LogInformation("Deleting the index since there are no more page items.");
await _storage.DeleteIndexAsync(hive, replicaHives, id);
}
// Delete orphan blobs.
await DeleteOrphansAsync(hive, replicaHives, existingPageUrls, indexInfo, mergeResult);
_logger.LogInformation(
"Done updating the {PackageId} registration index in the {Hive} hive and replica hives {ReplicaHives}. {ModifiedPages} pages were " +
"updated, {ModifiedLeaves} leaves were upserted, and {DeletedLeaves} leaves were deleted.",
id,
hive,
replicaHives,
mergeResult.ModifiedPages.Count,
mergeResult.ModifiedLeaves.Count,
mergeResult.DeletedLeaves.Count);
}
private async Task UpdateNonInlinedPagesAsync(
HiveType hive,
IReadOnlyList <HiveType> replicaHives,
string id,
IndexInfo indexInfo,
CatalogCommit registrationCommit,
HiveMergeResult mergeResult)
{
var taskFactories = new ConcurrentBag <Func <Task> >();
for (var pageIndex = 0; pageIndex < indexInfo.Items.Count; pageIndex++)
{
var pageInfo = indexInfo.Items[pageIndex];
if (pageInfo.IsInlined)
{
_logger.LogInformation(
"Moving page {PageNumber}/{PageCount} [{Lower}, {Upper}] from being inlined to having its own blob.",
pageIndex + 1,
indexInfo.Items.Count,
pageInfo.Lower.ToNormalizedString(),
pageInfo.Upper.ToNormalizedString());
pageInfo = await pageInfo.CloneToNonInlinedAsync();
indexInfo.RemoveAt(pageIndex);
indexInfo.Insert(pageIndex, pageInfo);
}
else if (!mergeResult.ModifiedPages.Contains(pageInfo))
{
_logger.LogInformation(
"Skipping unmodified page {PageNumber}/{PageCount} [{Lower}, {Upper}].",
pageIndex + 1,
indexInfo.Items.Count,
pageInfo.Lower.ToNormalizedString(),
pageInfo.Upper.ToNormalizedString());
continue;
}
Guard.Assert(!pageInfo.IsInlined, "The page should not be inlined at this point.");
var page = await pageInfo.GetPageAsync();
_entityBuilder.UpdateNonInlinedPageItem(pageInfo.PageItem, hive, id, pageInfo.Count, pageInfo.Lower, pageInfo.Upper);
_entityBuilder.UpdateCommit(pageInfo.PageItem, registrationCommit);
_entityBuilder.UpdatePage(page, hive, id, pageInfo.Count, pageInfo.Lower, pageInfo.Upper);
_entityBuilder.UpdateCommit(page, registrationCommit);
var pageNumber = pageIndex + 1;
taskFactories.Add(async() =>
{
_logger.LogInformation(
"Updating page {PageNumber}/{PageCount} [{Lower}, {Upper}].",
pageNumber,
indexInfo.Items.Count,
pageInfo.Lower.ToNormalizedString(),
pageInfo.Upper.ToNormalizedString());
await _storage.WritePageAsync(hive, replicaHives, id, pageInfo.Lower, pageInfo.Upper, page);
});
}
await ParallelAsync.Repeat(
async() =>
{
await Task.Yield();
while (taskFactories.TryTake(out var taskFactory))
{
await taskFactory();
}
},
_options.Value.MaxConcurrentOperationsPerHive);
}
public Facts(ITestOutputHelper output)
{
Storage = new Mock <IHiveStorage>();
Merger = new Mock <IHiveMerger>();
EntityBuilder = new Mock <IEntityBuilder>();
Options = new Mock <IOptionsSnapshot <Catalog2RegistrationConfiguration> >();
Logger = output.GetLogger <HiveUpdater>();
Config = new Catalog2RegistrationConfiguration();
Hive = HiveType.SemVer2;
ReplicaHives = new List <HiveType>();
Id = "NuGet.Versioning";
Entries = new List <CatalogCommitItem>();
EntryToCatalogLeaf = new Dictionary <CatalogCommitItem, PackageDetailsCatalogLeaf>(
ReferenceEqualityComparer <CatalogCommitItem> .Default);
RegistrationIndex = new RegistrationIndex
{
Items = new List <RegistrationPage>
{
new RegistrationPage
{
Lower = "1.0.0",
Upper = "3.0.0",
Count = 2,
Items = new List <RegistrationLeafItem>
{
new RegistrationLeafItem
{
Url = $"https://example/reg/{Id.ToLowerInvariant()}/1.0.0.json",
CatalogEntry = new RegistrationCatalogEntry
{
Version = "1.0.0",
}
},
new RegistrationLeafItem
{
Url = $"https://example/reg/{Id.ToLowerInvariant()}/3.0.0.json",
CatalogEntry = new RegistrationCatalogEntry
{
Version = "3.0.0",
}
},
}
}
}
};
MergeResult = new HiveMergeResult(
new HashSet <PageInfo>(),
new HashSet <LeafInfo>(),
new HashSet <LeafInfo>());
RegistrationLeaf = new RegistrationLeaf();
RegistrationCommit = new CatalogCommit(
"b580f835-f041-4361-aa46-57e5dc338a63",
new DateTimeOffset(2019, 10, 25, 0, 0, 0, TimeSpan.Zero));
Options.Setup(x => x.Value).Returns(() => Config);
Storage
.Setup(x => x.ReadIndexOrNullAsync(It.IsAny <HiveType>(), It.IsAny <string>()))
.ReturnsAsync(() => RegistrationIndex);
var concreteHiveMerger = new HiveMerger(Options.Object, output.GetLogger <HiveMerger>());
Merger
.Setup(x => x.MergeAsync(It.IsAny <IndexInfo>(), It.IsAny <IReadOnlyList <CatalogCommitItem> >()))
.Returns <IndexInfo, IReadOnlyList <CatalogCommitItem> >((i, e) => concreteHiveMerger.MergeAsync(i, e));
EntityBuilder
.Setup(x => x.NewLeaf(It.IsAny <RegistrationLeafItem>()))
.Returns(() => RegistrationLeaf);
EntityBuilder
.Setup(x => x.UpdateNonInlinedPageItem(
It.IsAny <RegistrationPage>(),
It.IsAny <HiveType>(),
It.IsAny <string>(),
It.IsAny <int>(),
It.IsAny <NuGetVersion>(),
It.IsAny <NuGetVersion>()))
.Callback <RegistrationPage, HiveType, string, int, NuGetVersion, NuGetVersion>((p, h, id, c, l, u) =>
{
p.Url = $"https://example/reg/" +
$"{id.ToLowerInvariant()}/" +
$"{l.ToNormalizedString().ToLowerInvariant()}/" +
$"{u.ToNormalizedString().ToLowerInvariant()}.json";
});
Target = new HiveUpdater(
Storage.Object,
Merger.Object,
EntityBuilder.Object,
Options.Object,
Logger);
}