public Facts(ITestOutputHelper output)
{
Options = new Mock <IOptionsSnapshot <Catalog2RegistrationConfiguration> >();
Logger = output.GetLogger <HiveMerger>();
Config = new Catalog2RegistrationConfiguration
{
MaxLeavesPerPage = 3,
};
Options.Setup(x => x.Value).Returns(() => Config);
Target = new HiveMerger(
Options.Object,
Logger);
}
public IntegrationTests(ITestOutputHelper output)
{
Options = new Mock <IOptionsSnapshot <Catalog2RegistrationConfiguration> >();
Config = new Catalog2RegistrationConfiguration
{
LegacyBaseUrl = "https://example/v3/reg",
LegacyStorageContainer = "v3-reg",
GzippedBaseUrl = "https://example/v3/reg-gz",
GzippedStorageContainer = "v3-reg-gz",
SemVer2BaseUrl = "https://example/v3/reg-gz-semver2",
SemVer2StorageContainer = "v3-reg-gz-semver2",
FlatContainerBaseUrl = "https://example/v3/flatcontainer",
GalleryBaseUrl = "https://example-gallery",
MaxConcurrentHivesPerId = 1,
MaxConcurrentIds = 1,
MaxConcurrentOperationsPerHive = 1,
MaxConcurrentStorageOperations = 1,
EnsureSingleSnapshot = false,
};
Options.Setup(x => x.Value).Returns(() => Config);
CloudBlobClient = new InMemoryCloudBlobClient();
RegistrationUrlBuilder = new RegistrationUrlBuilder(Options.Object);
EntityBuilder = new EntityBuilder(RegistrationUrlBuilder, Options.Object);
Throttle = NullThrottle.Instance;
HiveStorage = new HiveStorage(
CloudBlobClient,
RegistrationUrlBuilder,
EntityBuilder,
Throttle,
Options.Object,
output.GetLogger <HiveStorage>());
HiveMerger = new HiveMerger(Options.Object, output.GetLogger <HiveMerger>());
HiveUpdater = new HiveUpdater(
HiveStorage,
HiveMerger,
EntityBuilder,
Options.Object,
output.GetLogger <HiveUpdater>());
RegistrationUpdater = new RegistrationUpdater(
HiveUpdater,
Options.Object,
output.GetLogger <RegistrationUpdater>());
}
[InlineData(3, 2, 5, 1, 6)] // Use page size 3 to allow insertions without bounds changes.
public async Task Execute(int maxLeavesPerPage, int minExisting, int maxExisting, int minUpdated, int maxUpdated)
{
var config = new Catalog2RegistrationConfiguration();
var options = new SimpleOptions <Catalog2RegistrationConfiguration>(config);
var logger = new NullLogger <HiveMerger>();
var target = new HiveMerger(options, logger);
config.MaxLeavesPerPage = maxLeavesPerPage;
var allExisting = Enumerable.Range(minExisting, (maxExisting - minExisting) + 1).Select(m => new NuGetVersion(m, 0, 0)).ToList();
var allUpdated = Enumerable.Range(minUpdated, (maxUpdated - minUpdated) + 1).Select(m => new NuGetVersion(m, 0, 0)).ToList();
var versionToNormalized = allExisting.Concat(allUpdated).Distinct().ToDictionary(v => v, v => v.ToNormalizedString());
// Enumerate all of the updated version sets, which is up to 9 versions and for each set every
// combination of either PackageDelete or PackageDetails for each version.
var deleteOrNotDelete = new[] { true, false };
var updatedCases = IterTools
.SubsetsOf(allUpdated)
.SelectMany(vs => IterTools.CombinationsOfTwo(vs.ToList(), deleteOrNotDelete))
.Select(ts => ts.Select(t => new VersionAction(t.Item1, t.Item2)).OrderBy(v => v.Version).ToList())
.ToList();
// Enumerate all of the updated version sets, which is up to 7 versions.
var existingCases = IterTools
.SubsetsOf(allExisting)
.Select(vs => vs.OrderBy(v => v).ToList())
.ToList();
// Build all of the test cases which is every pairing of updated and existing cases.
var testCases = new ConcurrentBag <TestCase>();
foreach (var existing in existingCases)
{
foreach (var updated in updatedCases)
{
testCases.Add(new TestCase(existing, updated));
}
}
var completed = 0;
var total = testCases.Count;
var outputLock = new object();
await ParallelAsync
.Repeat(async() =>
{
await Task.Yield();
while (testCases.TryTake(out var testCase))
{
try
{
await ExecuteTestCaseAsync(config, target, versionToNormalized, testCase);
var thisCompleted = Interlocked.Increment(ref completed);
if (thisCompleted % 20000 == 0)
{
lock (outputLock)
{
_output.WriteLine($"Progress: {1.0 * thisCompleted / total:P}");
}
}
}
catch (Exception ex)
{
lock (outputLock)
{
_output.WriteLine(string.Empty);
_output.WriteLine("Test Case Failure");
_output.WriteLine(new string('=', 50));
_output.WriteLine(ex.Message);
_output.WriteLine(string.Empty);
_output.WriteLine("Existing: " + string.Join(", ", testCase.Existing));
_output.WriteLine("Changes: " + string.Join(", ", testCase.Updated.Select(t => $"{(t.IsDelete ? '-' : '+')}{t.Version}")));
_output.WriteLine(new string('=', 50));
_output.WriteLine(string.Empty);
return;
}
}
}
},
degreeOfParallelism : 8);
_output.WriteLine($"Progress: {1.0 * completed / total:P}");
_output.WriteLine($"Total test cases: {total}");
_output.WriteLine($"Completed test cases: {completed}");
Assert.Equal(total, completed);
}
private async Task ExecuteTestCaseAsync(
Catalog2RegistrationConfiguration config,
HiveMerger target,
Dictionary <NuGetVersion, string> versionToNormalized,
TestCase testCase)
{
// ARRANGE
var existing = testCase.Existing;
var allUpdated = testCase.Updated;
// Determine the sets of expected modified versions, deleted versions, and resulting versions.
var updatedGrouped = allUpdated.ToLookup(x => x.IsDelete);
var possibleDeletes = updatedGrouped[true].Select(x => x.Version);
var modified = updatedGrouped[false].Select(x => x.Version).OrderBy(v => v).ToList();
var deleted = existing.Intersect(possibleDeletes).OrderBy(v => v).ToList();
var unchanged = existing.Except(allUpdated.Select(x => x.Version)).OrderBy(v => v).ToList();
var expectedRemaining = modified.Concat(unchanged).OrderBy(v => v).ToList();
// Build the input data structures.
var sortedCatalog = MakeSortedCatalog(allUpdated);
var indexInfo = MakeIndexInfo(existing, config.MaxLeavesPerPage, versionToNormalized);
// Determine which pages will definitely not be affected.
var unaffectedPages = new List <PageInfo>();
if (existing.Any())
{
var minVersion = allUpdated.Min(x => x.Version);
unaffectedPages = indexInfo
.Items
.Take(indexInfo.Items.Count - 1)
.Where(x => x.Upper < minVersion)
.ToList();
}
// ACT
var result = await target.MergeAsync(indexInfo, sortedCatalog);
// ASSERT
// Verify the definitely unaffected pages are in still in the index and not loaded.
foreach (var page in unaffectedPages)
{
Assert.Contains(page, indexInfo.Items);
Assert.False(page.IsPageFetched, "A page before the lowest version input version must not be fetched.");
}
// Verify the modified version set.
Assert.Equal(modified, result.ModifiedLeaves.Select(x => x.Version).OrderBy(v => v).ToList());
// Verify the deleted version set.
Assert.Equal(deleted, result.DeletedLeaves.Select(x => x.Version).OrderBy(v => v).ToList());
// Verify the resulting set of versions.
var actualRemaining = await GetVersionsAsync(indexInfo);
Assert.Equal(expectedRemaining, actualRemaining);
// Verify all but the last page are full.
foreach (var pageInfo in indexInfo.Items.AsEnumerable().Reverse().Skip(1))
{
Assert.True(
pageInfo.Count == config.MaxLeavesPerPage,
"All but the last page must have the maximum number of leaf items per page.");
}
// Verify the last page is not too full.
if (indexInfo.Items.Count > 0)
{
Assert.True(
indexInfo.Items.Last().Count <= config.MaxLeavesPerPage,
"The last page must have less than or equal to the maximum number of leaf items per page.");
}
// Verify the page bounds are in ascending order.
var bounds = indexInfo.Items.SelectMany(GetUniqueBounds).ToList();
for (var i = 1; i < bounds.Count; i++)
{
Assert.True(bounds[i - 1] < bounds[i], "Each page bound must be less than the next.");
}
// Verify the modified pages are in the index.
foreach (var pageInfo in result.ModifiedPages)
{
Assert.True(indexInfo.Items.Contains(pageInfo), "A modified page must be in the index.");
}
// Verify the leaf item infos match the leaf items.
Assert.True(
indexInfo.Items.Count == indexInfo.Index.Items.Count,
"The number of page infos must match the number of page items.");
for (var pageIndex = 0; pageIndex < indexInfo.Items.Count; pageIndex++)
{
var pageInfo = indexInfo.Items[pageIndex];
var leafInfos = await pageInfo.GetLeafInfosAsync();
var page = await pageInfo.GetPageAsync();
Assert.True(
page.Items.Count == leafInfos.Count,
"The number of leaf items must match the number of leaf item infos.");
for (var leafIndex = 0; leafIndex < page.Items.Count; leafIndex++)
{
var leafInfoVersion = leafInfos[leafIndex].Version;
var leafItemVersion = NuGetVersion.Parse(page.Items[leafIndex].CatalogEntry.Version);
Assert.True(
leafInfoVersion == leafItemVersion,
"The list of leaf info versions must match the leaf item versions.");
}
}
// Verify the modified leaves and deleted leafs are disjoint sets.
var modifiedVersions = new HashSet <NuGetVersion>(result.ModifiedLeaves.Select(x => x.Version));
var deletedVersions = new HashSet <NuGetVersion>(result.DeletedLeaves.Select(x => x.Version));
Assert.True(
!modifiedVersions.Overlaps(deletedVersions),
"The deleted leaves and modified leaves must be disjoint sets.");
// Verify the modified leaves are visible in an inlined page or a downloaded page.
foreach (var leafInfo in result.ModifiedLeaves)
{
foreach (var pageInfo in indexInfo.Items)
{
if (leafInfo.Version < pageInfo.Lower || leafInfo.Version > pageInfo.Upper)
{
continue;
}
Assert.True(
pageInfo.IsInlined || pageInfo.IsPageFetched,
"A page bounding a modified leaf must either be inlined or the page must be fetched.");
var pageLeafInfos = await pageInfo.GetLeafInfosAsync();
Assert.True(
pageLeafInfos.Any(x => x.Version == leafInfo.Version),
"A modified leaf must be in the page bounding its version.");
}
}
// Verify the deleted leaves are not visible in any of the inlined or downloaded pages.
foreach (var leafInfo in result.DeletedLeaves)
{
foreach (var pageInfo in indexInfo.Items)
{
var pageLeafInfos = await pageInfo.GetLeafInfosAsync();
Assert.True(
!pageLeafInfos.Any(x => x.Version == leafInfo.Version),
"A deleted leaf must not be in any page.");
}
}
}
public Facts(ITestOutputHelper output)
{
Storage = new Mock <IHiveStorage>();
Merger = new Mock <IHiveMerger>();
EntityBuilder = new Mock <IEntityBuilder>();
Options = new Mock <IOptionsSnapshot <Catalog2RegistrationConfiguration> >();
Logger = output.GetLogger <HiveUpdater>();
Config = new Catalog2RegistrationConfiguration();
Hive = HiveType.SemVer2;
ReplicaHives = new List <HiveType>();
Id = "NuGet.Versioning";
Entries = new List <CatalogCommitItem>();
EntryToCatalogLeaf = new Dictionary <CatalogCommitItem, PackageDetailsCatalogLeaf>(
ReferenceEqualityComparer <CatalogCommitItem> .Default);
RegistrationIndex = new RegistrationIndex
{
Items = new List <RegistrationPage>
{
new RegistrationPage
{
Lower = "1.0.0",
Upper = "3.0.0",
Count = 2,
Items = new List <RegistrationLeafItem>
{
new RegistrationLeafItem
{
Url = $"https://example/reg/{Id.ToLowerInvariant()}/1.0.0.json",
CatalogEntry = new RegistrationCatalogEntry
{
Version = "1.0.0",
}
},
new RegistrationLeafItem
{
Url = $"https://example/reg/{Id.ToLowerInvariant()}/3.0.0.json",
CatalogEntry = new RegistrationCatalogEntry
{
Version = "3.0.0",
}
},
}
}
}
};
MergeResult = new HiveMergeResult(
new HashSet <PageInfo>(),
new HashSet <LeafInfo>(),
new HashSet <LeafInfo>());
RegistrationLeaf = new RegistrationLeaf();
RegistrationCommit = new CatalogCommit(
"b580f835-f041-4361-aa46-57e5dc338a63",
new DateTimeOffset(2019, 10, 25, 0, 0, 0, TimeSpan.Zero));
Options.Setup(x => x.Value).Returns(() => Config);
Storage
.Setup(x => x.ReadIndexOrNullAsync(It.IsAny <HiveType>(), It.IsAny <string>()))
.ReturnsAsync(() => RegistrationIndex);
var concreteHiveMerger = new HiveMerger(Options.Object, output.GetLogger <HiveMerger>());
Merger
.Setup(x => x.MergeAsync(It.IsAny <IndexInfo>(), It.IsAny <IReadOnlyList <CatalogCommitItem> >()))
.Returns <IndexInfo, IReadOnlyList <CatalogCommitItem> >((i, e) => concreteHiveMerger.MergeAsync(i, e));
EntityBuilder
.Setup(x => x.NewLeaf(It.IsAny <RegistrationLeafItem>()))
.Returns(() => RegistrationLeaf);
EntityBuilder
.Setup(x => x.UpdateNonInlinedPageItem(
It.IsAny <RegistrationPage>(),
It.IsAny <HiveType>(),
It.IsAny <string>(),
It.IsAny <int>(),
It.IsAny <NuGetVersion>(),
It.IsAny <NuGetVersion>()))
.Callback <RegistrationPage, HiveType, string, int, NuGetVersion, NuGetVersion>((p, h, id, c, l, u) =>
{
p.Url = $"https://example/reg/" +
$"{id.ToLowerInvariant()}/" +
$"{l.ToNormalizedString().ToLowerInvariant()}/" +
$"{u.ToNormalizedString().ToLowerInvariant()}.json";
});
Target = new HiveUpdater(
Storage.Object,
Merger.Object,
EntityBuilder.Object,
Options.Object,
Logger);
}
