public async Task FromCatalogEntry_HandlesDeleted_NotNull()
{
// Arrange
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
var client = await CreateDummyClient(catalogStorage);
var expectedTimestamp = DateTime.Parse("2015-01-01T01:01:01.0748028");
var uri = new Uri(catalogStorage.BaseAddress, "data/2015.10.13.06.40.07/otherpackage.1.0.0.json");
var catalogIndexEntry = new CatalogIndexEntry(
uri,
CatalogConstants.NuGetPackageDelete,
"afc8c1f4-486e-4142-b3ec-cf5841eb8883",
DateTime.ParseExact(
"2015-10-13T06:40:07.7850657Z",
CatalogConstants.CommitTimeStampFormat,
DateTimeFormatInfo.CurrentInfo,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal),
new PackageIdentity("OtherPackage", new NuGetVersion("1.0.0")));
// Act
var entry = await PackageTimestampMetadata.FromCatalogEntry(client, catalogIndexEntry);
// Assert
Assert.False(entry.Exists);
Assert.Null(entry.Created);
Assert.Null(entry.LastEdited);
Assert.Equal(expectedTimestamp.Ticks, entry.Deleted.Value.Ticks);
Assert.Equal(expectedTimestamp.Ticks, entry.Last.Value.Ticks);
}
public async Task RunAsync_WhenDownloadingPackage_RejectsUnexpectedHttpStatusCode(HttpStatusCode statusCode)
{
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.Return404OnUnknownAction = true;
_mockServer.SetAction(
"/packages/listedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(statusCode)
{
Content = _noContent
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
var exception = await Assert.ThrowsAsync <InvalidOperationException>(
() => _target.RunAsync(front, back, CancellationToken.None));
Assert.Equal(
$"Expected status code OK for package download, actual: {statusCode}",
exception.Message);
Assert.Equal(0, _catalogToDnxStorage.Content.Count);
}
public async Task FromCatalogEntry_HandlesCreatedLastEdited()
{
// Arrange
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
var client = await CreateDummyClient(catalogStorage);
var expectedTimestamp = DateTime.Parse("2015-01-01T00:00:00");
var uris = new List <Uri>
{
new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/listedpackage.1.0.0.json"),
new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json"),
new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.55/listedpackage.1.0.1.json")
};
// Act
var entries = await Task.WhenAll(uris.Select(uri => PackageTimestampMetadata.FromCatalogEntry(client, new CatalogIndexEntry(uri, null, null, DateTime.MinValue, null, null))));
// Assert
foreach (var entry in entries)
{
Assert.True(entry.Exists);
Assert.Equal(expectedTimestamp.Ticks, entry.Created.Value.Ticks);
Assert.Equal(expectedTimestamp.Ticks, entry.LastEdited.Value.Ticks);
Assert.Null(entry.Deleted);
Assert.Equal(expectedTimestamp.Ticks, entry.Last.Value.Ticks);
}
}
public async Task RunAsync_WithValidPackage_RespectsDeletion()
{
var indexJsonUri = _catalogToDnxStorage.ResolveUri("/otherpackage/index.json");
var nupkgUri = _catalogToDnxStorage.ResolveUri("/otherpackage/1.0.0/otherpackage.1.0.0.nupkg");
var nuspecUri = _catalogToDnxStorage.ResolveUri("/otherpackage/1.0.0/otherpackage.nuspec");
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/otherpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\OtherPackage.1.0.0.zip"))
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
await _target.RunAsync(front, back, CancellationToken.None);
Assert.Equal(1, _catalogToDnxStorage.Content.Count);
Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(indexJsonUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri));
}
public async Task FromCatalogEntry_HandlesCreatedLastEdited()
{
// Arrange
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
var client = await CreateDummyClient(catalogStorage);
var expectedTimestamp = DateTime.Parse("2015-01-01T00:00:00");
var commitTimeStamp1 = DateTime.ParseExact(
"2015-10-12T10:08:54.1506742Z",
CatalogConstants.CommitTimeStampFormat,
DateTimeFormatInfo.CurrentInfo,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);
var commitTimeStamp2 = DateTime.ParseExact(
"2015-10-12T10:08:55.3335317Z",
CatalogConstants.CommitTimeStampFormat,
DateTimeFormatInfo.CurrentInfo,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);
var tasks = new Task <PackageTimestampMetadata>[]
{
PackageTimestampMetadata.FromCatalogEntry(
client,
new CatalogIndexEntry(
new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/listedpackage.1.0.0.json"),
CatalogConstants.NuGetPackageDetails,
"9a37734f-1960-4c07-8934-c8bc797e35c1",
commitTimeStamp1,
new PackageIdentity("ListedPackage", new NuGetVersion("1.0.0")))),
PackageTimestampMetadata.FromCatalogEntry(
client,
new CatalogIndexEntry(
new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json"),
CatalogConstants.NuGetPackageDetails,
"9a37734f-1960-4c07-8934-c8bc797e35c1",
commitTimeStamp1,
new PackageIdentity("UnlistedPackage", new NuGetVersion("1.0.0")))),
PackageTimestampMetadata.FromCatalogEntry(
client,
new CatalogIndexEntry(
new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.55/listedpackage.1.0.1.json"),
CatalogConstants.NuGetPackageDetails,
"8a9e7694-73d4-4775-9b7a-20aa59b9773e",
commitTimeStamp2,
new PackageIdentity("ListedPackage", new NuGetVersion("1.0.1"))))
};
// Act
var entries = await Task.WhenAll(tasks);
// Assert
foreach (var entry in entries)
{
Assert.True(entry.Exists);
Assert.Equal(expectedTimestamp.Ticks, entry.Created.Value.Ticks);
Assert.Equal(expectedTimestamp.Ticks, entry.LastEdited.Value.Ticks);
Assert.Null(entry.Deleted);
Assert.Equal(expectedTimestamp.Ticks, entry.Last.Value.Ticks);
}
}
public async Task RunAsync_WhenDownloadingPackage_OnlyDownloadsNupkgOncePerCatalogLeaf()
{
// Arrange
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/listedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.0.zip"))
}));
_mockServer.SetAction(
"/packages/listedpackage.1.0.1.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.1.zip"))
}));
_mockServer.SetAction(
"/packages/unlistedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\UnlistedPackage.1.0.0.zip"))
}));
_mockServer.SetAction(
"/packages/otherpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\OtherPackage.1.0.0.zip"))
}));
ReadWriteCursor front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
// Act
await _target.RunAsync(front, back, CancellationToken.None);
// Assert
Assert.Equal(9, _catalogToDnxStorage.Content.Count);
Assert.Equal(5, _mockServer.Requests.Count);
Assert.EndsWith("/index.json", _mockServer.Requests[0].RequestUri.AbsoluteUri);
Assert.EndsWith("/page0.json", _mockServer.Requests[1].RequestUri.AbsoluteUri);
// The packages were processed in random order.
var remainingRequests = _mockServer.Requests
.Skip(2)
.Take(3)
.Select(request => request.RequestUri.AbsoluteUri)
.OrderBy(uri => uri)
.ToArray();
Assert.Contains("/listedpackage.1.0.0.nupkg", remainingRequests[0]);
Assert.Contains("/listedpackage.1.0.1.nupkg", remainingRequests[1]);
Assert.Contains("/unlistedpackage.1.0.0.nupkg", remainingRequests[2]);
}
public async Task RunAsync_WithNonIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRequiredProperties_SkipsPackage()
{
_catalogToDnxStorage = new SynchronizedMemoryStorage(new[]
{
new Uri("http://tempuri.org/packages/listedpackage.1.0.1.nupkg"),
});
_catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name));
var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json");
var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg");
var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec");
var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.1.zip");
var expectedNupkg = GetStreamBytes(nupkgStream);
await _catalogToDnxStorage.SaveAsync(
new Uri("http://tempuri.org/listedpackage/index.json"),
new StringStorageContent(GetExpectedIndexJsonContent("1.0.1")),
CancellationToken.None);
_target = new DnxCatalogCollector(
new Uri("http://tempuri.org/index.json"),
_catalogToDnxStorageFactory,
_nullPreferredPackageSourceStorage,
_contentBaseAddress,
Mock.Of <ITelemetryService>(),
_logger,
_maxDegreeOfParallelism,
() => _mockServer);
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/listedpackage.1.0.1.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(nupkgStream)
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
await _target.RunAsync(front, back, CancellationToken.None);
Assert.Equal(2, _catalogToDnxStorage.Content.Count);
Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri));
Assert.Equal(GetExpectedIndexJsonContent("1.0.1"), Encoding.UTF8.GetString(indexJson));
}
public async Task FromCatalogEntry_HandlesDeleted_NotNull()
{
// Arrange
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
var client = await CreateDummyClient(catalogStorage);
var expectedTimestamp = DateTime.Parse("2015-01-01T01:01:01.0748028");
var uri = new Uri(catalogStorage.BaseAddress, "data/2015.10.13.06.40.07/otherpackage.1.0.0.json");
// Act
var entry = await PackageTimestampMetadata.FromCatalogEntry(client, new CatalogIndexEntry(uri, "nuget:PackageDelete", null, DateTime.MinValue, null, null));
// Assert
Assert.False(entry.Exists);
Assert.Null(entry.Created);
Assert.Null(entry.LastEdited);
Assert.Equal(expectedTimestamp.Ticks, entry.Deleted.Value.Ticks);
Assert.Equal(expectedTimestamp.Ticks, entry.Last.Value.Ticks);
}
public async Task RunAsync_WithValidPackage_CreatesFlatContainer()
{
var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json");
var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg");
var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec");
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.0.zip");
var expectedNupkg = GetStreamBytes(nupkgStream);
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/listedpackage.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(nupkgStream)
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
await _target.RunAsync(front, back, CancellationToken.None);
Assert.Equal(4, _catalogToDnxStorage.Content.Count);
Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nupkgUri, out var nupkg));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nuspecUri, out var nuspec));
Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson));
Assert.Equal(expectedNupkg, nupkg);
Assert.Equal(
"<?xml version=\"1.0\" encoding=\"utf-8\"?>\r\n<package xmlns=\"http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd\">\r\n <metadata>\r\n <id>ListedPackage</id>\r\n <version>1.0.0</version>\r\n <authors>NuGet</authors>\r\n <requireLicenseAcceptance>false</requireLicenseAcceptance>\r\n <description>Package description.</description>\r\n </metadata>\r\n</package>",
Encoding.UTF8.GetString(nuspec));
}
public async Task CreatesRegistrationsAndRespectsDeletes()
{
// Arrange
SharedInit(useLegacy: true, useSemVer2: false);
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
await _mockServer.AddStorageAsync(catalogStorage);
ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
// Act
await _target.RunAsync(front, back, CancellationToken.None);
// Assert
Assert.Equal(6, _legacyStorage.Content.Count);
// Ensure storage has cursor.json
var cursorJson = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json"));
Assert.NotNull(cursorJson.Key);
// Check package entries - ListedPackage
var package1Index = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/index.json"));
Assert.NotNull(package1Index.Key);
Assert.Contains("\"listed\":true,", package1Index.Value.GetContentString());
Assert.Contains("\"catalog:CatalogRoot\"", package1Index.Value.GetContentString());
Assert.Contains("\"PackageRegistration\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.0.nupkg\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json\"", package1Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.1.nupkg\"", package1Index.Value.GetContentString());
Assert.Contains("\"lower\":\"1.0.0\",", package1Index.Value.GetContentString());
Assert.Contains("\"upper\":\"1.0.1\"", package1Index.Value.GetContentString());
var package1 = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.0.json"));
Assert.Contains("\"listed\":true,", package1.Value.GetContentString());
Assert.NotNull(package1.Key);
var package2 = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1.json"));
Assert.Contains("\"listed\":true,", package2.Value.GetContentString());
Assert.NotNull(package2.Key);
// Check package entries - UnlistedPackage
var package2Index = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/index.json"));
Assert.NotNull(package2Index.Key);
Assert.Contains("\"listed\":false,", package2Index.Value.GetContentString());
Assert.Contains("\"catalog:CatalogRoot\"", package2Index.Value.GetContentString());
Assert.Contains("\"PackageRegistration\"", package2Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json\"", package2Index.Value.GetContentString());
Assert.Contains("\"http://tempuri.org/packages/unlistedpackage.1.0.0.nupkg\"", package2Index.Value.GetContentString());
Assert.Contains("\"lower\":\"1.0.0\",", package2Index.Value.GetContentString());
Assert.Contains("\"upper\":\"1.0.0\"", package2Index.Value.GetContentString());
var package3 = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0.json"));
Assert.Contains("\"listed\":false,", package3.Value.GetContentString());
Assert.NotNull(package3.Key);
// Ensure storage does not have the deleted "OtherPackage"
var otherPackageIndex = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/index.json"));
Assert.Null(otherPackageIndex.Key);
}