private void SharedInit(bool useLegacy, bool useSemVer2, Uri baseUri = null, Uri indexUri = null, Uri contentBaseUri = null, Uri galleryBaseUri = null)
{
if (useLegacy)
{
_legacyStorage = new MemoryStorage(baseUri ?? new Uri("http://tempuri.org"));
_legacyStorageFactory = new TestStorageFactory(name => _legacyStorage.WithName(name));
}
if (useSemVer2)
{
_semVer2Storage = new MemoryStorage(baseUri ?? new Uri("http://tempuri.org"));
_semVer2StorageFactory = new TestStorageFactory(name => _semVer2Storage.WithName(name));
}
_mockServer = new MockServerHttpClientHandler();
_mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
_target = new RegistrationCollector(
indexUri ?? new Uri("http://tempuri.org/index.json"),
_legacyStorageFactory,
_semVer2StorageFactory,
contentBaseUri ?? new Uri("http://tempuri.org/packages"),
galleryBaseUri ?? new Uri("http://tempuri.org/gallery"),
Mock.Of <ITelemetryService>(),
Mock.Of <ILogger>(),
handlerFunc: () => _mockServer,
httpRetryStrategy: new NoRetryStrategy());
RegistrationMakerCatalogItem.PackagePathProvider = new PackagesFolderPackagePathProvider();
}
public async Task RunInternal_CallsCatalogStorageLoadStringExactlyOnce()
{
var messageHandler = new MockServerHttpClientHandler();
var catalogStorage = new Mock <IStorage>(MockBehavior.Strict);
var datetime = DateTime.MinValue.ToString("O") + "Z";
var json = $"{{\"nuget:lastCreated\":\"{datetime}\"," +
$"\"nuget:lastDeleted\":\"{datetime}\"," +
$"\"nuget:lastEdited\":\"{datetime}\"}}";
var packageId = "a";
var packageVersion = "1.0.0";
catalogStorage.Setup(x => x.ResolveUri(It.IsNotNull <string>()))
.Returns(new Uri(_feedBaseUri));
catalogStorage.Setup(x => x.LoadStringAsync(It.IsNotNull <Uri>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(json);
messageHandler.SetAction("/", GetRootActionAsync);
messageHandler.SetAction(GetPath(packageId, packageVersion), GetEmptyPackages);
var job = new TestPackage2CatalogJob(
messageHandler,
catalogStorage.Object,
_feedBaseUri,
packageId,
packageVersion,
verbose: true);
await job.RunOnceAsync(CancellationToken.None);
catalogStorage.Verify(x => x.ResolveUri(It.IsNotNull <string>()), Times.AtLeastOnce());
catalogStorage.Verify(x => x.LoadStringAsync(It.IsNotNull <Uri>(), It.IsAny <CancellationToken>()), Times.Once());
}
public async Task ValidateAsync_WhenRequirePackageSignatureIsTrue_RequiresPackageSignatureFile(
bool includePackageSignatureFile)
{
var storage = new MemoryStorage(_baseUri);
var now = GetDateTime();
var storageContent = CreateStorageContent(now, includePackageSignatureFile);
storage.Content.TryAdd(_packageDetailsUri, storageContent);
using (var clientHandler = new MockServerHttpClientHandler())
using (CollectorHttpClient client = await CreateCollectorHttpClientStubAsync(clientHandler, storage))
{
ValidationContext context = CreateContext(client, now);
CatalogAggregateValidator validator = CreateValidator(context, requirePackageSignature: true);
AggregateValidationResult result = await validator.ValidateAsync(context);
Assert.Single(result.ValidationResults);
var expectedResult = includePackageSignatureFile ? TestResult.Pass : TestResult.Fail;
var actualResult = result.ValidationResults.Single();
Assert.Equal(expectedResult, actualResult.Result);
if (includePackageSignatureFile)
{
Assert.Null(actualResult.Exception);
}
else
{
Assert.IsType <MissingPackageSignatureFileException>(actualResult.Exception);
}
}
}
public CollectorHttpClientTests()
{
_telemetryService = new Mock <ITelemetryService>();
_handler = new MockServerHttpClientHandler();
_target = new CollectorHttpClient(_handler);
}
public ValidatePackageHashHandlerFacts()
{
var messageHandler = new MockServerHttpClientHandler();
messageHandler.SetAction(
"/packages/testunsigned.1.0.0.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\TestUnsigned.1.0.0.nupkg"))
}
));
_packageEntry = new CatalogIndexEntry(
new Uri("http://localhost/catalog/entry.json"),
"nuget:PackageDetails",
"123",
DateTime.UtcNow,
PackageIdentity);
_blob = new Mock <ICloudBlockBlob>();
_blob.Setup(b => b.Uri).Returns(new Uri("http://localhost/packages/testunsigned.1.0.0.nupkg"));
_telemetryService = new Mock <ITelemetryService>();
_target = new ValidatePackageHashHandler(
new HttpClient(messageHandler),
_telemetryService.Object,
Mock.Of <ILogger <ValidatePackageHashHandler> >());
}
private async Task <CollectorHttpClient> CreateDummyClient(MemoryStorage catalogStorage)
{
var mockServer = new MockServerHttpClientHandler();
mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
await mockServer.AddStorageAsync(catalogStorage);
return(new CollectorHttpClient(mockServer));
}
private void Initialize()
{
_mockServer = new MockServerHttpClientHandler();
_mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
_collector = new TestSortingGraphCollector(
new Uri("http://tempuri.org/index.json"),
new Uri[] { Schema.DataTypes.PackageDetails, Schema.DataTypes.PackageDelete },
handlerFunc: () => _mockServer);
}
public FactsBase()
{
var feedToSource = new Mock <IDictionary <FeedType, SourceRepository> >();
var logger = Mock.Of <ILogger <PackageHasSignatureValidator> >();
feedToSource.Setup(x => x[It.IsAny <FeedType>()]).Returns(new Mock <SourceRepository>().Object);
_mockServer = new MockServerHttpClientHandler();
_target = new PackageHasSignatureValidator(feedToSource.Object, logger);
}
private static async Task <CollectorHttpClient> CreateCollectorHttpClientStubAsync(
MockServerHttpClientHandler clientHandler,
MemoryStorage catalogStorage)
{
clientHandler.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
await clientHandler.AddStorageAsync(catalogStorage);
return(new CollectorHttpClient(clientHandler));
}
public async Task ThrowsIfCommitTimesOut()
{
// Arrange
var storage = new MemoryStorage();
var storageFactory = new TestStorageFactory(name => storage.WithName(name));
MockServerHttpClientHandler mockServer;
mockServer = new MockServerHttpClientHandler();
mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
var catalogStorage = Catalogs.CreateTestCatalogWithOnePackage();
await mockServer.AddStorageAsync(catalogStorage);
ReadWriteCursor front = new DurableCursor(
storage.ResolveUri("cursor.json"),
storage,
MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
var commitTimeout = TimeSpan.FromSeconds(1);
var stuckDuration = TimeSpan.FromMinutes(1);
var telemetryService = new Mock <ITelemetryService>();
var indexCommitDurationMetric = new Mock <IDisposable>();
telemetryService.Setup(t => t.TrackIndexCommitDuration()).Returns(indexCommitDurationMetric.Object);
using (var testDirectory = TestDirectory.Create())
{
var luceneDirectory = new SimpleFSDirectory(new DirectoryInfo(testDirectory));
using (var indexWriter = Catalog2LuceneJob.CreateIndexWriter(luceneDirectory))
using (var stuckIndexWriter = StuckIndexWriter.FromIndexWriter(indexWriter, stuckDuration))
{
var target = new SearchIndexFromCatalogCollector(
new Uri("http://tempuri.org/index.json"),
stuckIndexWriter,
commitEachBatch: true,
commitTimeout: commitTimeout,
baseAddress: null,
telemetryService: telemetryService.Object,
logger: new TestLogger(),
handlerFunc: () => mockServer,
httpRetryStrategy: new NoRetryStrategy());
// Act & Assert
await Assert.ThrowsAsync <OperationCanceledException>(() => target.RunAsync(front, back, CancellationToken.None));
telemetryService.Verify(t => t.TrackIndexCommitDuration(), Times.Once);
telemetryService.Verify(t => t.TrackIndexCommitTimeout(), Times.Once);
indexCommitDurationMetric.Verify(m => m.Dispose(), Times.Never);
}
}
}
internal static void AddCatalogLeafToMockServer(MockServerHttpClientHandler clientHandler, Uri uri, CatalogLeaf leaf)
{
string relativeUrl = uri.IsAbsoluteUri ? uri.AbsolutePath : uri.ToString();
clientHandler.SetAction(relativeUrl, request =>
{
return(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(JsonConvert.SerializeObject(leaf))
}));
});
}
public async Task RunAsync_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPackage()
{
_catalogToDnxStorage = new SynchronizedMemoryStorage(new[]
{
new Uri("http://tempuri.org/packages/listedpackage.1.0.1.nupkg"),
});
_catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name));
_mockServer = new MockServerHttpClientHandler();
_mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json");
var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg");
var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec");
_target = new DnxCatalogCollector(
new Uri("http://tempuri.org/index.json"),
_catalogToDnxStorageFactory,
_nullPreferredPackageSourceStorage,
_contentBaseAddress,
Mock.Of <ITelemetryService>(),
new Mock <ILogger>().Object,
_maxDegreeOfParallelism,
() => _mockServer);
var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete();
await _mockServer.AddStorageAsync(catalogStorage);
_mockServer.SetAction(
"/packages/listedpackage.1.0.1.nupkg",
request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.1.zip"))
}));
var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
await _target.RunAsync(front, back, CancellationToken.None);
Assert.Equal(2, _catalogToDnxStorage.Content.Count);
Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri));
Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri));
Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri));
Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri));
Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri));
Assert.Equal(GetExpectedIndexJsonContent("1.0.1"), Encoding.UTF8.GetString(indexJson));
}
internal static void AddPackageToMockServer(MockServerHttpClientHandler clientHandler, PackageIdentity packageIdentity, string filePath)
{
string packageId = packageIdentity.Id.ToLowerInvariant();
string packageVersion = packageIdentity.Version.ToNormalizedString().ToLowerInvariant();
clientHandler.SetAction($"/packages/{packageId}/{packageVersion}/{packageId}.{packageVersion}.nupkg", request =>
{
byte[] bytes = File.ReadAllBytes(filePath);
return(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new ByteArrayContent(bytes)
}));
});
}
public async Task Name_Always_ReturnsTypeName()
{
using (var clientHandler = new MockServerHttpClientHandler())
using (CollectorHttpClient client = await CreateCollectorHttpClientStubAsync(
clientHandler,
new MemoryStorage()))
{
ValidationContext context = CreateContext(client, DateTime.UtcNow);
CatalogAggregateValidator validator = CreateValidator(
context,
requirePackageSignature: false);
Assert.Equal(typeof(CatalogAggregateValidator).FullName, validator.Name);
}
}
private HttpMessageHandler GetMessageHandlerForPackages(IEnumerable <ODataPackage> oDataPackages)
{
var mockServer = new MockServerHttpClientHandler();
mockServer.SetAction("/test", (request) =>
{
return(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(
ODataFeedHelper.ToODataFeed(oDataPackages, new Uri(_baseUri), "Packages"))
}));
});
return(mockServer);
}
private void FailFirstRequest(MockServerHttpClientHandler mockServer, string relativeUri)
{
var originalAction = mockServer.Actions[relativeUri];
var hasFailed = false;
Func <HttpRequestMessage, Task <HttpResponseMessage> > failFirst = request =>
{
if (!hasFailed)
{
hasFailed = true;
throw new HttpRequestException("Simulated HTTP failure.");
}
return(originalAction(request));
};
mockServer.SetAction(relativeUri, failFirst);
}
public FactsBase()
{
var feedToSource = new Mock <IDictionary <FeedType, SourceRepository> >();
var timestampResource = new Mock <IPackageTimestampMetadataResource>();
var logger = Mock.Of <ILogger <PackageIsRepositorySignedValidator> >();
_source = new Mock <SourceRepository>();
_mockServer = new MockServerHttpClientHandler();
feedToSource.Setup(x => x[It.IsAny <FeedType>()]).Returns(_source.Object);
// Mock a catalog entry and leaf for the package we are validating.
_catalogEntries = new[]
{
new CatalogIndexEntry(
new Uri("https://localhost/catalog/leaf.json"),
string.Empty,
string.Empty,
DateTime.UtcNow,
PackageId,
PackageNuGetVersion)
};
AddCatalogLeafToMockServer("/catalog/leaf.json", new CatalogLeaf
{
Created = PackageCreationTime,
LastEdited = PackageCreationTime,
});
// Mock V2 feed response for the package's Created/LastEdited timestamps. These timestamps must match
// the mocked catalog entry's timestamps.
var timestamp = PackageTimestampMetadata.CreateForPackageExistingOnFeed(created: PackageCreationTime, lastEdited: PackageCreationTime);
timestampResource.Setup(t => t.GetAsync(It.IsAny <ValidationContext>())).ReturnsAsync(timestamp);
_source.Setup(s => s.GetResource <IPackageTimestampMetadataResource>()).Returns(timestampResource.Object);
// Add the package base address resource
var resource = new PackageBaseAddressResource("https://localhost/packages/");
_source.Setup(s => s.GetResource <PackageBaseAddressResource>()).Returns(resource);
_target = new PackageIsRepositorySignedValidator(feedToSource.Object, logger);
}
public async Task ValidateAsync_WhenRequirePackageSignatureIsFalse_DoesNotRequirePackageSignatureFile(
bool includePackageSignatureFile)
{
var storage = new MemoryStorage(_baseUri);
var now = GetDateTime();
var storageContent = CreateStorageContent(now, includePackageSignatureFile);
storage.Content.TryAdd(_packageDetailsUri, storageContent);
using (var clientHandler = new MockServerHttpClientHandler())
using (CollectorHttpClient client = await CreateCollectorHttpClientStubAsync(clientHandler, storage))
{
ValidationContext context = CreateContext(client, now);
CatalogAggregateValidator validator = CreateValidator(context, requirePackageSignature: false);
AggregateValidationResult result = await validator.ValidateAsync(context);
Assert.Empty(result.ValidationResults);
}
}
public FactsBase()
{
_mockServer = new MockServerHttpClientHandler();
// Mock a catalog entry and leaf for the package we are validating.
_catalogEntries = new[]
{
new CatalogIndexEntry(
new Uri("https://nuget.test/catalog/leaf.json"),
CatalogConstants.NuGetPackageDetails,
Guid.NewGuid().ToString(),
DateTime.UtcNow,
PackageIdentity)
};
ValidatorTestUtility.AddCatalogLeafToMockServer(_mockServer, new Uri("/catalog/leaf.json", UriKind.Relative), new CatalogLeaf
{
Created = PackageCreationTime,
LastEdited = PackageCreationTime
});
}
internal Test(bool useStorage = true)
{
Handler = new MockServerHttpClientHandler();
_httpClient = new HttpClient(Handler);
PackageFileName = $"{PackageId.ToLowerInvariant()}.{PackageVersion.ToLowerInvariant()}.nupkg";
Timestamp = DateTime.UtcNow;
ContentUri = new Uri($"https://nuget.test/packages/{PackageFileName}");
PackageUri = Utilities.GetNugetCacheBustingUri(ContentUri, Timestamp.ToString("O"));
Storage = new Mock <IAzureStorage>(MockBehavior.Strict);
Blob = new Mock <ICloudBlockBlob>(MockBehavior.Strict);
FeedPackageDetails = new FeedPackageDetails(
ContentUri,
Timestamp.AddHours(-3),
Timestamp.AddHours(-2),
Timestamp.AddHours(-1),
PackageId,
PackageVersion,
PackageVersion);
var stream = TestHelper.GetStream(PackageFileName + ".testdata");
Handler.SetAction(
PackageUri.PathAndQuery,
request => Task.FromResult(
new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(stream)
}));
TelemetryService = new MockTelemetryService();
Creator = PackageCatalogItemCreator.Create(
_httpClient,
TelemetryService,
Mock.Of <ILogger>(),
useStorage ? Storage.Object : null);
}
public DnxCatalogCollectorTests()
{
_catalogToDnxStorage = new MemoryStorage();
_catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name));
_mockServer = new MockServerHttpClientHandler();
_mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
var loggerFactory = new LoggerFactory();
_logger = loggerFactory.CreateLogger <DnxCatalogCollector>();
_target = new DnxCatalogCollector(
new Uri("http://tempuri.org/index.json"),
_catalogToDnxStorageFactory,
_nullPreferredPackageSourceStorage,
_contentBaseAddress,
Mock.Of <ITelemetryService>(),
_logger,
_maxDegreeOfParallelism,
() => _mockServer);
_cursorJsonUri = _catalogToDnxStorage.ResolveUri("cursor.json");
}
public async Task DoesNotSkipPackagesWhenExceptionOccurs(string catalogUri, string expectedCursorBeforeRetry)
{
// Arrange
var storage = new MemoryStorage();
var storageFactory = new TestStorageFactory(name => storage.WithName(name));
MockServerHttpClientHandler mockServer;
mockServer = new MockServerHttpClientHandler();
mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit();
await mockServer.AddStorageAsync(catalogStorage);
// Make the first request for a catalog leaf node fail. This will cause the registration collector
// to fail the first time but pass the second time.
FailFirstRequest(mockServer, catalogUri);
expectedCursorBeforeRetry = expectedCursorBeforeRetry ?? MemoryCursor.MinValue.ToString("O");
ReadWriteCursor front = new DurableCursor(
storage.ResolveUri("cursor.json"),
storage,
MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
using (var testDirectory = TestDirectory.Create())
{
var luceneDirectory = new SimpleFSDirectory(new DirectoryInfo(testDirectory));
using (var indexWriter = Catalog2LuceneJob.CreateIndexWriter(luceneDirectory))
{
var target = new SearchIndexFromCatalogCollector(
new Uri("http://tempuri.org/index.json"),
indexWriter,
commitEachBatch: true,
baseAddress: null,
telemetryService: new Mock <ITelemetryService>().Object,
logger: new TestLogger(),
handlerFunc: () => mockServer);
// Act
await Assert.ThrowsAsync <Exception>(() => target.RunAsync(front, back, CancellationToken.None));
var cursorBeforeRetry = front.Value;
await target.RunAsync(front, back, CancellationToken.None);
var cursorAfterRetry = front.Value;
// Assert
var reader = indexWriter.GetReader();
var documents = Enumerable
.Range(0, reader.NumDeletedDocs + reader.NumDocs())
.Where(i => !reader.IsDeleted(i))
.Select(i => reader.Document(i))
.ToList();
Assert.Equal(4, documents.Count);
var documentsByType = documents
.ToLookup(doc => doc
.fields_ForNUnit
.FirstOrDefault(f => f.Name == "@type")?
.StringValue);
var commitDocuments = documentsByType[Schema.DataTypes.CatalogInfastructure.AbsoluteUri.ToString()].ToList();
var packageDocuments = documentsByType[null].ToList();
Assert.Equal(1, commitDocuments.Count);
Assert.Equal(3, packageDocuments.Count);
Assert.Equal(
"UnlistedPackage",
packageDocuments[0].fields_ForNUnit.FirstOrDefault(x => x.Name == Constants.LucenePropertyId)?.StringValue);
Assert.Equal(
"ListedPackage",
packageDocuments[1].fields_ForNUnit.FirstOrDefault(x => x.Name == Constants.LucenePropertyId)?.StringValue);
Assert.Equal(
"AnotherPackage",
packageDocuments[2].fields_ForNUnit.FirstOrDefault(x => x.Name == Constants.LucenePropertyId)?.StringValue);
Assert.Equal(DateTime.Parse(expectedCursorBeforeRetry).ToUniversalTime(), cursorBeforeRetry);
Assert.Equal(DateTime.Parse("2015-10-12T10:08:55.3335317Z").ToUniversalTime(), cursorAfterRetry);
}
}
}
public FactsBase()
{
_logger = new Mock <ILogger <PackageHasSignatureValidator> >();
_mockServer = new MockServerHttpClientHandler();
}