public void Create_WhenStorageIsNull_ReturnsInstance() { var creator = PackageCatalogItemCreator.Create( Mock.Of <HttpClient>(), Mock.Of <ITelemetryService>(), Mock.Of <ILogger>(), storage: null); Assert.NotNull(creator); }
public void Create_WhenLoggerIsNull_Throws() { ILogger logger = null; var exception = Assert.Throws <ArgumentNullException>( () => PackageCatalogItemCreator.Create( Mock.Of <HttpClient>(), Mock.Of <ITelemetryService>(), logger, Mock.Of <IStorage>())); Assert.Equal("logger", exception.ParamName); }
public void Create_WhenHttpClientIsNull_Throws() { HttpClient httpClient = null; var exception = Assert.Throws <ArgumentNullException>( () => PackageCatalogItemCreator.Create( httpClient, Mock.Of <ITelemetryService>(), Mock.Of <ILogger>(), Mock.Of <IStorage>())); Assert.Equal("httpClient", exception.ParamName); }
public async Task CreateAsync_WhenPackageItemIsNull_Throws() { var creator = PackageCatalogItemCreator.Create( Mock.Of <HttpClient>(), _telemetryService, Mock.Of <ILogger>(), storage: null); FeedPackageDetails packageItem = null; var exception = await Assert.ThrowsAsync <ArgumentNullException>( () => creator.CreateAsync(packageItem, DateTime.UtcNow, CancellationToken.None)); Assert.Equal("packageItem", exception.ParamName); }
public async Task CreateAsync_WhenCancellationTokenIsCancelled_Throws() { var creator = PackageCatalogItemCreator.Create( Mock.Of <HttpClient>(), _telemetryService, Mock.Of <ILogger>(), storage: null); var packageItem = new FeedPackageDetails( new Uri("https://nuget.test"), DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, packageId: "a", packageVersion: "1.0.0"); await Assert.ThrowsAsync <OperationCanceledException>( () => creator.CreateAsync(packageItem, DateTime.UtcNow, new CancellationToken(canceled: true))); }
protected override async Task RunInternalAsync(CancellationToken cancellationToken) { var timeout = TimeSpan.FromSeconds(300); using (var client = CreateHttpClient()) { client.Timeout = timeout; // if the version is specified a single package is processed otherwise all the packages corresponding to that id are processed var uri = (_version == null) ? MakePackageUri(_gallery, _id) : MakePackageUri(_gallery, _id, _version); var packages = await FeedHelpers.GetPackagesInOrder(client, uri, package => package.CreatedDate); Logger.LogInformation($"Downloading {packages.Select(t => t.Value.Count).Sum()} packages"); // the idea here is to leave the lastCreated, lastEdited and lastDeleted values exactly as they were var catalogProperties = await CatalogProperties.ReadAsync(_storage, TelemetryService, cancellationToken); var lastCreated = catalogProperties.LastCreated ?? DateTime.MinValue.ToUniversalTime(); var lastEdited = catalogProperties.LastEdited ?? DateTime.MinValue.ToUniversalTime(); var lastDeleted = catalogProperties.LastDeleted ?? DateTime.MinValue.ToUniversalTime(); var packageCatalogItemCreator = PackageCatalogItemCreator.Create( client, TelemetryService, Logger, storage: null); await FeedHelpers.DownloadMetadata2CatalogAsync( packageCatalogItemCreator, packages, _storage, lastCreated, lastEdited, lastDeleted, MaxDegreeOfParallelism, createdPackages : null, updateCreatedFromEdited : false, cancellationToken : cancellationToken, telemetryService : TelemetryService, logger : Logger); } }
internal Test(bool useStorage = true) { Handler = new MockServerHttpClientHandler(); _httpClient = new HttpClient(Handler); PackageFileName = $"{PackageId.ToLowerInvariant()}.{PackageVersion.ToLowerInvariant()}.nupkg"; Timestamp = DateTime.UtcNow; ContentUri = new Uri($"https://nuget.test/packages/{PackageFileName}"); PackageUri = Utilities.GetNugetCacheBustingUri(ContentUri, Timestamp.ToString("O")); Storage = new Mock <IAzureStorage>(MockBehavior.Strict); Blob = new Mock <ICloudBlockBlob>(MockBehavior.Strict); FeedPackageDetails = new FeedPackageDetails( ContentUri, Timestamp.AddHours(-3), Timestamp.AddHours(-2), Timestamp.AddHours(-1), PackageId, PackageVersion, PackageVersion); var stream = TestHelper.GetStream(PackageFileName + ".testdata"); Handler.SetAction( PackageUri.PathAndQuery, request => Task.FromResult( new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(stream) })); TelemetryService = new MockTelemetryService(); Creator = PackageCatalogItemCreator.Create( _httpClient, TelemetryService, Mock.Of <ILogger>(), useStorage ? Storage.Object : null); }
protected override async Task RunInternalAsync(CancellationToken cancellationToken) { using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", Destination.AbsoluteUri)) using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds)) using (var client = CreateHttpClient()) { uint packagesDeleted; uint packagesCreated; uint packagesEdited; client.Timeout = Timeout; var packageCatalogItemCreator = PackageCatalogItemCreator.Create( client, TelemetryService, Logger, PreferredPackageSourceStorage); do { packagesDeleted = 0; packagesCreated = 0; packagesEdited = 0; // baseline timestamps var catalogProperties = await CatalogProperties.ReadAsync(CatalogStorage, TelemetryService, cancellationToken); var lastCreated = catalogProperties.LastCreated ?? (StartDate ?? Constants.DateTimeMinValueUtc); var lastEdited = catalogProperties.LastEdited ?? lastCreated; var lastDeleted = catalogProperties.LastDeleted ?? lastCreated; if (lastDeleted == Constants.DateTimeMinValueUtc) { lastDeleted = SkipCreatedPackagesProcessing ? lastEdited : lastCreated; } try { if (lastDeleted > Constants.DateTimeMinValueUtc) { using (TelemetryService.TrackDuration(TelemetryConstants.DeletedPackagesSeconds)) { Logger.LogInformation("CATALOG LastDeleted: {CatalogDeletedTime}", lastDeleted.ToString("O")); var deletedPackages = await GetDeletedPackages(AuditingStorage, lastDeleted); packagesDeleted = (uint)deletedPackages.SelectMany(x => x.Value).Count(); Logger.LogInformation("FEED DeletedPackages: {DeletedPackagesCount}", packagesDeleted); // We want to ensure a commit only contains each package once at most. // Therefore we segment by package id + version. var deletedPackagesSegments = SegmentPackageDeletes(deletedPackages); foreach (var deletedPackagesSegment in deletedPackagesSegments) { lastDeleted = await Deletes2Catalog( deletedPackagesSegment, CatalogStorage, lastCreated, lastEdited, lastDeleted, cancellationToken); // Wait for one second to ensure the next catalog commit gets a new timestamp Thread.Sleep(TimeSpan.FromSeconds(1)); } } } if (!SkipCreatedPackagesProcessing) { using (TelemetryService.TrackDuration(TelemetryConstants.CreatedPackagesSeconds)) { Logger.LogInformation("CATALOG LastCreated: {CatalogLastCreatedTime}", lastCreated.ToString("O")); var createdPackages = await GalleryDatabaseQueryService.GetPackagesCreatedSince(lastCreated, Top); packagesCreated = (uint)createdPackages.SelectMany(x => x.Value).Count(); Logger.LogInformation("DATABASE CreatedPackages: {CreatedPackagesCount}", packagesCreated); lastCreated = await CatalogWriterHelper.WritePackageDetailsToCatalogAsync( packageCatalogItemCreator, createdPackages, CatalogStorage, lastCreated, lastEdited, lastDeleted, MaxDegreeOfParallelism, createdPackages : true, updateCreatedFromEdited : false, cancellationToken : cancellationToken, telemetryService : TelemetryService, logger : Logger); } } using (TelemetryService.TrackDuration(TelemetryConstants.EditedPackagesSeconds)) { Logger.LogInformation("CATALOG LastEdited: {CatalogLastEditedTime}", lastEdited.ToString("O")); var editedPackages = await GalleryDatabaseQueryService.GetPackagesEditedSince(lastEdited, Top); packagesEdited = (uint)editedPackages.SelectMany(x => x.Value).Count(); Logger.LogInformation("DATABASE EditedPackages: {EditedPackagesCount}", packagesEdited); lastEdited = await CatalogWriterHelper.WritePackageDetailsToCatalogAsync( packageCatalogItemCreator, editedPackages, CatalogStorage, lastCreated, lastEdited, lastDeleted, MaxDegreeOfParallelism, createdPackages : false, updateCreatedFromEdited : SkipCreatedPackagesProcessing, cancellationToken : cancellationToken, telemetryService : TelemetryService, logger : Logger); } } finally { TelemetryService.TrackMetric(TelemetryConstants.DeletedPackagesCount, packagesDeleted); if (!SkipCreatedPackagesProcessing) { TelemetryService.TrackMetric(TelemetryConstants.CreatedPackagesCount, packagesCreated); } TelemetryService.TrackMetric(TelemetryConstants.EditedPackagesCount, packagesEdited); } } while (packagesDeleted > 0 || packagesCreated > 0 || packagesEdited > 0); } }
private async Task <Uri> ExecuteFeedToCatalogAsync(PerBatchContext context, IReadOnlyList <PerPackageContext> packageContexts) { var serviceProvider = GetServiceProvider( context, context.Global.CatalogContainerName, context.Worker.CatalogStoragePath); var now = DateTime.UtcNow; var offset = 0; var packages = new SortedList <DateTime, IList <FeedPackageDetails> >(); var maxDegreeOfParallelism = ServicePointManager.DefaultConnectionLimit; foreach (var packageContext in packageContexts) { // These timestamps don't matter too much since the order that items are processed within a catalog // commit is not defined. This is just a convenient way to get a bunch of unique timestamps to ease // debugging. var key = now.AddSeconds(offset--); var published = now.AddSeconds(offset--); var lastEdited = now.AddSeconds(offset--); var created = now.AddSeconds(offset--); packages.Add(key, new List <FeedPackageDetails> { new FeedPackageDetails( packageContext.PackageUri, created, lastEdited, published, packageContext.PackageId, packageContext.PackageVersion) }); } var storage = serviceProvider.GetRequiredService <IStorage>(); var createdPackages = true; var updateCreatedFromEdited = false; using (var httpClient = serviceProvider.GetRequiredService <HttpClient>()) { var telemetryService = serviceProvider.GetRequiredService <ITelemetryService>(); var logger = serviceProvider.GetRequiredService <ILogger>(); var packageCatalogItemCreator = PackageCatalogItemCreator.Create( httpClient, telemetryService, logger, storage: null); await FeedHelpers.DownloadMetadata2CatalogAsync( packageCatalogItemCreator, packages, storage, now, now, now, maxDegreeOfParallelism, createdPackages, updateCreatedFromEdited, CancellationToken.None, telemetryService, logger); } return(storage.ResolveUri("index.json")); }