/// <remarks> /// Any values greater than <see cref="Constants.MaxPageSize"/> will be ignored by <see cref="IGalleryDatabaseQueryService"/>. /// </remarks> protected override async Task <IReadOnlyCollection <FeedPackageDetails> > GetPackagesToCheckAsync(DateTime since, DateTime max, int top, CancellationToken cancellationToken) { return((await _galleryDatabaseQueryService.GetPackagesEditedSince(since, top)) .SelectMany(p => p.Value) .Where(p => p.LastEditedDate < max) .ToList()); }
protected override async Task RunInternalAsync(CancellationToken cancellationToken) { using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", Destination.AbsoluteUri)) using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds)) using (var client = CreateHttpClient()) { uint packagesDeleted; uint packagesCreated; uint packagesEdited; client.Timeout = Timeout; var packageCatalogItemCreator = PackageCatalogItemCreator.Create( client, TelemetryService, Logger, PreferredPackageSourceStorage); do { packagesDeleted = 0; packagesCreated = 0; packagesEdited = 0; // baseline timestamps var catalogProperties = await CatalogProperties.ReadAsync(CatalogStorage, TelemetryService, cancellationToken); var lastCreated = catalogProperties.LastCreated ?? (StartDate ?? Constants.DateTimeMinValueUtc); var lastEdited = catalogProperties.LastEdited ?? lastCreated; var lastDeleted = catalogProperties.LastDeleted ?? lastCreated; if (lastDeleted == Constants.DateTimeMinValueUtc) { lastDeleted = SkipCreatedPackagesProcessing ? lastEdited : lastCreated; } try { if (lastDeleted > Constants.DateTimeMinValueUtc) { using (TelemetryService.TrackDuration(TelemetryConstants.DeletedPackagesSeconds)) { Logger.LogInformation("CATALOG LastDeleted: {CatalogDeletedTime}", lastDeleted.ToString("O")); var deletedPackages = await GetDeletedPackages(AuditingStorage, lastDeleted); packagesDeleted = (uint)deletedPackages.SelectMany(x => x.Value).Count(); Logger.LogInformation("FEED DeletedPackages: {DeletedPackagesCount}", packagesDeleted); // We want to ensure a commit only contains each package once at most. // Therefore we segment by package id + version. var deletedPackagesSegments = SegmentPackageDeletes(deletedPackages); foreach (var deletedPackagesSegment in deletedPackagesSegments) { lastDeleted = await Deletes2Catalog( deletedPackagesSegment, CatalogStorage, lastCreated, lastEdited, lastDeleted, cancellationToken); // Wait for one second to ensure the next catalog commit gets a new timestamp Thread.Sleep(TimeSpan.FromSeconds(1)); } } } if (!SkipCreatedPackagesProcessing) { using (TelemetryService.TrackDuration(TelemetryConstants.CreatedPackagesSeconds)) { Logger.LogInformation("CATALOG LastCreated: {CatalogLastCreatedTime}", lastCreated.ToString("O")); var createdPackages = await GalleryDatabaseQueryService.GetPackagesCreatedSince(lastCreated, Top); packagesCreated = (uint)createdPackages.SelectMany(x => x.Value).Count(); Logger.LogInformation("DATABASE CreatedPackages: {CreatedPackagesCount}", packagesCreated); lastCreated = await CatalogWriterHelper.WritePackageDetailsToCatalogAsync( packageCatalogItemCreator, createdPackages, CatalogStorage, lastCreated, lastEdited, lastDeleted, MaxDegreeOfParallelism, createdPackages : true, updateCreatedFromEdited : false, cancellationToken : cancellationToken, telemetryService : TelemetryService, logger : Logger); } } using (TelemetryService.TrackDuration(TelemetryConstants.EditedPackagesSeconds)) { Logger.LogInformation("CATALOG LastEdited: {CatalogLastEditedTime}", lastEdited.ToString("O")); var editedPackages = await GalleryDatabaseQueryService.GetPackagesEditedSince(lastEdited, Top); packagesEdited = (uint)editedPackages.SelectMany(x => x.Value).Count(); Logger.LogInformation("DATABASE EditedPackages: {EditedPackagesCount}", packagesEdited); lastEdited = await CatalogWriterHelper.WritePackageDetailsToCatalogAsync( packageCatalogItemCreator, editedPackages, CatalogStorage, lastCreated, lastEdited, lastDeleted, MaxDegreeOfParallelism, createdPackages : false, updateCreatedFromEdited : SkipCreatedPackagesProcessing, cancellationToken : cancellationToken, telemetryService : TelemetryService, logger : Logger); } } finally { TelemetryService.TrackMetric(TelemetryConstants.DeletedPackagesCount, packagesDeleted); if (!SkipCreatedPackagesProcessing) { TelemetryService.TrackMetric(TelemetryConstants.CreatedPackagesCount, packagesCreated); } TelemetryService.TrackMetric(TelemetryConstants.EditedPackagesCount, packagesEdited); } } while (packagesDeleted > 0 || packagesCreated > 0 || packagesEdited > 0); } }