public async Task WhenCancellationTokenIsCancelled_Throws() { await Assert.ThrowsAsync <OperationCanceledException>( () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync( Mock.Of <IPackageCatalogItemCreator>(), new SortedList <DateTime, IList <FeedPackageDetails> >(), Mock.Of <IStorage>(), DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, maxDegreeOfParallelism: 1, createdPackages: null, updateCreatedFromEdited: false, cancellationToken: new CancellationToken(canceled: true), telemetryService: Mock.Of <ITelemetryService>(), logger: Mock.Of <ILogger>())); }
internal Task <DateTime> WritePackageDetailsToCatalogAsync() { const int maxDegreeOfParallelism = 1; return(CatalogWriterHelper.WritePackageDetailsToCatalogAsync( PackageCatalogItemCreator.Object, Packages, Storage.Object, LastCreated, LastEdited, LastDeleted, maxDegreeOfParallelism, CreatedPackages, UpdateCreatedFromEdited, CancellationToken.None, TelemetryService.Object, Logger.Object)); }
public async Task WhenLoggerIsNull_Throws() { var exception = await Assert.ThrowsAsync <ArgumentNullException>( () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync( Mock.Of <IPackageCatalogItemCreator>(), new SortedList <DateTime, IList <FeedPackageDetails> >(), Mock.Of <IStorage>(), DateTime.MinValue, DateTime.MinValue, DateTime.MinValue, maxDegreeOfParallelism: 1, createdPackages: null, updateCreatedFromEdited: false, cancellationToken: CancellationToken.None, telemetryService: Mock.Of <ITelemetryService>(), logger: null)); Assert.Equal("logger", exception.ParamName); }
public async Task WhenMaxDegreeOfParallelismIsOutOfRange_Throws(int maxDegreeOfParallelism) { var exception = await Assert.ThrowsAsync <ArgumentOutOfRangeException>( () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync( Mock.Of <IPackageCatalogItemCreator>(), new SortedList <DateTime, IList <FeedPackageDetails> >(), Mock.Of <IStorage>(), DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, maxDegreeOfParallelism, createdPackages: false, updateCreatedFromEdited: false, cancellationToken: CancellationToken.None, telemetryService: Mock.Of <ITelemetryService>(), logger: Mock.Of <ILogger>())); Assert.Equal("maxDegreeOfParallelism", exception.ParamName); Assert.StartsWith($"The argument must be within the range from 1 (inclusive) to {int.MaxValue} (inclusive).", exception.Message); }
protected override async Task RunInternalAsync(CancellationToken cancellationToken) { using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", Destination.AbsoluteUri)) using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds)) using (var client = CreateHttpClient()) { uint packagesDeleted; uint packagesCreated; uint packagesEdited; client.Timeout = Timeout; var packageCatalogItemCreator = PackageCatalogItemCreator.Create( client, TelemetryService, Logger, PreferredPackageSourceStorage); do { packagesDeleted = 0; packagesCreated = 0; packagesEdited = 0; // baseline timestamps var catalogProperties = await CatalogProperties.ReadAsync(CatalogStorage, TelemetryService, cancellationToken); var lastCreated = catalogProperties.LastCreated ?? (StartDate ?? Constants.DateTimeMinValueUtc); var lastEdited = catalogProperties.LastEdited ?? lastCreated; var lastDeleted = catalogProperties.LastDeleted ?? lastCreated; if (lastDeleted == Constants.DateTimeMinValueUtc) { lastDeleted = SkipCreatedPackagesProcessing ? lastEdited : lastCreated; } try { if (lastDeleted > Constants.DateTimeMinValueUtc) { using (TelemetryService.TrackDuration(TelemetryConstants.DeletedPackagesSeconds)) { Logger.LogInformation("CATALOG LastDeleted: {CatalogDeletedTime}", lastDeleted.ToString("O")); var deletedPackages = await GetDeletedPackages(AuditingStorage, lastDeleted); packagesDeleted = (uint)deletedPackages.SelectMany(x => x.Value).Count(); Logger.LogInformation("FEED DeletedPackages: {DeletedPackagesCount}", packagesDeleted); // We want to ensure a commit only contains each package once at most. // Therefore we segment by package id + version. var deletedPackagesSegments = SegmentPackageDeletes(deletedPackages); foreach (var deletedPackagesSegment in deletedPackagesSegments) { lastDeleted = await Deletes2Catalog( deletedPackagesSegment, CatalogStorage, lastCreated, lastEdited, lastDeleted, cancellationToken); // Wait for one second to ensure the next catalog commit gets a new timestamp Thread.Sleep(TimeSpan.FromSeconds(1)); } } } if (!SkipCreatedPackagesProcessing) { using (TelemetryService.TrackDuration(TelemetryConstants.CreatedPackagesSeconds)) { Logger.LogInformation("CATALOG LastCreated: {CatalogLastCreatedTime}", lastCreated.ToString("O")); var createdPackages = await GalleryDatabaseQueryService.GetPackagesCreatedSince(lastCreated, Top); packagesCreated = (uint)createdPackages.SelectMany(x => x.Value).Count(); Logger.LogInformation("DATABASE CreatedPackages: {CreatedPackagesCount}", packagesCreated); lastCreated = await CatalogWriterHelper.WritePackageDetailsToCatalogAsync( packageCatalogItemCreator, createdPackages, CatalogStorage, lastCreated, lastEdited, lastDeleted, MaxDegreeOfParallelism, createdPackages : true, updateCreatedFromEdited : false, cancellationToken : cancellationToken, telemetryService : TelemetryService, logger : Logger); } } using (TelemetryService.TrackDuration(TelemetryConstants.EditedPackagesSeconds)) { Logger.LogInformation("CATALOG LastEdited: {CatalogLastEditedTime}", lastEdited.ToString("O")); var editedPackages = await GalleryDatabaseQueryService.GetPackagesEditedSince(lastEdited, Top); packagesEdited = (uint)editedPackages.SelectMany(x => x.Value).Count(); Logger.LogInformation("DATABASE EditedPackages: {EditedPackagesCount}", packagesEdited); lastEdited = await CatalogWriterHelper.WritePackageDetailsToCatalogAsync( packageCatalogItemCreator, editedPackages, CatalogStorage, lastCreated, lastEdited, lastDeleted, MaxDegreeOfParallelism, createdPackages : false, updateCreatedFromEdited : SkipCreatedPackagesProcessing, cancellationToken : cancellationToken, telemetryService : TelemetryService, logger : Logger); } } finally { TelemetryService.TrackMetric(TelemetryConstants.DeletedPackagesCount, packagesDeleted); if (!SkipCreatedPackagesProcessing) { TelemetryService.TrackMetric(TelemetryConstants.CreatedPackagesCount, packagesCreated); } TelemetryService.TrackMetric(TelemetryConstants.EditedPackagesCount, packagesEdited); } } while (packagesDeleted > 0 || packagesCreated > 0 || packagesEdited > 0); } }