Ejemplo n.º 1
0
 public async Task WhenCancellationTokenIsCancelled_Throws()
 {
     await Assert.ThrowsAsync <OperationCanceledException>(
         () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync(
             Mock.Of <IPackageCatalogItemCreator>(),
             new SortedList <DateTime, IList <FeedPackageDetails> >(),
             Mock.Of <IStorage>(),
             DateTime.MinValue,
             DateTime.MinValue,
             DateTime.MinValue,
             maxDegreeOfParallelism: 1,
             createdPackages: null,
             updateCreatedFromEdited: false,
             cancellationToken: new CancellationToken(canceled: true),
             telemetryService: Mock.Of <ITelemetryService>(),
             logger: Mock.Of <ILogger>()));
 }
Ejemplo n.º 2
0
            internal Task <DateTime> WritePackageDetailsToCatalogAsync()
            {
                const int maxDegreeOfParallelism = 1;

                return(CatalogWriterHelper.WritePackageDetailsToCatalogAsync(
                           PackageCatalogItemCreator.Object,
                           Packages,
                           Storage.Object,
                           LastCreated,
                           LastEdited,
                           LastDeleted,
                           maxDegreeOfParallelism,
                           CreatedPackages,
                           UpdateCreatedFromEdited,
                           CancellationToken.None,
                           TelemetryService.Object,
                           Logger.Object));
            }
Ejemplo n.º 3
0
            public async Task WhenLoggerIsNull_Throws()
            {
                var exception = await Assert.ThrowsAsync <ArgumentNullException>(
                    () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync(
                        Mock.Of <IPackageCatalogItemCreator>(),
                        new SortedList <DateTime, IList <FeedPackageDetails> >(),
                        Mock.Of <IStorage>(),
                        DateTime.MinValue,
                        DateTime.MinValue,
                        DateTime.MinValue,
                        maxDegreeOfParallelism: 1,
                        createdPackages: null,
                        updateCreatedFromEdited: false,
                        cancellationToken: CancellationToken.None,
                        telemetryService: Mock.Of <ITelemetryService>(),
                        logger: null));

                Assert.Equal("logger", exception.ParamName);
            }
Ejemplo n.º 4
0
            public async Task WhenMaxDegreeOfParallelismIsOutOfRange_Throws(int maxDegreeOfParallelism)
            {
                var exception = await Assert.ThrowsAsync <ArgumentOutOfRangeException>(
                    () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync(
                        Mock.Of <IPackageCatalogItemCreator>(),
                        new SortedList <DateTime, IList <FeedPackageDetails> >(),
                        Mock.Of <IStorage>(),
                        DateTime.UtcNow,
                        DateTime.UtcNow,
                        DateTime.UtcNow,
                        maxDegreeOfParallelism,
                        createdPackages: false,
                        updateCreatedFromEdited: false,
                        cancellationToken: CancellationToken.None,
                        telemetryService: Mock.Of <ITelemetryService>(),
                        logger: Mock.Of <ILogger>()));

                Assert.Equal("maxDegreeOfParallelism", exception.ParamName);
                Assert.StartsWith($"The argument must be within the range from 1 (inclusive) to {int.MaxValue} (inclusive).", exception.Message);
            }
Ejemplo n.º 5
0
        protected override async Task RunInternalAsync(CancellationToken cancellationToken)
        {
            using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", Destination.AbsoluteUri))
                using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds))
                    using (var client = CreateHttpClient())
                    {
                        uint packagesDeleted;
                        uint packagesCreated;
                        uint packagesEdited;

                        client.Timeout = Timeout;

                        var packageCatalogItemCreator = PackageCatalogItemCreator.Create(
                            client,
                            TelemetryService,
                            Logger,
                            PreferredPackageSourceStorage);

                        do
                        {
                            packagesDeleted = 0;
                            packagesCreated = 0;
                            packagesEdited  = 0;

                            // baseline timestamps
                            var catalogProperties = await CatalogProperties.ReadAsync(CatalogStorage, TelemetryService, cancellationToken);

                            var lastCreated = catalogProperties.LastCreated ?? (StartDate ?? Constants.DateTimeMinValueUtc);
                            var lastEdited  = catalogProperties.LastEdited ?? lastCreated;
                            var lastDeleted = catalogProperties.LastDeleted ?? lastCreated;

                            if (lastDeleted == Constants.DateTimeMinValueUtc)
                            {
                                lastDeleted = SkipCreatedPackagesProcessing ? lastEdited : lastCreated;
                            }

                            try
                            {
                                if (lastDeleted > Constants.DateTimeMinValueUtc)
                                {
                                    using (TelemetryService.TrackDuration(TelemetryConstants.DeletedPackagesSeconds))
                                    {
                                        Logger.LogInformation("CATALOG LastDeleted: {CatalogDeletedTime}", lastDeleted.ToString("O"));

                                        var deletedPackages = await GetDeletedPackages(AuditingStorage, lastDeleted);

                                        packagesDeleted = (uint)deletedPackages.SelectMany(x => x.Value).Count();
                                        Logger.LogInformation("FEED DeletedPackages: {DeletedPackagesCount}", packagesDeleted);

                                        // We want to ensure a commit only contains each package once at most.
                                        // Therefore we segment by package id + version.
                                        var deletedPackagesSegments = SegmentPackageDeletes(deletedPackages);
                                        foreach (var deletedPackagesSegment in deletedPackagesSegments)
                                        {
                                            lastDeleted = await Deletes2Catalog(
                                                deletedPackagesSegment, CatalogStorage, lastCreated, lastEdited, lastDeleted, cancellationToken);

                                            // Wait for one second to ensure the next catalog commit gets a new timestamp
                                            Thread.Sleep(TimeSpan.FromSeconds(1));
                                        }
                                    }
                                }

                                if (!SkipCreatedPackagesProcessing)
                                {
                                    using (TelemetryService.TrackDuration(TelemetryConstants.CreatedPackagesSeconds))
                                    {
                                        Logger.LogInformation("CATALOG LastCreated: {CatalogLastCreatedTime}", lastCreated.ToString("O"));

                                        var createdPackages = await GalleryDatabaseQueryService.GetPackagesCreatedSince(lastCreated, Top);

                                        packagesCreated = (uint)createdPackages.SelectMany(x => x.Value).Count();
                                        Logger.LogInformation("DATABASE CreatedPackages: {CreatedPackagesCount}", packagesCreated);

                                        lastCreated = await CatalogWriterHelper.WritePackageDetailsToCatalogAsync(
                                            packageCatalogItemCreator,
                                            createdPackages,
                                            CatalogStorage,
                                            lastCreated,
                                            lastEdited,
                                            lastDeleted,
                                            MaxDegreeOfParallelism,
                                            createdPackages : true,
                                            updateCreatedFromEdited : false,
                                            cancellationToken : cancellationToken,
                                            telemetryService : TelemetryService,
                                            logger : Logger);
                                    }
                                }

                                using (TelemetryService.TrackDuration(TelemetryConstants.EditedPackagesSeconds))
                                {
                                    Logger.LogInformation("CATALOG LastEdited: {CatalogLastEditedTime}", lastEdited.ToString("O"));

                                    var editedPackages = await GalleryDatabaseQueryService.GetPackagesEditedSince(lastEdited, Top);

                                    packagesEdited = (uint)editedPackages.SelectMany(x => x.Value).Count();
                                    Logger.LogInformation("DATABASE EditedPackages: {EditedPackagesCount}", packagesEdited);

                                    lastEdited = await CatalogWriterHelper.WritePackageDetailsToCatalogAsync(
                                        packageCatalogItemCreator,
                                        editedPackages,
                                        CatalogStorage,
                                        lastCreated,
                                        lastEdited,
                                        lastDeleted,
                                        MaxDegreeOfParallelism,
                                        createdPackages : false,
                                        updateCreatedFromEdited : SkipCreatedPackagesProcessing,
                                        cancellationToken : cancellationToken,
                                        telemetryService : TelemetryService,
                                        logger : Logger);
                                }
                            }
                            finally
                            {
                                TelemetryService.TrackMetric(TelemetryConstants.DeletedPackagesCount, packagesDeleted);

                                if (!SkipCreatedPackagesProcessing)
                                {
                                    TelemetryService.TrackMetric(TelemetryConstants.CreatedPackagesCount, packagesCreated);
                                }

                                TelemetryService.TrackMetric(TelemetryConstants.EditedPackagesCount, packagesEdited);
                            }
                        } while (packagesDeleted > 0 || packagesCreated > 0 || packagesEdited > 0);
                    }
        }