예제 #1
0
            public async Task FlagDisablesPopularityTransfers()
            {
                var downloadChanges = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase)
                {
                    { "Package1", 5 }
                };

                DownloadSetComparer
                .Setup(c => c.Compare(It.IsAny <DownloadData>(), It.IsAny <DownloadData>()))
                .Returns <DownloadData, DownloadData>((oldData, newData) =>
                {
                    return(downloadChanges);
                });

                OldTransfers.AddTransfer("Package1", "Package2");
                NewTransfers.AddTransfer("Package1", "Package2");

                FeatureFlags
                .Setup(x => x.IsPopularityTransferEnabled())
                .Returns(false);

                await Target.ExecuteAsync();

                PopularityTransferDataClient
                .Verify(
                    c => c.ReadLatestIndexedAsync(It.IsAny <IAccessCondition>(), It.IsAny <StringCache>()),
                    Times.Once);
                DatabaseFetcher
                .Verify(
                    d => d.GetPopularityTransfersAsync(),
                    Times.Never);

                // The popularity transfers should not be given to the download transferrer.
                DownloadTransferrer
                .Verify(
                    x => x.UpdateDownloadTransfers(
                        NewDownloadData,
                        downloadChanges,
                        OldTransfers,
                        It.Is <PopularityTransferData>(d => d.Count == 0)),
                    Times.Once);

                // Popularity transfers auxiliary file should be empty.
                PopularityTransferDataClient.Verify(
                    c => c.ReplaceLatestIndexedAsync(
                        It.Is <PopularityTransferData>(d => d.Count == 0),
                        It.IsAny <IAccessCondition>()),
                    Times.Once);
            }
예제 #2
0
            public async Task PushesNothingWhenThereAreNoChanges()
            {
                await Target.ExecuteAsync();

                VerifyCompletedTelemetry(JobOutcome.NoOp);
                VerifyAllIdsAreProcessed(changeCount: 0);
                IndexActionBuilder.Verify(
                    x => x.UpdateAsync(
                        It.IsAny <string>(),
                        It.IsAny <Func <SearchFilters, KeyedDocument> >()),
                    Times.Never);
                BatchPusher.Verify(x => x.TryFinishAsync(), Times.Never);
                BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never);
                DownloadDataClient.Verify(
                    x => x.ReplaceLatestIndexedAsync(It.IsAny <DownloadData>(), It.IsAny <IAccessCondition>()),
                    Times.Never);
                PopularityTransferDataClient.Verify(
                    x => x.ReplaceLatestIndexedAsync(It.IsAny <PopularityTransferData>(), It.IsAny <IAccessCondition>()),
                    Times.Never);
            }
        public PopularityTransferIntegrationTests(ITestOutputHelper output)
        {
            _featureFlags = new Mock <IFeatureFlagService>();
            _telemetry    = new Mock <IAzureSearchTelemetryService>();

            _config = new Auxiliary2AzureSearchConfiguration
            {
                AuxiliaryDataStorageContainer = "auxiliary-container",
                EnablePopularityTransfers     = true,
                StorageContainer = "storage-container",
                Scoring          = new AzureSearchScoringConfiguration()
            };

            var options = new Mock <IOptionsSnapshot <Auxiliary2AzureSearchConfiguration> >();

            options
            .Setup(x => x.Value)
            .Returns(_config);

            _developmentConfig = new AzureSearchJobDevelopmentConfiguration();
            var developmentOptions = new Mock <IOptionsSnapshot <AzureSearchJobDevelopmentConfiguration> >();

            developmentOptions
            .Setup(x => x.Value)
            .Returns(_developmentConfig);

            var auxiliaryConfig = new AuxiliaryDataStorageConfiguration
            {
                AuxiliaryDataStorageContainer            = "auxiliary-container",
                AuxiliaryDataStorageDownloadsPath        = "downloads.json",
                AuxiliaryDataStorageExcludedPackagesPath = "excludedPackages.json",
            };

            var auxiliaryOptions = new Mock <IOptionsSnapshot <AuxiliaryDataStorageConfiguration> >();

            auxiliaryOptions
            .Setup(x => x.Value)
            .Returns(auxiliaryConfig);

            _auxilliaryContainer = new InMemoryCloudBlobContainer();
            _storageContainer    = new InMemoryCloudBlobContainer();

            _blobClient = new InMemoryCloudBlobClient();
            _blobClient.Containers["auxiliary-container"] = _auxilliaryContainer;
            _blobClient.Containers["storage-container"]   = _storageContainer;

            var auxiliaryFileClient = new AuxiliaryFileClient(
                _blobClient,
                auxiliaryOptions.Object,
                _telemetry.Object,
                output.GetLogger <AuxiliaryFileClient>());

            _newPopularityTransfers = new PopularityTransferData();
            var databaseFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>();

            databaseFetcher
            .Setup(x => x.GetPopularityTransfersAsync())
            .ReturnsAsync(_newPopularityTransfers);

            var downloadDataClient = new DownloadDataClient(
                _blobClient,
                options.Object,
                _telemetry.Object,
                output.GetLogger <DownloadDataClient>());

            var popularityTransferDataClient = new PopularityTransferDataClient(
                _blobClient,
                options.Object,
                _telemetry.Object,
                output.GetLogger <PopularityTransferDataClient>());

            var versionListDataClient = new VersionListDataClient(
                _blobClient,
                options.Object,
                output.GetLogger <VersionListDataClient>());

            var downloadComparer = new DownloadSetComparer(
                _telemetry.Object,
                options.Object,
                output.GetLogger <DownloadSetComparer>());

            var dataComparer = new DataSetComparer(
                _telemetry.Object,
                output.GetLogger <DataSetComparer>());

            var downloadTransferrer = new DownloadTransferrer(
                dataComparer,
                options.Object,
                output.GetLogger <DownloadTransferrer>());

            var baseDocumentBuilder      = new BaseDocumentBuilder(options.Object);
            var searchDocumentBuilder    = new SearchDocumentBuilder(baseDocumentBuilder);
            var searchIndexActionBuilder = new SearchIndexActionBuilder(
                versionListDataClient,
                output.GetLogger <SearchIndexActionBuilder>());

            _searchOperations = new Mock <IDocumentsOperationsWrapper>();
            _searchOperations
            .Setup(x => x.IndexAsync(It.IsAny <IndexBatch <KeyedDocument> >()))
            .Callback <IndexBatch <KeyedDocument> >(batch =>
            {
                _indexedBatch = batch;
            })
            .ReturnsAsync(new DocumentIndexResult());

            var hijackIndexClient = new Mock <ISearchIndexClientWrapper>();
            var searchIndexClient = new Mock <ISearchIndexClientWrapper>();

            searchIndexClient
            .Setup(x => x.Documents)
            .Returns(_searchOperations.Object);

            var batchPusher = new BatchPusher(
                searchIndexClient.Object,
                hijackIndexClient.Object,
                versionListDataClient,
                options.Object,
                developmentOptions.Object,
                _telemetry.Object,
                output.GetLogger <BatchPusher>());

            Func <IBatchPusher> batchPusherFactory = () => batchPusher;

            var time = new Mock <ISystemTime>();

            _featureFlags.Setup(x => x.IsPopularityTransferEnabled()).Returns(true);

            _target = new UpdateDownloadsCommand(
                auxiliaryFileClient,
                databaseFetcher.Object,
                downloadDataClient,
                downloadComparer,
                downloadTransferrer,
                popularityTransferDataClient,
                searchDocumentBuilder,
                searchIndexActionBuilder,
                batchPusherFactory,
                time.Object,
                _featureFlags.Object,
                options.Object,
                _telemetry.Object,
                output.GetLogger <Auxiliary2AzureSearchCommand>());
        }
예제 #4
0
            public async Task TransferChangesOverrideDownloadChanges()
            {
                DownloadSetComparer
                .Setup(c => c.Compare(It.IsAny <DownloadData>(), It.IsAny <DownloadData>()))
                .Returns <DownloadData, DownloadData>((oldData, newData) =>
                {
                    return(new SortedDictionary <string, long>(
                               newData.ToDictionary(d => d.Key, d => d.Value.Total),
                               StringComparer.OrdinalIgnoreCase));
                });

                NewDownloadData.SetDownloadCount("A", "1.0.0", 12);
                NewDownloadData.SetDownloadCount("A", "2.0.0", 34);

                NewDownloadData.SetDownloadCount("B", "3.0.0", 5);
                NewDownloadData.SetDownloadCount("B", "4.0.0", 4);

                NewDownloadData.SetDownloadCount("C", "5.0.0", 2);
                NewDownloadData.SetDownloadCount("C", "6.0.0", 3);

                TransferChanges["A"] = 55;
                TransferChanges["b"] = 66;

                NewTransfers.AddTransfer("A", "b");

                await Target.ExecuteAsync();

                // Documents should have new data with transfer changes.
                SearchDocumentBuilder
                .Verify(
                    b => b.UpdateDownloadCount("A", SearchFilters.IncludePrereleaseAndSemVer2, 55),
                    Times.Once);
                SearchDocumentBuilder
                .Verify(
                    b => b.UpdateDownloadCount("B", SearchFilters.IncludePrereleaseAndSemVer2, 66),
                    Times.Once);
                SearchDocumentBuilder
                .Verify(
                    b => b.UpdateDownloadCount("C", SearchFilters.IncludePrereleaseAndSemVer2, 5),
                    Times.Once);

                // Downloads auxiliary file should not reflect transfer changes.
                DownloadDataClient.Verify(
                    c => c.ReplaceLatestIndexedAsync(
                        It.Is <DownloadData>(d =>
                                             d["A"].Total == 46 &&
                                             d["A"]["1.0.0"] == 12 &&
                                             d["A"]["2.0.0"] == 34 &&

                                             d["B"].Total == 9 &&
                                             d["B"]["3.0.0"] == 5 &&
                                             d["B"]["4.0.0"] == 4 &&

                                             d["C"].Total == 5 &&
                                             d["C"]["5.0.0"] == 2 &&
                                             d["C"]["6.0.0"] == 3),
                        It.IsAny <IAccessCondition>()),
                    Times.Once);

                // Popularity transfers auxiliary file should have new data.
                PopularityTransferDataClient.Verify(
                    c => c.ReplaceLatestIndexedAsync(
                        It.Is <PopularityTransferData>(d =>
                                                       d.Count == 1 &&
                                                       d["A"].Count == 1 &&
                                                       d["A"].Contains("b")),
                        It.IsAny <IAccessCondition>()),
                    Times.Once);
            }
예제 #5
0
            public async Task AppliesTransferChanges()
            {
                var downloadChanges = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase);

                DownloadSetComparer
                .Setup(c => c.Compare(It.IsAny <DownloadData>(), It.IsAny <DownloadData>()))
                .Returns <DownloadData, DownloadData>((oldData, newData) =>
                {
                    return(downloadChanges);
                });

                TransferChanges["Package1"] = 100;
                TransferChanges["Package2"] = 200;

                NewTransfers.AddTransfer("Package1", "Package2");

                await Target.ExecuteAsync();

                PopularityTransferDataClient
                .Verify(
                    c => c.ReadLatestIndexedAsync(
                        It.Is <IAccessCondition>(x => x.IfMatchETag == null && x.IfNoneMatchETag == null),
                        It.IsAny <StringCache>()),
                    Times.Once);
                DatabaseFetcher
                .Verify(
                    d => d.GetPopularityTransfersAsync(),
                    Times.Once);

                DownloadTransferrer
                .Verify(
                    x => x.UpdateDownloadTransfers(
                        NewDownloadData,
                        downloadChanges,
                        OldTransfers,
                        NewTransfers),
                    Times.Once);

                // Documents should be updated.
                SearchDocumentBuilder
                .Verify(
                    b => b.UpdateDownloadCount("Package1", SearchFilters.IncludePrereleaseAndSemVer2, 100),
                    Times.Once);
                SearchDocumentBuilder
                .Verify(
                    b => b.UpdateDownloadCount("Package2", SearchFilters.IncludePrereleaseAndSemVer2, 200),
                    Times.Once);

                // Downloads auxiliary file should not include transfer changes.
                DownloadDataClient.Verify(
                    c => c.ReplaceLatestIndexedAsync(
                        It.Is <DownloadData>(d => d.Count == 0),
                        It.IsAny <IAccessCondition>()),
                    Times.Once);

                // Popularity transfers auxiliary file should have new data.
                PopularityTransferDataClient.Verify(
                    c => c.ReplaceLatestIndexedAsync(
                        It.Is <PopularityTransferData>(d =>
                                                       d.Count == 1 &&
                                                       d["Package1"].Count == 1 &&
                                                       d["Package1"].Contains("Package2")),
                        It.IsAny <IAccessCondition>()),
                    Times.Once);
            }