public Facts()
            {
                TransferChanges = new SortedDictionary <string, string[]>();
                DataComparer    = new Mock <IDataSetComparer>();
                DataComparer
                .Setup(x => x.ComparePopularityTransfers(
                           It.IsAny <PopularityTransferData>(),
                           It.IsAny <PopularityTransferData>()))
                .Returns(TransferChanges);

                PopularityTransfers = new PopularityTransferData();

                var options = new Mock <IOptionsSnapshot <AzureSearchJobConfiguration> >();

                options
                .Setup(x => x.Value)
                .Returns(() => new AzureSearchJobConfiguration
                {
                    Scoring = new AzureSearchScoringConfiguration
                    {
                        PopularityTransfer = PopularityTransfer
                    }
                });

                DownloadData = new DownloadData();

                Target = new DownloadTransferrer(
                    DataComparer.Object,
                    options.Object,
                    Mock.Of <ILogger <DownloadTransferrer> >());
            }
Exemple #2
0
        public async Task PushesPopularityTransferData()
        {
            PopularityTransferData data            = null;
            IAccessCondition       accessCondition = null;

            _popularityTransferDataClient
            .Setup(x => x.ReplaceLatestIndexedAsync(It.IsAny <PopularityTransferData>(), It.IsAny <IAccessCondition>()))
            .Returns(Task.CompletedTask)
            .Callback <PopularityTransferData, IAccessCondition>((d, a) =>
            {
                data            = d;
                accessCondition = a;
            });

            await _target.ExecuteAsync();

            Assert.Same(_initialAuxiliaryData.PopularityTransfers, data);

            Assert.Equal("*", accessCondition.IfNoneMatchETag);
            Assert.Null(accessCondition.IfMatchETag);

            _popularityTransferDataClient.Verify(
                x => x.ReplaceLatestIndexedAsync(It.IsAny <PopularityTransferData>(), It.IsAny <IAccessCondition>()),
                Times.Once);
        }
Exemple #3
0
        private async Task WritePopularityTransfersDataAsync(PopularityTransferData popularityTransfers)
        {
            _logger.LogInformation("Writing the initial popularity transfers data file.");
            await _popularityTransferDataClient.ReplaceLatestIndexedAsync(
                popularityTransfers,
                AccessConditionWrapper.GenerateIfNotExistsCondition());

            _logger.LogInformation("Done uploading the initial popularity transfers data file.");
        }
Exemple #4
0
 public InitialAuxiliaryData(
     SortedDictionary <string, SortedSet <string> > owners,
     DownloadData downloads,
     HashSet <string> excludedPackages,
     HashSet <string> verifiedPackages,
     PopularityTransferData popularityTransfers)
 {
     Owners              = owners ?? throw new ArgumentNullException(nameof(owners));
     Downloads           = downloads ?? throw new ArgumentNullException(nameof(downloads));
     ExcludedPackages    = excludedPackages ?? throw new ArgumentNullException(nameof(excludedPackages));
     VerifiedPackages    = verifiedPackages ?? throw new ArgumentNullException(nameof(verifiedPackages));
     PopularityTransfers = popularityTransfers ?? throw new ArgumentNullException(nameof(popularityTransfers));
 }
        private HashSet <string> GetPackagesAffectedByChanges(
            PopularityTransferData oldOutgoingTransfers,
            PopularityTransferData outgoingTransfers,
            SortedDictionary <string, SortedSet <string> > incomingTransfers,
            SortedDictionary <string, string[]> transferChanges,
            SortedDictionary <string, long> downloadChanges)
        {
            var affectedPackages = new HashSet <string>(StringComparer.OrdinalIgnoreCase);

            // If a package adds, changes, or removes outgoing transfers:
            //    Update "from" package
            //    Update all new "to" packages
            //    Update all old "to" packages (in case "to" packages were removed)
            foreach (var transferChange in transferChanges)
            {
                var fromPackage = transferChange.Key;
                var toPackages  = transferChange.Value;

                affectedPackages.Add(fromPackage);
                affectedPackages.UnionWith(toPackages);

                if (oldOutgoingTransfers.TryGetValue(fromPackage, out var oldToPackages))
                {
                    affectedPackages.UnionWith(oldToPackages);
                }
            }

            // If a package has download changes and outgoing transfers
            //    Update "from" package
            //    Update all "to" packages
            //
            // If a package has download changes and incoming transfers
            //    Update "to" package
            foreach (var packageId in downloadChanges.Keys)
            {
                if (outgoingTransfers.TryGetValue(packageId, out var toPackages))
                {
                    affectedPackages.Add(packageId);
                    affectedPackages.UnionWith(toPackages);
                }

                if (incomingTransfers.ContainsKey(packageId))
                {
                    affectedPackages.Add(packageId);
                }
            }

            return(affectedPackages);
        }
        public async Task <PopularityTransferData> GetPopularityTransfersAsync()
        {
            return(await RetrySqlAsync(async() =>
            {
                var stopwatch = Stopwatch.StartNew();
                var output = new PopularityTransferData();
                using (var connection = await _connectionFactory.OpenAsync())
                    using (var command = connection.CreateCommand())
                    {
                        command.CommandText = GetPopularityTransfersSql;
                        command.CommandTimeout = SqlCommandTimeoutSeconds;
                        command.Parameters.Add(GetPopularityTransfersSkipParameter, SqlDbType.Int);
                        command.Parameters.AddWithValue(GetPopularityTransfersTakeParameter, GetPopularityTransfersPageSize);

                        // Load popularity transfers by paging through the database.
                        // We continue paging until we receive fewer results than the page size.
                        int currentPageResults;
                        int totalResults = 0;
                        do
                        {
                            command.Parameters[GetPopularityTransfersSkipParameter].Value = totalResults;

                            using (var reader = await command.ExecuteReaderAsync())
                            {
                                currentPageResults = 0;

                                while (await reader.ReadAsync())
                                {
                                    currentPageResults++;

                                    var fromId = reader.GetString(0);
                                    var toId = reader.GetString(1);

                                    output.AddTransfer(fromId, toId);
                                }
                            }

                            totalResults += currentPageResults;
                        }while (currentPageResults == GetPopularityTransfersPageSize);

                        stopwatch.Stop();
                        _telemetryService.TrackReadLatestPopularityTransfersFromDatabase(output.Count, stopwatch.Elapsed);

                        return output;
                    }
            }));
        }
        public SortedDictionary <string, string[]> ComparePopularityTransfers(
            PopularityTransferData oldData,
            PopularityTransferData newData)
        {
            // Ignore case changes in popularity transfers.
            var stopwatch = Stopwatch.StartNew();
            var result    = CompareData(
                oldData,
                newData,
                "package ID",
                "popularity transfers",
                StringComparer.OrdinalIgnoreCase);

            stopwatch.Stop();
            _telemetryService.TrackPopularityTransfersSetComparison(oldData.Count, newData.Count, result.Count, stopwatch.Elapsed);

            return(result);
        }
        public SortedDictionary <string, long> InitializeDownloadTransfers(
            DownloadData downloads,
            PopularityTransferData outgoingTransfers)
        {
            // Downloads are transferred from a "from" package to one or more "to" packages.
            // The "outgoingTransfers" maps "from" packages to their corresponding "to" packages.
            // The "incomingTransfers" maps "to" packages to their corresponding "from" packages.
            var incomingTransfers = GetIncomingTransfers(outgoingTransfers);

            // Get the transfer changes for all packages that have popularity transfers.
            var packageIds = new HashSet <string>(StringComparer.OrdinalIgnoreCase);

            packageIds.UnionWith(outgoingTransfers.Keys);
            packageIds.UnionWith(incomingTransfers.Keys);

            return(ApplyDownloadTransfers(
                       downloads,
                       outgoingTransfers,
                       incomingTransfers,
                       packageIds));
        }
        private Dictionary <string, long> GetTransferredDownloads(
            DownloadData downloads,
            PopularityTransferData popularityTransfers)
        {
            var transferChanges = _downloadTransferrer.InitializeDownloadTransfers(
                downloads,
                popularityTransfers);

            var result = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase);

            foreach (var packageDownload in downloads)
            {
                result[packageDownload.Key] = packageDownload.Value.Total;
            }

            foreach (var transferChange in transferChanges)
            {
                result[transferChange.Key] = transferChange.Value;
            }

            return(result);
        }
        private long TransferPackageDownloads(
            string packageId,
            PopularityTransferData outgoingTransfers,
            SortedDictionary <string, SortedSet <string> > incomingTransfers,
            DownloadData downloads)
        {
            var originalDownloads  = downloads.GetDownloadCount(packageId);
            var transferPercentage = _options.Value.Scoring.PopularityTransfer;

            // Calculate packages with outgoing transfers first. These packages transfer a percentage
            // or their downloads equally to a set of "incoming" packages. Packages with both outgoing
            // and incoming transfers "reject" the incoming transfers.
            if (outgoingTransfers.ContainsKey(packageId))
            {
                var keepPercentage = 1 - transferPercentage;

                return((long)(originalDownloads * keepPercentage));
            }

            // Next, calculate packages with incoming transfers. These packages receive downloads
            // from one or more "outgoing" packages.
            if (incomingTransfers.TryGetValue(packageId, out var incomingTransferIds))
            {
                var result = originalDownloads;

                foreach (var incomingTransferId in incomingTransferIds)
                {
                    var incomingDownloads = downloads.GetDownloadCount(incomingTransferId);
                    var incomingSplit     = outgoingTransfers[incomingTransferId].Count;

                    result += (long)(incomingDownloads * transferPercentage / incomingSplit);
                }

                return(result);
            }

            // The package has no outgoing or incoming transfers. Return its downloads unchanged.
            return(originalDownloads);
        }
        public SortedDictionary <string, long> UpdateDownloadTransfers(
            DownloadData downloads,
            SortedDictionary <string, long> downloadChanges,
            PopularityTransferData oldTransfers,
            PopularityTransferData newTransfers)
        {
            Guard.Assert(
                downloadChanges.Comparer == StringComparer.OrdinalIgnoreCase,
                $"Download changes should have comparer {nameof(StringComparer.OrdinalIgnoreCase)}");

            Guard.Assert(
                downloadChanges.All(x => downloads.GetDownloadCount(x.Key) == x.Value),
                "The download changes should match the latest downloads");

            // Downloads are transferred from a "from" package to one or more "to" packages.
            // The "oldTransfers" and "newTransfers" maps "from" packages to their corresponding "to" packages.
            // The "incomingTransfers" maps "to" packages to their corresponding "from" packages.
            var incomingTransfers = GetIncomingTransfers(newTransfers);

            _logger.LogInformation("Detecting changes in popularity transfers.");
            var transferChanges = _dataComparer.ComparePopularityTransfers(oldTransfers, newTransfers);

            _logger.LogInformation("{Count} popularity transfers have changed.", transferChanges.Count);

            // Get the transfer changes for packages affected by the download and transfer changes.
            var affectedPackages = GetPackagesAffectedByChanges(
                oldTransfers,
                newTransfers,
                incomingTransfers,
                transferChanges,
                downloadChanges);

            return(ApplyDownloadTransfers(
                       downloads,
                       newTransfers,
                       incomingTransfers,
                       affectedPackages));
        }
        private SortedDictionary <string, long> ApplyDownloadTransfers(
            DownloadData downloads,
            PopularityTransferData outgoingTransfers,
            SortedDictionary <string, SortedSet <string> > incomingTransfers,
            HashSet <string> packageIds)
        {
            _logger.LogInformation(
                "{Count} package IDs have download changes due to popularity transfers.",
                packageIds.Count);

            var result = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase);

            foreach (var packageId in packageIds)
            {
                result[packageId] = TransferPackageDownloads(
                    packageId,
                    outgoingTransfers,
                    incomingTransfers,
                    downloads);
            }

            return(result);
        }
        private SortedDictionary <string, SortedSet <string> > GetIncomingTransfers(
            PopularityTransferData outgoingTransfers)
        {
            var result = new SortedDictionary <string, SortedSet <string> >(StringComparer.OrdinalIgnoreCase);

            foreach (var outgoingTransfer in outgoingTransfers)
            {
                var fromPackage = outgoingTransfer.Key;

                foreach (var toPackage in outgoingTransfer.Value)
                {
                    if (!result.TryGetValue(toPackage, out var incomingTransfer))
                    {
                        incomingTransfer = new SortedSet <string>(StringComparer.OrdinalIgnoreCase);
                        result.Add(toPackage, incomingTransfer);
                    }

                    incomingTransfer.Add(fromPackage);
                }
            }

            return(result);
        }
        private void ApplyDownloadTransfers(
            DownloadData newData,
            PopularityTransferData oldTransfers,
            PopularityTransferData newTransfers,
            SortedDictionary <string, long> downloadChanges)
        {
            _logger.LogInformation("Finding download changes from popularity transfers.");
            var transferChanges = _downloadTransferrer.UpdateDownloadTransfers(
                newData,
                downloadChanges,
                oldTransfers,
                newTransfers);

            _logger.LogInformation(
                "{Count} package IDs have download count changes from popularity transfers.",
                transferChanges.Count);

            // Apply the transfer changes to the overall download changes.
            foreach (var transferChange in transferChanges)
            {
                downloadChanges[transferChange.Key] = transferChange.Value;
            }
        }
Exemple #15
0
            public Facts(ITestOutputHelper output)
            {
                AuxiliaryFileClient          = new Mock <IAuxiliaryFileClient>();
                DatabaseFetcher              = new Mock <IDatabaseAuxiliaryDataFetcher>();
                DownloadDataClient           = new Mock <IDownloadDataClient>();
                DownloadSetComparer          = new Mock <IDownloadSetComparer>();
                DownloadTransferrer          = new Mock <IDownloadTransferrer>();
                PopularityTransferDataClient = new Mock <IPopularityTransferDataClient>();
                SearchDocumentBuilder        = new Mock <ISearchDocumentBuilder>();
                IndexActionBuilder           = new Mock <ISearchIndexActionBuilder>();
                BatchPusher      = new Mock <IBatchPusher>();
                SystemTime       = new Mock <ISystemTime>();
                FeatureFlags     = new Mock <IFeatureFlagService>();
                Options          = new Mock <IOptionsSnapshot <Auxiliary2AzureSearchConfiguration> >();
                TelemetryService = new Mock <IAzureSearchTelemetryService>();
                Logger           = output.GetLogger <Auxiliary2AzureSearchCommand>();

                Config = new Auxiliary2AzureSearchConfiguration
                {
                    AzureSearchBatchSize            = 10,
                    MaxConcurrentBatches            = 1,
                    MaxConcurrentVersionListWriters = 1,
                    EnablePopularityTransfers       = true,
                    MinPushPeriod = TimeSpan.FromSeconds(5),
                };
                Options.Setup(x => x.Value).Returns(() => Config);

                OldDownloadData   = new DownloadData();
                OldDownloadResult = Data.GetAuxiliaryFileResult(OldDownloadData, "download-data-etag");
                DownloadDataClient
                .Setup(x => x.ReadLatestIndexedAsync(It.IsAny <IAccessCondition>(), It.IsAny <StringCache>()))
                .ReturnsAsync(() => OldDownloadResult);
                NewDownloadData = new DownloadData();
                AuxiliaryFileClient.Setup(x => x.LoadDownloadDataAsync()).ReturnsAsync(() => NewDownloadData);

                Changes = new SortedDictionary <string, long>();
                DownloadSetComparer
                .Setup(x => x.Compare(It.IsAny <DownloadData>(), It.IsAny <DownloadData>()))
                .Returns(() => Changes);

                OldTransfers      = new PopularityTransferData();
                OldTransferResult = new AuxiliaryFileResult <PopularityTransferData>(
                    modified: true,
                    data: OldTransfers,
                    metadata: new AuxiliaryFileMetadata(
                        DateTimeOffset.UtcNow,
                        TimeSpan.Zero,
                        fileSize: 0,
                        etag: "etag"));
                PopularityTransferDataClient
                .Setup(x => x.ReadLatestIndexedAsync(It.IsAny <IAccessCondition>(), It.IsAny <StringCache>()))
                .ReturnsAsync(OldTransferResult);

                NewTransfers = new PopularityTransferData();
                DatabaseFetcher
                .Setup(x => x.GetPopularityTransfersAsync())
                .ReturnsAsync(NewTransfers);

                TransferChanges = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase);
                DownloadTransferrer
                .Setup(x => x.UpdateDownloadTransfers(
                           It.IsAny <DownloadData>(),
                           It.IsAny <SortedDictionary <string, long> >(),
                           It.IsAny <PopularityTransferData>(),
                           It.IsAny <PopularityTransferData>()))
                .Returns(TransferChanges);

                IndexActions = new IndexActions(
                    new List <IndexAction <KeyedDocument> > {
                    IndexAction.Merge(new KeyedDocument())
                },
                    new List <IndexAction <KeyedDocument> >(),
                    new ResultAndAccessCondition <VersionListData>(
                        new VersionListData(new Dictionary <string, VersionPropertiesData>()),
                        Mock.Of <IAccessCondition>()));
                ProcessedIds = new ConcurrentBag <string>();
                IndexActionBuilder
                .Setup(x => x.UpdateAsync(It.IsAny <string>(), It.IsAny <Func <SearchFilters, KeyedDocument> >()))
                .ReturnsAsync(() => IndexActions)
                .Callback <string, Func <SearchFilters, KeyedDocument> >((id, b) =>
                {
                    ProcessedIds.Add(id);
                    b(SearchFilters.IncludePrereleaseAndSemVer2);
                });

                // When pushing, delay for a little bit of time so the stopwatch has some measurable duration.
                PushedIds       = new ConcurrentBag <string>();
                CurrentBatch    = new ConcurrentBag <IndexActions>();
                FinishedBatches = new ConcurrentBag <List <IndexActions> >();
                BatchPusher
                .Setup(x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()))
                .Callback <string, IndexActions>((id, indexActions) =>
                {
                    CurrentBatch.Add(indexActions);
                    PushedIds.Add(id);
                });
                BatchPusher
                .Setup(x => x.TryFinishAsync())
                .Returns(async() =>
                {
                    await Task.Delay(TimeSpan.FromMilliseconds(1));
                    return(new BatchPusherResult());
                })
                .Callback(() =>
                {
                    FinishedBatches.Add(CurrentBatch.ToList());
                    CurrentBatch = new ConcurrentBag <IndexActions>();
                });

                FeatureFlags.Setup(x => x.IsPopularityTransferEnabled()).Returns(true);

                Target = new UpdateDownloadsCommand(
                    AuxiliaryFileClient.Object,
                    DatabaseFetcher.Object,
                    DownloadDataClient.Object,
                    DownloadSetComparer.Object,
                    DownloadTransferrer.Object,
                    PopularityTransferDataClient.Object,
                    SearchDocumentBuilder.Object,
                    IndexActionBuilder.Object,
                    () => BatchPusher.Object,
                    SystemTime.Object,
                    FeatureFlags.Object,
                    Options.Object,
                    TelemetryService.Object,
                    Logger);
            }
        public PopularityTransferIntegrationTests(ITestOutputHelper output)
        {
            _featureFlags = new Mock <IFeatureFlagService>();
            _telemetry    = new Mock <IAzureSearchTelemetryService>();

            _config = new Auxiliary2AzureSearchConfiguration
            {
                AuxiliaryDataStorageContainer = "auxiliary-container",
                EnablePopularityTransfers     = true,
                StorageContainer = "storage-container",
                Scoring          = new AzureSearchScoringConfiguration()
            };

            var options = new Mock <IOptionsSnapshot <Auxiliary2AzureSearchConfiguration> >();

            options
            .Setup(x => x.Value)
            .Returns(_config);

            _developmentConfig = new AzureSearchJobDevelopmentConfiguration();
            var developmentOptions = new Mock <IOptionsSnapshot <AzureSearchJobDevelopmentConfiguration> >();

            developmentOptions
            .Setup(x => x.Value)
            .Returns(_developmentConfig);

            var auxiliaryConfig = new AuxiliaryDataStorageConfiguration
            {
                AuxiliaryDataStorageContainer            = "auxiliary-container",
                AuxiliaryDataStorageDownloadsPath        = "downloads.json",
                AuxiliaryDataStorageExcludedPackagesPath = "excludedPackages.json",
            };

            var auxiliaryOptions = new Mock <IOptionsSnapshot <AuxiliaryDataStorageConfiguration> >();

            auxiliaryOptions
            .Setup(x => x.Value)
            .Returns(auxiliaryConfig);

            _auxilliaryContainer = new InMemoryCloudBlobContainer();
            _storageContainer    = new InMemoryCloudBlobContainer();

            _blobClient = new InMemoryCloudBlobClient();
            _blobClient.Containers["auxiliary-container"] = _auxilliaryContainer;
            _blobClient.Containers["storage-container"]   = _storageContainer;

            var auxiliaryFileClient = new AuxiliaryFileClient(
                _blobClient,
                auxiliaryOptions.Object,
                _telemetry.Object,
                output.GetLogger <AuxiliaryFileClient>());

            _newPopularityTransfers = new PopularityTransferData();
            var databaseFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>();

            databaseFetcher
            .Setup(x => x.GetPopularityTransfersAsync())
            .ReturnsAsync(_newPopularityTransfers);

            var downloadDataClient = new DownloadDataClient(
                _blobClient,
                options.Object,
                _telemetry.Object,
                output.GetLogger <DownloadDataClient>());

            var popularityTransferDataClient = new PopularityTransferDataClient(
                _blobClient,
                options.Object,
                _telemetry.Object,
                output.GetLogger <PopularityTransferDataClient>());

            var versionListDataClient = new VersionListDataClient(
                _blobClient,
                options.Object,
                output.GetLogger <VersionListDataClient>());

            var downloadComparer = new DownloadSetComparer(
                _telemetry.Object,
                options.Object,
                output.GetLogger <DownloadSetComparer>());

            var dataComparer = new DataSetComparer(
                _telemetry.Object,
                output.GetLogger <DataSetComparer>());

            var downloadTransferrer = new DownloadTransferrer(
                dataComparer,
                options.Object,
                output.GetLogger <DownloadTransferrer>());

            var baseDocumentBuilder      = new BaseDocumentBuilder(options.Object);
            var searchDocumentBuilder    = new SearchDocumentBuilder(baseDocumentBuilder);
            var searchIndexActionBuilder = new SearchIndexActionBuilder(
                versionListDataClient,
                output.GetLogger <SearchIndexActionBuilder>());

            _searchOperations = new Mock <IDocumentsOperationsWrapper>();
            _searchOperations
            .Setup(x => x.IndexAsync(It.IsAny <IndexBatch <KeyedDocument> >()))
            .Callback <IndexBatch <KeyedDocument> >(batch =>
            {
                _indexedBatch = batch;
            })
            .ReturnsAsync(new DocumentIndexResult());

            var hijackIndexClient = new Mock <ISearchIndexClientWrapper>();
            var searchIndexClient = new Mock <ISearchIndexClientWrapper>();

            searchIndexClient
            .Setup(x => x.Documents)
            .Returns(_searchOperations.Object);

            var batchPusher = new BatchPusher(
                searchIndexClient.Object,
                hijackIndexClient.Object,
                versionListDataClient,
                options.Object,
                developmentOptions.Object,
                _telemetry.Object,
                output.GetLogger <BatchPusher>());

            Func <IBatchPusher> batchPusherFactory = () => batchPusher;

            var time = new Mock <ISystemTime>();

            _featureFlags.Setup(x => x.IsPopularityTransferEnabled()).Returns(true);

            _target = new UpdateDownloadsCommand(
                auxiliaryFileClient,
                databaseFetcher.Object,
                downloadDataClient,
                downloadComparer,
                downloadTransferrer,
                popularityTransferDataClient,
                searchDocumentBuilder,
                searchIndexActionBuilder,
                batchPusherFactory,
                time.Object,
                _featureFlags.Object,
                options.Object,
                _telemetry.Object,
                output.GetLogger <Auxiliary2AzureSearchCommand>());
        }
 public GetUpdatedTransferChanges()
 {
     DownloadChanges = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase);
     OldTransfers    = new PopularityTransferData();
 }
Exemple #18
0
            public ProduceWorkAsync(ITestOutputHelper output)
            {
                _entitiesContextFactory = new Mock <IEntitiesContextFactory>();
                _entitiesContext        = new Mock <IEntitiesContext>();
                _options = new Mock <IOptionsSnapshot <Db2AzureSearchConfiguration> >();
                _config  = new Db2AzureSearchConfiguration
                {
                    DatabaseBatchSize         = 2,
                    EnablePopularityTransfers = true,
                };
                _developmentOptions   = new Mock <IOptionsSnapshot <Db2AzureSearchDevelopmentConfiguration> >();
                _developmentConfig    = new Db2AzureSearchDevelopmentConfiguration();
                _logger               = output.GetLogger <NewPackageRegistrationProducer>();
                _packageRegistrations = DbSetMockFactory.Create <PackageRegistration>();
                _packages             = DbSetMockFactory.Create <Package>();
                _work  = new ConcurrentBag <NewPackageRegistration>();
                _token = CancellationToken.None;

                _auxiliaryFileClient = new Mock <IAuxiliaryFileClient>();
                _excludedPackages    = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
                _auxiliaryFileClient
                .Setup(x => x.LoadExcludedPackagesAsync())
                .ReturnsAsync(() => _excludedPackages);
                _downloads = new DownloadData();
                _auxiliaryFileClient
                .Setup(x => x.LoadDownloadDataAsync())
                .ReturnsAsync(() => _downloads);

                _popularityTransfers = new PopularityTransferData();
                _databaseFetcher     = new Mock <IDatabaseAuxiliaryDataFetcher>();
                _databaseFetcher
                .Setup(x => x.GetPopularityTransfersAsync())
                .ReturnsAsync(() => _popularityTransfers);

                _downloadTransferrer = new Mock <IDownloadTransferrer>();
                _transferChanges     = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase);
                _downloadTransferrer
                .Setup(x => x.InitializeDownloadTransfers(
                           It.IsAny <DownloadData>(),
                           It.IsAny <PopularityTransferData>()))
                .Returns(_transferChanges);

                _featureFlags = new Mock <IFeatureFlagService>();
                _featureFlags
                .Setup(x => x.IsPopularityTransferEnabled())
                .Returns(true);

                _entitiesContextFactory
                .Setup(x => x.CreateAsync(It.IsAny <bool>()))
                .ReturnsAsync(() => _entitiesContext.Object);
                _entitiesContext
                .Setup(x => x.Set <PackageRegistration>())
                .Returns(() => _packageRegistrations);
                _entitiesContext
                .Setup(x => x.Set <Package>())
                .Returns(() => _packages);
                _options
                .Setup(x => x.Value)
                .Returns(() => _config);
                _developmentOptions
                .Setup(x => x.Value)
                .Returns(() => _developmentConfig);

                _target = new NewPackageRegistrationProducer(
                    _entitiesContextFactory.Object,
                    _auxiliaryFileClient.Object,
                    _databaseFetcher.Object,
                    _downloadTransferrer.Object,
                    _featureFlags.Object,
                    _options.Object,
                    _developmentOptions.Object,
                    _logger);
            }
 public ComparePopularityTransfers(ITestOutputHelper output) : base(output)
 {
     OldData = new PopularityTransferData();
     NewData = new PopularityTransferData();
 }