public Facts() { TransferChanges = new SortedDictionary <string, string[]>(); DataComparer = new Mock <IDataSetComparer>(); DataComparer .Setup(x => x.ComparePopularityTransfers( It.IsAny <PopularityTransferData>(), It.IsAny <PopularityTransferData>())) .Returns(TransferChanges); PopularityTransfers = new PopularityTransferData(); var options = new Mock <IOptionsSnapshot <AzureSearchJobConfiguration> >(); options .Setup(x => x.Value) .Returns(() => new AzureSearchJobConfiguration { Scoring = new AzureSearchScoringConfiguration { PopularityTransfer = PopularityTransfer } }); DownloadData = new DownloadData(); Target = new DownloadTransferrer( DataComparer.Object, options.Object, Mock.Of <ILogger <DownloadTransferrer> >()); }
public async Task PushesPopularityTransferData() { PopularityTransferData data = null; IAccessCondition accessCondition = null; _popularityTransferDataClient .Setup(x => x.ReplaceLatestIndexedAsync(It.IsAny <PopularityTransferData>(), It.IsAny <IAccessCondition>())) .Returns(Task.CompletedTask) .Callback <PopularityTransferData, IAccessCondition>((d, a) => { data = d; accessCondition = a; }); await _target.ExecuteAsync(); Assert.Same(_initialAuxiliaryData.PopularityTransfers, data); Assert.Equal("*", accessCondition.IfNoneMatchETag); Assert.Null(accessCondition.IfMatchETag); _popularityTransferDataClient.Verify( x => x.ReplaceLatestIndexedAsync(It.IsAny <PopularityTransferData>(), It.IsAny <IAccessCondition>()), Times.Once); }
private async Task WritePopularityTransfersDataAsync(PopularityTransferData popularityTransfers) { _logger.LogInformation("Writing the initial popularity transfers data file."); await _popularityTransferDataClient.ReplaceLatestIndexedAsync( popularityTransfers, AccessConditionWrapper.GenerateIfNotExistsCondition()); _logger.LogInformation("Done uploading the initial popularity transfers data file."); }
public InitialAuxiliaryData( SortedDictionary <string, SortedSet <string> > owners, DownloadData downloads, HashSet <string> excludedPackages, HashSet <string> verifiedPackages, PopularityTransferData popularityTransfers) { Owners = owners ?? throw new ArgumentNullException(nameof(owners)); Downloads = downloads ?? throw new ArgumentNullException(nameof(downloads)); ExcludedPackages = excludedPackages ?? throw new ArgumentNullException(nameof(excludedPackages)); VerifiedPackages = verifiedPackages ?? throw new ArgumentNullException(nameof(verifiedPackages)); PopularityTransfers = popularityTransfers ?? throw new ArgumentNullException(nameof(popularityTransfers)); }
private HashSet <string> GetPackagesAffectedByChanges( PopularityTransferData oldOutgoingTransfers, PopularityTransferData outgoingTransfers, SortedDictionary <string, SortedSet <string> > incomingTransfers, SortedDictionary <string, string[]> transferChanges, SortedDictionary <string, long> downloadChanges) { var affectedPackages = new HashSet <string>(StringComparer.OrdinalIgnoreCase); // If a package adds, changes, or removes outgoing transfers: // Update "from" package // Update all new "to" packages // Update all old "to" packages (in case "to" packages were removed) foreach (var transferChange in transferChanges) { var fromPackage = transferChange.Key; var toPackages = transferChange.Value; affectedPackages.Add(fromPackage); affectedPackages.UnionWith(toPackages); if (oldOutgoingTransfers.TryGetValue(fromPackage, out var oldToPackages)) { affectedPackages.UnionWith(oldToPackages); } } // If a package has download changes and outgoing transfers // Update "from" package // Update all "to" packages // // If a package has download changes and incoming transfers // Update "to" package foreach (var packageId in downloadChanges.Keys) { if (outgoingTransfers.TryGetValue(packageId, out var toPackages)) { affectedPackages.Add(packageId); affectedPackages.UnionWith(toPackages); } if (incomingTransfers.ContainsKey(packageId)) { affectedPackages.Add(packageId); } } return(affectedPackages); }
public async Task <PopularityTransferData> GetPopularityTransfersAsync() { return(await RetrySqlAsync(async() => { var stopwatch = Stopwatch.StartNew(); var output = new PopularityTransferData(); using (var connection = await _connectionFactory.OpenAsync()) using (var command = connection.CreateCommand()) { command.CommandText = GetPopularityTransfersSql; command.CommandTimeout = SqlCommandTimeoutSeconds; command.Parameters.Add(GetPopularityTransfersSkipParameter, SqlDbType.Int); command.Parameters.AddWithValue(GetPopularityTransfersTakeParameter, GetPopularityTransfersPageSize); // Load popularity transfers by paging through the database. // We continue paging until we receive fewer results than the page size. int currentPageResults; int totalResults = 0; do { command.Parameters[GetPopularityTransfersSkipParameter].Value = totalResults; using (var reader = await command.ExecuteReaderAsync()) { currentPageResults = 0; while (await reader.ReadAsync()) { currentPageResults++; var fromId = reader.GetString(0); var toId = reader.GetString(1); output.AddTransfer(fromId, toId); } } totalResults += currentPageResults; }while (currentPageResults == GetPopularityTransfersPageSize); stopwatch.Stop(); _telemetryService.TrackReadLatestPopularityTransfersFromDatabase(output.Count, stopwatch.Elapsed); return output; } })); }
public SortedDictionary <string, string[]> ComparePopularityTransfers( PopularityTransferData oldData, PopularityTransferData newData) { // Ignore case changes in popularity transfers. var stopwatch = Stopwatch.StartNew(); var result = CompareData( oldData, newData, "package ID", "popularity transfers", StringComparer.OrdinalIgnoreCase); stopwatch.Stop(); _telemetryService.TrackPopularityTransfersSetComparison(oldData.Count, newData.Count, result.Count, stopwatch.Elapsed); return(result); }
public SortedDictionary <string, long> InitializeDownloadTransfers( DownloadData downloads, PopularityTransferData outgoingTransfers) { // Downloads are transferred from a "from" package to one or more "to" packages. // The "outgoingTransfers" maps "from" packages to their corresponding "to" packages. // The "incomingTransfers" maps "to" packages to their corresponding "from" packages. var incomingTransfers = GetIncomingTransfers(outgoingTransfers); // Get the transfer changes for all packages that have popularity transfers. var packageIds = new HashSet <string>(StringComparer.OrdinalIgnoreCase); packageIds.UnionWith(outgoingTransfers.Keys); packageIds.UnionWith(incomingTransfers.Keys); return(ApplyDownloadTransfers( downloads, outgoingTransfers, incomingTransfers, packageIds)); }
private Dictionary <string, long> GetTransferredDownloads( DownloadData downloads, PopularityTransferData popularityTransfers) { var transferChanges = _downloadTransferrer.InitializeDownloadTransfers( downloads, popularityTransfers); var result = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase); foreach (var packageDownload in downloads) { result[packageDownload.Key] = packageDownload.Value.Total; } foreach (var transferChange in transferChanges) { result[transferChange.Key] = transferChange.Value; } return(result); }
private long TransferPackageDownloads( string packageId, PopularityTransferData outgoingTransfers, SortedDictionary <string, SortedSet <string> > incomingTransfers, DownloadData downloads) { var originalDownloads = downloads.GetDownloadCount(packageId); var transferPercentage = _options.Value.Scoring.PopularityTransfer; // Calculate packages with outgoing transfers first. These packages transfer a percentage // or their downloads equally to a set of "incoming" packages. Packages with both outgoing // and incoming transfers "reject" the incoming transfers. if (outgoingTransfers.ContainsKey(packageId)) { var keepPercentage = 1 - transferPercentage; return((long)(originalDownloads * keepPercentage)); } // Next, calculate packages with incoming transfers. These packages receive downloads // from one or more "outgoing" packages. if (incomingTransfers.TryGetValue(packageId, out var incomingTransferIds)) { var result = originalDownloads; foreach (var incomingTransferId in incomingTransferIds) { var incomingDownloads = downloads.GetDownloadCount(incomingTransferId); var incomingSplit = outgoingTransfers[incomingTransferId].Count; result += (long)(incomingDownloads * transferPercentage / incomingSplit); } return(result); } // The package has no outgoing or incoming transfers. Return its downloads unchanged. return(originalDownloads); }
public SortedDictionary <string, long> UpdateDownloadTransfers( DownloadData downloads, SortedDictionary <string, long> downloadChanges, PopularityTransferData oldTransfers, PopularityTransferData newTransfers) { Guard.Assert( downloadChanges.Comparer == StringComparer.OrdinalIgnoreCase, $"Download changes should have comparer {nameof(StringComparer.OrdinalIgnoreCase)}"); Guard.Assert( downloadChanges.All(x => downloads.GetDownloadCount(x.Key) == x.Value), "The download changes should match the latest downloads"); // Downloads are transferred from a "from" package to one or more "to" packages. // The "oldTransfers" and "newTransfers" maps "from" packages to their corresponding "to" packages. // The "incomingTransfers" maps "to" packages to their corresponding "from" packages. var incomingTransfers = GetIncomingTransfers(newTransfers); _logger.LogInformation("Detecting changes in popularity transfers."); var transferChanges = _dataComparer.ComparePopularityTransfers(oldTransfers, newTransfers); _logger.LogInformation("{Count} popularity transfers have changed.", transferChanges.Count); // Get the transfer changes for packages affected by the download and transfer changes. var affectedPackages = GetPackagesAffectedByChanges( oldTransfers, newTransfers, incomingTransfers, transferChanges, downloadChanges); return(ApplyDownloadTransfers( downloads, newTransfers, incomingTransfers, affectedPackages)); }
private SortedDictionary <string, long> ApplyDownloadTransfers( DownloadData downloads, PopularityTransferData outgoingTransfers, SortedDictionary <string, SortedSet <string> > incomingTransfers, HashSet <string> packageIds) { _logger.LogInformation( "{Count} package IDs have download changes due to popularity transfers.", packageIds.Count); var result = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase); foreach (var packageId in packageIds) { result[packageId] = TransferPackageDownloads( packageId, outgoingTransfers, incomingTransfers, downloads); } return(result); }
private SortedDictionary <string, SortedSet <string> > GetIncomingTransfers( PopularityTransferData outgoingTransfers) { var result = new SortedDictionary <string, SortedSet <string> >(StringComparer.OrdinalIgnoreCase); foreach (var outgoingTransfer in outgoingTransfers) { var fromPackage = outgoingTransfer.Key; foreach (var toPackage in outgoingTransfer.Value) { if (!result.TryGetValue(toPackage, out var incomingTransfer)) { incomingTransfer = new SortedSet <string>(StringComparer.OrdinalIgnoreCase); result.Add(toPackage, incomingTransfer); } incomingTransfer.Add(fromPackage); } } return(result); }
private void ApplyDownloadTransfers( DownloadData newData, PopularityTransferData oldTransfers, PopularityTransferData newTransfers, SortedDictionary <string, long> downloadChanges) { _logger.LogInformation("Finding download changes from popularity transfers."); var transferChanges = _downloadTransferrer.UpdateDownloadTransfers( newData, downloadChanges, oldTransfers, newTransfers); _logger.LogInformation( "{Count} package IDs have download count changes from popularity transfers.", transferChanges.Count); // Apply the transfer changes to the overall download changes. foreach (var transferChange in transferChanges) { downloadChanges[transferChange.Key] = transferChange.Value; } }
public Facts(ITestOutputHelper output) { AuxiliaryFileClient = new Mock <IAuxiliaryFileClient>(); DatabaseFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>(); DownloadDataClient = new Mock <IDownloadDataClient>(); DownloadSetComparer = new Mock <IDownloadSetComparer>(); DownloadTransferrer = new Mock <IDownloadTransferrer>(); PopularityTransferDataClient = new Mock <IPopularityTransferDataClient>(); SearchDocumentBuilder = new Mock <ISearchDocumentBuilder>(); IndexActionBuilder = new Mock <ISearchIndexActionBuilder>(); BatchPusher = new Mock <IBatchPusher>(); SystemTime = new Mock <ISystemTime>(); FeatureFlags = new Mock <IFeatureFlagService>(); Options = new Mock <IOptionsSnapshot <Auxiliary2AzureSearchConfiguration> >(); TelemetryService = new Mock <IAzureSearchTelemetryService>(); Logger = output.GetLogger <Auxiliary2AzureSearchCommand>(); Config = new Auxiliary2AzureSearchConfiguration { AzureSearchBatchSize = 10, MaxConcurrentBatches = 1, MaxConcurrentVersionListWriters = 1, EnablePopularityTransfers = true, MinPushPeriod = TimeSpan.FromSeconds(5), }; Options.Setup(x => x.Value).Returns(() => Config); OldDownloadData = new DownloadData(); OldDownloadResult = Data.GetAuxiliaryFileResult(OldDownloadData, "download-data-etag"); DownloadDataClient .Setup(x => x.ReadLatestIndexedAsync(It.IsAny <IAccessCondition>(), It.IsAny <StringCache>())) .ReturnsAsync(() => OldDownloadResult); NewDownloadData = new DownloadData(); AuxiliaryFileClient.Setup(x => x.LoadDownloadDataAsync()).ReturnsAsync(() => NewDownloadData); Changes = new SortedDictionary <string, long>(); DownloadSetComparer .Setup(x => x.Compare(It.IsAny <DownloadData>(), It.IsAny <DownloadData>())) .Returns(() => Changes); OldTransfers = new PopularityTransferData(); OldTransferResult = new AuxiliaryFileResult <PopularityTransferData>( modified: true, data: OldTransfers, metadata: new AuxiliaryFileMetadata( DateTimeOffset.UtcNow, TimeSpan.Zero, fileSize: 0, etag: "etag")); PopularityTransferDataClient .Setup(x => x.ReadLatestIndexedAsync(It.IsAny <IAccessCondition>(), It.IsAny <StringCache>())) .ReturnsAsync(OldTransferResult); NewTransfers = new PopularityTransferData(); DatabaseFetcher .Setup(x => x.GetPopularityTransfersAsync()) .ReturnsAsync(NewTransfers); TransferChanges = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase); DownloadTransferrer .Setup(x => x.UpdateDownloadTransfers( It.IsAny <DownloadData>(), It.IsAny <SortedDictionary <string, long> >(), It.IsAny <PopularityTransferData>(), It.IsAny <PopularityTransferData>())) .Returns(TransferChanges); IndexActions = new IndexActions( new List <IndexAction <KeyedDocument> > { IndexAction.Merge(new KeyedDocument()) }, new List <IndexAction <KeyedDocument> >(), new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), Mock.Of <IAccessCondition>())); ProcessedIds = new ConcurrentBag <string>(); IndexActionBuilder .Setup(x => x.UpdateAsync(It.IsAny <string>(), It.IsAny <Func <SearchFilters, KeyedDocument> >())) .ReturnsAsync(() => IndexActions) .Callback <string, Func <SearchFilters, KeyedDocument> >((id, b) => { ProcessedIds.Add(id); b(SearchFilters.IncludePrereleaseAndSemVer2); }); // When pushing, delay for a little bit of time so the stopwatch has some measurable duration. PushedIds = new ConcurrentBag <string>(); CurrentBatch = new ConcurrentBag <IndexActions>(); FinishedBatches = new ConcurrentBag <List <IndexActions> >(); BatchPusher .Setup(x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>())) .Callback <string, IndexActions>((id, indexActions) => { CurrentBatch.Add(indexActions); PushedIds.Add(id); }); BatchPusher .Setup(x => x.TryFinishAsync()) .Returns(async() => { await Task.Delay(TimeSpan.FromMilliseconds(1)); return(new BatchPusherResult()); }) .Callback(() => { FinishedBatches.Add(CurrentBatch.ToList()); CurrentBatch = new ConcurrentBag <IndexActions>(); }); FeatureFlags.Setup(x => x.IsPopularityTransferEnabled()).Returns(true); Target = new UpdateDownloadsCommand( AuxiliaryFileClient.Object, DatabaseFetcher.Object, DownloadDataClient.Object, DownloadSetComparer.Object, DownloadTransferrer.Object, PopularityTransferDataClient.Object, SearchDocumentBuilder.Object, IndexActionBuilder.Object, () => BatchPusher.Object, SystemTime.Object, FeatureFlags.Object, Options.Object, TelemetryService.Object, Logger); }
public PopularityTransferIntegrationTests(ITestOutputHelper output) { _featureFlags = new Mock <IFeatureFlagService>(); _telemetry = new Mock <IAzureSearchTelemetryService>(); _config = new Auxiliary2AzureSearchConfiguration { AuxiliaryDataStorageContainer = "auxiliary-container", EnablePopularityTransfers = true, StorageContainer = "storage-container", Scoring = new AzureSearchScoringConfiguration() }; var options = new Mock <IOptionsSnapshot <Auxiliary2AzureSearchConfiguration> >(); options .Setup(x => x.Value) .Returns(_config); _developmentConfig = new AzureSearchJobDevelopmentConfiguration(); var developmentOptions = new Mock <IOptionsSnapshot <AzureSearchJobDevelopmentConfiguration> >(); developmentOptions .Setup(x => x.Value) .Returns(_developmentConfig); var auxiliaryConfig = new AuxiliaryDataStorageConfiguration { AuxiliaryDataStorageContainer = "auxiliary-container", AuxiliaryDataStorageDownloadsPath = "downloads.json", AuxiliaryDataStorageExcludedPackagesPath = "excludedPackages.json", }; var auxiliaryOptions = new Mock <IOptionsSnapshot <AuxiliaryDataStorageConfiguration> >(); auxiliaryOptions .Setup(x => x.Value) .Returns(auxiliaryConfig); _auxilliaryContainer = new InMemoryCloudBlobContainer(); _storageContainer = new InMemoryCloudBlobContainer(); _blobClient = new InMemoryCloudBlobClient(); _blobClient.Containers["auxiliary-container"] = _auxilliaryContainer; _blobClient.Containers["storage-container"] = _storageContainer; var auxiliaryFileClient = new AuxiliaryFileClient( _blobClient, auxiliaryOptions.Object, _telemetry.Object, output.GetLogger <AuxiliaryFileClient>()); _newPopularityTransfers = new PopularityTransferData(); var databaseFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>(); databaseFetcher .Setup(x => x.GetPopularityTransfersAsync()) .ReturnsAsync(_newPopularityTransfers); var downloadDataClient = new DownloadDataClient( _blobClient, options.Object, _telemetry.Object, output.GetLogger <DownloadDataClient>()); var popularityTransferDataClient = new PopularityTransferDataClient( _blobClient, options.Object, _telemetry.Object, output.GetLogger <PopularityTransferDataClient>()); var versionListDataClient = new VersionListDataClient( _blobClient, options.Object, output.GetLogger <VersionListDataClient>()); var downloadComparer = new DownloadSetComparer( _telemetry.Object, options.Object, output.GetLogger <DownloadSetComparer>()); var dataComparer = new DataSetComparer( _telemetry.Object, output.GetLogger <DataSetComparer>()); var downloadTransferrer = new DownloadTransferrer( dataComparer, options.Object, output.GetLogger <DownloadTransferrer>()); var baseDocumentBuilder = new BaseDocumentBuilder(options.Object); var searchDocumentBuilder = new SearchDocumentBuilder(baseDocumentBuilder); var searchIndexActionBuilder = new SearchIndexActionBuilder( versionListDataClient, output.GetLogger <SearchIndexActionBuilder>()); _searchOperations = new Mock <IDocumentsOperationsWrapper>(); _searchOperations .Setup(x => x.IndexAsync(It.IsAny <IndexBatch <KeyedDocument> >())) .Callback <IndexBatch <KeyedDocument> >(batch => { _indexedBatch = batch; }) .ReturnsAsync(new DocumentIndexResult()); var hijackIndexClient = new Mock <ISearchIndexClientWrapper>(); var searchIndexClient = new Mock <ISearchIndexClientWrapper>(); searchIndexClient .Setup(x => x.Documents) .Returns(_searchOperations.Object); var batchPusher = new BatchPusher( searchIndexClient.Object, hijackIndexClient.Object, versionListDataClient, options.Object, developmentOptions.Object, _telemetry.Object, output.GetLogger <BatchPusher>()); Func <IBatchPusher> batchPusherFactory = () => batchPusher; var time = new Mock <ISystemTime>(); _featureFlags.Setup(x => x.IsPopularityTransferEnabled()).Returns(true); _target = new UpdateDownloadsCommand( auxiliaryFileClient, databaseFetcher.Object, downloadDataClient, downloadComparer, downloadTransferrer, popularityTransferDataClient, searchDocumentBuilder, searchIndexActionBuilder, batchPusherFactory, time.Object, _featureFlags.Object, options.Object, _telemetry.Object, output.GetLogger <Auxiliary2AzureSearchCommand>()); }
public GetUpdatedTransferChanges() { DownloadChanges = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase); OldTransfers = new PopularityTransferData(); }
public ProduceWorkAsync(ITestOutputHelper output) { _entitiesContextFactory = new Mock <IEntitiesContextFactory>(); _entitiesContext = new Mock <IEntitiesContext>(); _options = new Mock <IOptionsSnapshot <Db2AzureSearchConfiguration> >(); _config = new Db2AzureSearchConfiguration { DatabaseBatchSize = 2, EnablePopularityTransfers = true, }; _developmentOptions = new Mock <IOptionsSnapshot <Db2AzureSearchDevelopmentConfiguration> >(); _developmentConfig = new Db2AzureSearchDevelopmentConfiguration(); _logger = output.GetLogger <NewPackageRegistrationProducer>(); _packageRegistrations = DbSetMockFactory.Create <PackageRegistration>(); _packages = DbSetMockFactory.Create <Package>(); _work = new ConcurrentBag <NewPackageRegistration>(); _token = CancellationToken.None; _auxiliaryFileClient = new Mock <IAuxiliaryFileClient>(); _excludedPackages = new HashSet <string>(StringComparer.OrdinalIgnoreCase); _auxiliaryFileClient .Setup(x => x.LoadExcludedPackagesAsync()) .ReturnsAsync(() => _excludedPackages); _downloads = new DownloadData(); _auxiliaryFileClient .Setup(x => x.LoadDownloadDataAsync()) .ReturnsAsync(() => _downloads); _popularityTransfers = new PopularityTransferData(); _databaseFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>(); _databaseFetcher .Setup(x => x.GetPopularityTransfersAsync()) .ReturnsAsync(() => _popularityTransfers); _downloadTransferrer = new Mock <IDownloadTransferrer>(); _transferChanges = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase); _downloadTransferrer .Setup(x => x.InitializeDownloadTransfers( It.IsAny <DownloadData>(), It.IsAny <PopularityTransferData>())) .Returns(_transferChanges); _featureFlags = new Mock <IFeatureFlagService>(); _featureFlags .Setup(x => x.IsPopularityTransferEnabled()) .Returns(true); _entitiesContextFactory .Setup(x => x.CreateAsync(It.IsAny <bool>())) .ReturnsAsync(() => _entitiesContext.Object); _entitiesContext .Setup(x => x.Set <PackageRegistration>()) .Returns(() => _packageRegistrations); _entitiesContext .Setup(x => x.Set <Package>()) .Returns(() => _packages); _options .Setup(x => x.Value) .Returns(() => _config); _developmentOptions .Setup(x => x.Value) .Returns(() => _developmentConfig); _target = new NewPackageRegistrationProducer( _entitiesContextFactory.Object, _auxiliaryFileClient.Object, _databaseFetcher.Object, _downloadTransferrer.Object, _featureFlags.Object, _options.Object, _developmentOptions.Object, _logger); }
public ComparePopularityTransfers(ITestOutputHelper output) : base(output) { OldData = new PopularityTransferData(); NewData = new PopularityTransferData(); }