private async Task <bool> UpdateVerifiedPackagesAsync() { // The "old" data in this case is the latest file that was copied to the region's storage container by this // job (or initialized by Db2AzureSearch). var oldResult = await _verifiedPackagesDataClient.ReadLatestAsync( AccessConditionWrapper.GenerateEmptyCondition(), _stringCache); // The "new" data in this case is from the database. var newData = await _databaseFetcher.GetVerifiedPackagesAsync(); var changes = new HashSet <string>(oldResult.Data, oldResult.Data.Comparer); changes.SymmetricExceptWith(newData); _logger.LogInformation("{Count} package IDs have verified status changes.", changes.Count); if (changes.Count == 0) { return(false); } else { await _verifiedPackagesDataClient.ReplaceLatestAsync(newData, oldResult.Metadata.GetIfMatchCondition()); return(true); } }
public void GenerateIfNoneMatchCondition(string etag) { var actual = AccessConditionWrapper.GenerateIfNoneMatchCondition(etag); Assert.Null(actual.IfMatchETag); Assert.Equal(etag, actual.IfNoneMatchETag); }
public async Task UpdatesSearchDocumentsWithVersionMatchingAllFilters() { VersionListDataResult = new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData> { { "1.0.0", new VersionPropertiesData(listed: true, semVer2: false) }, }), AccessConditionWrapper.GenerateIfNotExistsCondition()); var indexActions = await Target.UpdateAsync(Data.PackageId, BuildDocument); Assert.Same(VersionListDataResult, indexActions.VersionListDataResult); Assert.Empty(indexActions.Hijack); Assert.Equal(4, indexActions.Search.Count); Assert.All(indexActions.Search, x => Assert.IsType <SearchDocument.UpdateOwners>(x.Document)); Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.Merge, x.ActionType)); Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.Default.ToString()); Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.IncludePrerelease.ToString()); Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.IncludeSemVer2.ToString()); Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.IncludePrereleaseAndSemVer2.ToString()); }
public void GenerateIfNotExistsCondition() { var actual = AccessConditionWrapper.GenerateIfNotExistsCondition(); Assert.Null(actual.IfMatchETag); Assert.Equal("*", actual.IfNoneMatchETag); }
public async Task <ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > > > ReadLatestIndexedAsync() { var stopwatch = Stopwatch.StartNew(); var blobName = GetLatestIndexedBlobName(); var blobReference = Container.GetBlobReference(blobName); _logger.LogInformation("Reading the latest indexed owners from {BlobName}.", blobName); var builder = new PackageIdToOwnersBuilder(_logger); IAccessCondition accessCondition; try { using (var stream = await blobReference.OpenReadAsync(AccessCondition.GenerateEmptyCondition())) { accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blobReference.ETag); ReadStream(stream, builder.Add); } } catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound) { accessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition(); _logger.LogInformation("The blob {BlobName} does not exist.", blobName); } var output = new ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > >( builder.GetResult(), accessCondition); stopwatch.Stop(); _telemetryService.TrackReadLatestIndexedOwners(output.Result.Count, stopwatch.Elapsed); return(output); }
public void GenerateEmptyCondition() { var actual = AccessConditionWrapper.GenerateEmptyCondition(); Assert.Null(actual.IfMatchETag); Assert.Null(actual.IfNoneMatchETag); }
public async Task <ResultAndAccessCondition <VersionListData> > ReadAsync(string id) { var blobReference = Container.GetBlobReference(GetFileName(id)); _logger.LogInformation("Reading the version list for package ID {PackageId}.", id); VersionListData data; IAccessCondition accessCondition; try { using (var stream = await blobReference.OpenReadAsync(AccessCondition.GenerateEmptyCondition())) using (var streamReader = new StreamReader(stream)) using (var jsonTextReader = new JsonTextReader(streamReader)) { data = Serializer.Deserialize <VersionListData>(jsonTextReader); } accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blobReference.ETag); } catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound) { data = new VersionListData(new Dictionary <string, VersionPropertiesData>()); accessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition(); } return(new ResultAndAccessCondition <VersionListData>(data, accessCondition)); }
private async Task <AuxiliaryFileResult <T> > LoadAsync <T>( AuxiliaryFileResult <T> previousResult, Func <IAccessCondition, StringCache, Task <AuxiliaryFileResult <T> > > getResult) where T : class { await Task.Yield(); IAccessCondition accessCondition; if (previousResult == null) { accessCondition = AccessConditionWrapper.GenerateEmptyCondition(); } else { accessCondition = AccessConditionWrapper.GenerateIfNoneMatchCondition(previousResult.Metadata.ETag); } var newResult = await getResult(accessCondition, _stringCache); if (newResult.Modified) { return(newResult); } else { return(previousResult); } }
private async Task WriteDownloadDataAsync(DownloadData downloadData) { _logger.LogInformation("Writing the initial download data file."); await _downloadDataClient.ReplaceLatestIndexedAsync( downloadData, AccessConditionWrapper.GenerateIfNotExistsCondition()); _logger.LogInformation("Done uploading the initial download data file."); }
private async Task WriteVerifiedPackagesDataAsync(HashSet <string> verifiedPackages) { _logger.LogInformation("Writing the initial verified packages data file."); await _verifiedPackagesDataClient.ReplaceLatestAsync( verifiedPackages, AccessConditionWrapper.GenerateIfNotExistsCondition()); _logger.LogInformation("Done uploading the initial verified packages data file."); }
private async Task WritePopularityTransfersDataAsync(PopularityTransferData popularityTransfers) { _logger.LogInformation("Writing the initial popularity transfers data file."); await _popularityTransferDataClient.ReplaceLatestIndexedAsync( popularityTransfers, AccessConditionWrapper.GenerateIfNotExistsCondition()); _logger.LogInformation("Done uploading the initial popularity transfers data file."); }
public async Task DoesNotEnqueueChangesForNoIndexActions() { _config.AzureSearchBatchSize = 2; _producer .Setup(x => x.ProduceWorkAsync(It.IsAny <ConcurrentBag <NewPackageRegistration> >(), It.IsAny <CancellationToken>())) .ReturnsAsync(() => _initialAuxiliaryData) .Callback <ConcurrentBag <NewPackageRegistration>, CancellationToken>((w, _) => { w.Add(new NewPackageRegistration("A", 0, new[] { "Microsoft", "EntityFramework" }, new Package[0], false)); w.Add(new NewPackageRegistration("B", 0, new[] { "nuget" }, new Package[0], false)); w.Add(new NewPackageRegistration("C", 0, new[] { "aspnet" }, new Package[0], false)); }); // Return empty index action for ID "B". This package ID will not be pushed to Azure Search but will appear // in the initial owners data file. _builder .Setup(x => x.AddNewPackageRegistration(It.Is <NewPackageRegistration>(y => y.PackageId != "B"))) .Returns <NewPackageRegistration>(x => new IndexActions( new List <IndexAction <KeyedDocument> > { IndexAction.Upload(new KeyedDocument { Key = x.PackageId }) }, new List <IndexAction <KeyedDocument> >(), new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), AccessConditionWrapper.GenerateEmptyCondition()))); _builder .Setup(x => x.AddNewPackageRegistration(It.Is <NewPackageRegistration>(y => y.PackageId == "B"))) .Returns <NewPackageRegistration>(x => new IndexActions( new List <IndexAction <KeyedDocument> >(), new List <IndexAction <KeyedDocument> >(), new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), AccessConditionWrapper.GenerateEmptyCondition()))); var enqueuedIndexActions = new List <KeyValuePair <string, IndexActions> >(); _batchPusher .Setup(x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>())) .Callback <string, IndexActions>((id, actions) => { enqueuedIndexActions.Add(KeyValuePair.Create(id, actions)); }); await _target.ExecuteAsync(); Assert.Equal(2, enqueuedIndexActions.Count); var keys = enqueuedIndexActions .Select(x => x.Key) .OrderBy(x => x) .ToArray(); Assert.Equal( new[] { "A", "C" }, keys); }
private async Task WriteOwnerDataAsync(SortedDictionary <string, SortedSet <string> > owners) { _logger.LogInformation("Writing the initial owners file."); await _ownerDataClient.ReplaceLatestIndexedAsync( owners, AccessConditionWrapper.GenerateIfNotExistsCondition()); _logger.LogInformation("Done uploading the initial owners file."); }
public Task <string> CopyPackageFileForValidationSetAsync(PackageValidationSet validationSet) { var srcFileName = BuildFileName(validationSet, _fileMetadataService.FileSavePathTemplate, _fileMetadataService.FileExtension); return(CopyFileAsync( _fileMetadataService.FileFolderName, srcFileName, _fileMetadataService.ValidationFolderName, BuildValidationSetPackageFileName(validationSet, _fileMetadataService.FileExtension), AccessConditionWrapper.GenerateEmptyCondition())); }
public async Task UpdatesSearchDocumentsWithNoVersions() { VersionListDataResult = new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), AccessConditionWrapper.GenerateIfNotExistsCondition()); var indexActions = await Target.UpdateAsync(Data.PackageId, BuildDocument); Assert.Same(VersionListDataResult, indexActions.VersionListDataResult); Assert.Empty(indexActions.Hijack); Assert.Empty(indexActions.Search); }
private async Task <bool> PushIndexChangesAsync() { // The "old" data in this case is the download count data that was last indexed by this job (or // initialized by Db2AzureSearch). _logger.LogInformation("Fetching old download count data from blob storage."); var oldResult = await _downloadDataClient.ReadLatestIndexedAsync( AccessConditionWrapper.GenerateEmptyCondition(), _stringCache); // The "new" data in this case is from the statistics pipeline. _logger.LogInformation("Fetching new download count data from blob storage."); var newData = await _auxiliaryFileClient.LoadDownloadDataAsync(); _logger.LogInformation("Removing invalid IDs and versions from the old data."); CleanDownloadData(oldResult.Data); _logger.LogInformation("Removing invalid IDs and versions from the new data."); CleanDownloadData(newData); // Fetch the download overrides from the auxiliary file. Note that the overriden downloads are kept // separate from downloads data as the original data will be persisted to auxiliary data, whereas the // overriden data will be persisted to Azure Search. _logger.LogInformation("Overriding download count data."); var downloadOverrides = await _auxiliaryFileClient.LoadDownloadOverridesAsync(); var overridenDownloads = newData.ApplyDownloadOverrides(downloadOverrides, _logger); _logger.LogInformation("Detecting download count changes."); var changes = _downloadSetComparer.Compare(oldResult.Data, overridenDownloads); var idBag = new ConcurrentBag <string>(changes.Keys); _logger.LogInformation("{Count} package IDs have download count changes.", idBag.Count); if (!changes.Any()) { return(false); } _logger.LogInformation( "Starting {Count} workers pushing download count changes to Azure Search.", _options.Value.MaxConcurrentBatches); await ParallelAsync.Repeat( () => WorkAsync(idBag, changes), _options.Value.MaxConcurrentBatches); _logger.LogInformation("All of the download count changes have been pushed to Azure Search."); _logger.LogInformation("Uploading the new download count data to blob storage."); await _downloadDataClient.ReplaceLatestIndexedAsync(newData, oldResult.Metadata.GetIfMatchCondition()); return(true); }
public virtual Task CopyValidationPackageToPackageFileAsync(PackageValidationSet validationSet) { var fileName = BuildFileName(validationSet, _fileMetadataService.FileSavePathTemplate, _fileMetadataService.FileExtension); return(CopyFileAsync( _fileMetadataService.ValidationFolderName, fileName, _fileMetadataService.FileFolderName, fileName, AccessConditionWrapper.GenerateIfNotExistsCondition())); }
public async Task PushesToIndexesUsingMaximumBatchSize() { _config.AzureSearchBatchSize = 2; _producer .Setup(x => x.ProduceWorkAsync(It.IsAny <ConcurrentBag <NewPackageRegistration> >(), It.IsAny <CancellationToken>())) .ReturnsAsync(() => _initialAuxiliaryData) .Callback <ConcurrentBag <NewPackageRegistration>, CancellationToken>((w, _) => { w.Add(new NewPackageRegistration("A", 0, new string[0], new Package[0], false)); w.Add(new NewPackageRegistration("B", 0, new string[0], new Package[0], false)); w.Add(new NewPackageRegistration("C", 0, new string[0], new Package[0], false)); w.Add(new NewPackageRegistration("D", 0, new string[0], new Package[0], false)); w.Add(new NewPackageRegistration("E", 0, new string[0], new Package[0], false)); }); _builder .Setup(x => x.AddNewPackageRegistration(It.IsAny <NewPackageRegistration>())) .Returns <NewPackageRegistration>(x => new IndexActions( new List <IndexAction <KeyedDocument> > { IndexAction.Upload(new KeyedDocument { Key = x.PackageId }) }, new List <IndexAction <KeyedDocument> >(), new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), AccessConditionWrapper.GenerateEmptyCondition()))); var enqueuedIndexActions = new List <KeyValuePair <string, IndexActions> >(); _batchPusher .Setup(x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>())) .Callback <string, IndexActions>((id, actions) => { enqueuedIndexActions.Add(KeyValuePair.Create(id, actions)); }); await _target.ExecuteAsync(); Assert.Equal(5, enqueuedIndexActions.Count); var keys = enqueuedIndexActions .Select(x => x.Key) .OrderBy(x => x) .ToArray(); Assert.Equal( new[] { "A", "B", "C", "D", "E" }, keys); _batchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Exactly(5)); _batchPusher.Verify(x => x.TryFinishAsync(), Times.Once); }
public Task <string> CopyPackageFileForValidationSetAsync(PackageValidationSet validationSet) { var srcFileName = BuildFileName( validationSet.PackageId, validationSet.PackageNormalizedVersion, CoreConstants.PackageFileSavePathTemplate, CoreConstants.NuGetPackageFileExtension); return(CopyFileAsync( CoreConstants.PackagesFolderName, srcFileName, CoreConstants.ValidationFolderName, BuildValidationSetPackageFileName(validationSet), AccessConditionWrapper.GenerateEmptyCondition())); }
public Task CopyValidationPackageToPackageFileAsync(string id, string normalizedVersion) { var fileName = BuildFileName( id, normalizedVersion, CoreConstants.PackageFileSavePathTemplate, CoreConstants.NuGetPackageFileExtension); return(CopyFileAsync( CoreConstants.ValidationFolderName, fileName, CoreConstants.PackagesFolderName, fileName, AccessConditionWrapper.GenerateIfNotExistsCondition())); }
public Task CopyPackageUrlForValidationSetAsync(PackageValidationSet validationSet, string srcPackageUrl) { var destFileName = BuildValidationSetPackageFileName(validationSet, _fileMetadataService.FileExtension); _logger.LogInformation( "Copying URL {SrcPackageUrl} to {DestFolderName}/{DestFileName}.", srcPackageUrl, _fileMetadataService.ValidationFolderName, srcPackageUrl); return(_fileStorageService.CopyFileAsync( new Uri(srcPackageUrl), _fileMetadataService.ValidationFolderName, destFileName, AccessConditionWrapper.GenerateEmptyCondition())); }
public void RejectsEmptyEnqueue() { var emptyIndexActions = new IndexActions( new List <IndexAction <KeyedDocument> >(), new List <IndexAction <KeyedDocument> >(), new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), AccessConditionWrapper.GenerateEmptyCondition())); var ex = Assert.Throws <ArgumentException>( () => _target.EnqueueIndexActions(IdA, emptyIndexActions)); Assert.Contains("There must be at least one index action.", ex.Message); Assert.Empty(_target._searchActions); Assert.Empty(_target._hijackActions); Assert.Empty(_target._idReferenceCount); Assert.Empty(_target._versionListDataResults); }
public async Task CopyValidationSetPackageToPackageFileAsync() { var accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(_etag); _fileStorageService .Setup(x => x.CopyFileAsync( _validationContainerName, _validationSetPackageFileName, _packagesContainerName, _packageFileName, accessCondition)) .ReturnsAsync(_etag) .Verifiable(); await _target.CopyValidationSetPackageToPackageFileAsync(_validationSet, accessCondition); _fileStorageService.Verify(); }
public IndexActions AddNewPackageRegistration(NewPackageRegistration packageRegistration) { var versionProperties = new Dictionary <string, VersionPropertiesData>(); var versionListData = new VersionListData(versionProperties); var versionLists = new VersionLists(versionListData); var changes = packageRegistration .Packages .Select(GetVersionListChange) .ToList(); var indexChanges = versionLists.ApplyChanges(changes); var versionToPackage = packageRegistration .Packages .ToDictionary(p => NuGetVersion.Parse(p.Version)); var search = indexChanges .Search .Select(p => GetSearchIndexAction( packageRegistration, versionToPackage, versionLists, p.Key, p.Value)) .ToList(); var hijack = indexChanges .Hijack .Select(p => GetHijackIndexAction( packageRegistration.PackageId, versionToPackage[p.Key], p.Value)) .ToList(); return(new IndexActions( search, hijack, new ResultAndAccessCondition <VersionListData>( versionLists.GetVersionListData(), AccessConditionWrapper.GenerateEmptyCondition()))); }
public Facts(ITestOutputHelper output) { VersionListDataClient = new Mock <IVersionListDataClient>(); Search = new Mock <ISearchDocumentBuilder>(); Logger = output.GetLogger <SearchIndexActionBuilder>(); VersionListDataResult = new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), AccessConditionWrapper.GenerateIfNotExistsCondition()); VersionListDataClient .Setup(x => x.ReadAsync(It.IsAny <string>())) .ReturnsAsync(() => VersionListDataResult); Search .Setup(x => x.UpdateOwners(It.IsAny <string>(), It.IsAny <SearchFilters>(), It.IsAny <string[]>())) .Returns <string, SearchFilters, string[]>((_, sf, __) => new SearchDocument.UpdateOwners { Key = sf.ToString(), }); Target = new SearchIndexActionBuilder(VersionListDataClient.Object, Logger); }
public async Task <FeatureFlagSaveResult> TrySaveAsync(FeatureFlags flags, string contentId) { var accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(contentId); try { using (var stream = new MemoryStream()) using (var writer = new StreamWriter(stream)) using (var jsonWriter = new JsonTextWriter(writer)) { Serializer.Serialize(jsonWriter, flags); jsonWriter.Flush(); stream.Position = 0; await _storage.SaveFileAsync(CoreConstants.Folders.ContentFolderName, CoreConstants.FeatureFlagsFileName, stream, accessCondition); return(FeatureFlagSaveResult.Ok); } } catch (StorageException e) when(e.IsPreconditionFailedException()) { return(FeatureFlagSaveResult.Conflict); } }
private async Task <bool> UpdatePublicPackageAsync(PackageValidationSet validationSet) { _logger.LogInformation("Copying .nupkg to public storage for package {PackageId} {PackageVersion}, validation set {ValidationSetId}", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId); // If the validation set contains any processors, we must use the copy of the package that is specific to // this validation set. We can't use the original validation package because it does not have any of the // changes that the processors made. If the validation set package does not exist for some reason and there // are processors in the validation set, this indicates a bug and an exception will be thrown by the copy // operation below. This will cause the validation queue message to eventually dead-letter at which point // the on-call person should investigate. bool copied; if (validationSet.PackageValidations.Any(x => _validatorProvider.IsProcessor(x.Type)) || await _packageFileService.DoesValidationSetPackageExistAsync(validationSet)) { IAccessCondition destAccessCondition; // The package etag will be null if this validation set is expecting the package to not yet exist in // the packages container. if (validationSet.PackageETag == null) { // This will fail with HTTP 409 if the package already exists. This means that another validation // set has completed and moved the package into the Available state first, with different package // content. destAccessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition(); _logger.LogInformation( "Attempting to copy validation set {ValidationSetId} package {PackageId} {PackageVersion} to" + " the packages container, assuming that the package does not already exist.", validationSet.ValidationTrackingId, validationSet.PackageId, validationSet.PackageNormalizedVersion); } else { // This will fail with HTTP 412 if the package has been modified by another validation set. This // would only happen if this validation set and another validation set are operating on a package // already in the Available state. destAccessCondition = AccessConditionWrapper.GenerateIfMatchCondition(validationSet.PackageETag); _logger.LogInformation( "Attempting to copy validation set {ValidationSetId} package {PackageId} {PackageVersion} to" + " the packages container, assuming that the package has etag {PackageETag}.", validationSet.ValidationTrackingId, validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.PackageETag); } // Failures here should result in an unhandled exception. This means that this validation set has // modified the package but is unable to copy the modified package into the packages container because // another validation set completed first. await _packageFileService.CopyValidationSetPackageToPackageFileAsync( validationSet, destAccessCondition); copied = true; } else { _logger.LogInformation( "The package specific to the validation set does not exist. Falling back to the validation " + "container for package {PackageId} {PackageVersion}, validation set {ValidationSetId}", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId); try { await _packageFileService.CopyValidationPackageToPackageFileAsync(validationSet); copied = true; } catch (InvalidOperationException) { // The package already exists in the packages container. This can happen if the DB commit below fails // and this flow is retried or another validation set for the package completed first. Either way, we // will later attempt to use the hash from the package in the packages container (the destination). // In other words, we don't care which copy wins when copying from the validation package because // we know the package has not been modified. _logger.LogInformation( "Package already exists in packages container for {PackageId} {PackageVersion}, validation set {ValidationSetId}", validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId); copied = false; } } return(copied); }
public Db2AzureSearchCommandFacts(ITestOutputHelper output) { _producer = new Mock <INewPackageRegistrationProducer>(); _builder = new Mock <IPackageEntityIndexActionBuilder>(); _blobContainerBuilder = new Mock <IBlobContainerBuilder>(); _indexBuilder = new Mock <IIndexBuilder>(); _batchPusher = new Mock <IBatchPusher>(); _catalogClient = new Mock <ICatalogClient>(); _storageFactory = new Mock <IStorageFactory>(); _ownerDataClient = new Mock <IOwnerDataClient>(); _downloadDataClient = new Mock <IDownloadDataClient>(); _verifiedPackagesDataClient = new Mock <IVerifiedPackagesDataClient>(); _popularityTransferDataClient = new Mock <IPopularityTransferDataClient>(); _options = new Mock <IOptionsSnapshot <Db2AzureSearchConfiguration> >(); _developmentOptions = new Mock <IOptionsSnapshot <Db2AzureSearchDevelopmentConfiguration> >(); _logger = output.GetLogger <Db2AzureSearchCommand>(); _config = new Db2AzureSearchConfiguration { MaxConcurrentBatches = 1, StorageContainer = "container-name", }; _developmentConfig = new Db2AzureSearchDevelopmentConfiguration(); _storage = new TestCursorStorage(new Uri("https://example/base/")); _initialAuxiliaryData = new InitialAuxiliaryData( owners: new SortedDictionary <string, SortedSet <string> >(), downloads: new DownloadData(), excludedPackages: new HashSet <string>(), verifiedPackages: new HashSet <string>(), popularityTransfers: new PopularityTransferData()); _options .Setup(x => x.Value) .Returns(() => _config); _developmentOptions .Setup(x => x.Value) .Returns(() => _developmentConfig); _producer .Setup(x => x.ProduceWorkAsync(It.IsAny <ConcurrentBag <NewPackageRegistration> >(), It.IsAny <CancellationToken>())) .ReturnsAsync(() => _initialAuxiliaryData); _builder .Setup(x => x.AddNewPackageRegistration(It.IsAny <NewPackageRegistration>())) .Returns(() => new IndexActions( new IndexAction <KeyedDocument> [0], new IndexAction <KeyedDocument> [0], new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), AccessConditionWrapper.GenerateEmptyCondition()))); _batchPusher.SetReturnsDefault(Task.FromResult(new BatchPusherResult())); _catalogClient .Setup(x => x.GetIndexAsync(It.IsAny <string>())) .ReturnsAsync(new CatalogIndex()); _storageFactory .Setup(x => x.Create(It.IsAny <string>())) .Returns(() => _storage); _blobContainerBuilder .Setup(x => x.DeleteIfExistsAsync()) .ReturnsAsync(true); _target = new Db2AzureSearchCommand( _producer.Object, _builder.Object, _blobContainerBuilder.Object, _indexBuilder.Object, () => _batchPusher.Object, _catalogClient.Object, _storageFactory.Object, _ownerDataClient.Object, _downloadDataClient.Object, _verifiedPackagesDataClient.Object, _popularityTransferDataClient.Object, _options.Object, _developmentOptions.Object, _logger); }
private async Task <bool> PushIndexChangesAsync() { // The "old" data in this case is the download count data that was last indexed by this job (or // initialized by Db2AzureSearch). _logger.LogInformation("Fetching old download count data from blob storage."); var oldResult = await _downloadDataClient.ReadLatestIndexedAsync( AccessConditionWrapper.GenerateEmptyCondition(), _stringCache); // The "new" data in this case is from the statistics pipeline. _logger.LogInformation("Fetching new download count data from blob storage."); var newData = await _auxiliaryFileClient.LoadDownloadDataAsync(); _logger.LogInformation("Removing invalid IDs and versions from the old downloads data."); CleanDownloadData(oldResult.Data); _logger.LogInformation("Removing invalid IDs and versions from the new downloads data."); CleanDownloadData(newData); _logger.LogInformation("Detecting download count changes."); var changes = _downloadSetComparer.Compare(oldResult.Data, newData); _logger.LogInformation("{Count} package IDs have download count changes.", changes.Count); // The "old" data is the popularity transfers data that was last indexed by this job (or // initialized by Db2AzureSearch). _logger.LogInformation("Fetching old popularity transfer data from blob storage."); var oldTransfers = await _popularityTransferDataClient.ReadLatestIndexedAsync( AccessConditionWrapper.GenerateEmptyCondition(), _stringCache); // The "new" data is the latest popularity transfers data from the database. _logger.LogInformation("Fetching new popularity transfer data from database."); var newTransfers = await GetPopularityTransfersAsync(); _logger.LogInformation("Applying download transfers to download changes."); ApplyDownloadTransfers( newData, oldTransfers.Data, newTransfers, changes); var idBag = new ConcurrentBag <string>(changes.Keys); _logger.LogInformation("{Count} package IDs need to be updated.", idBag.Count); if (!changes.Any()) { return(false); } _logger.LogInformation( "Starting {Count} workers pushing download count changes to Azure Search.", _options.Value.MaxConcurrentBatches); await ParallelAsync.Repeat( () => WorkAndRetryAsync(idBag, changes), _options.Value.MaxConcurrentBatches); _logger.LogInformation("All of the download count changes have been pushed to Azure Search."); _logger.LogInformation("Uploading the new download count data to blob storage."); await _downloadDataClient.ReplaceLatestIndexedAsync(newData, oldResult.Metadata.GetIfMatchCondition()); _logger.LogInformation("Uploading the new popularity transfer data to blob storage."); await _popularityTransferDataClient.ReplaceLatestIndexedAsync( newTransfers, oldTransfers.Metadata.GetIfMatchCondition()); return(true); }
public BaseFacts(ITestOutputHelper output) { _logger = output.GetLogger <BatchPusher>(); _searchIndexClientWrapper = new Mock <ISearchIndexClientWrapper>(); _searchDocumentsWrapper = new Mock <IDocumentsOperationsWrapper>(); _hijackIndexClientWrapper = new Mock <ISearchIndexClientWrapper>(); _hijackDocumentsWrapper = new Mock <IDocumentsOperationsWrapper>(); _versionListDataClient = new Mock <IVersionListDataClient>(); _config = new AzureSearchJobConfiguration(); _developmentConfig = new AzureSearchJobDevelopmentConfiguration(); _options = new Mock <IOptionsSnapshot <AzureSearchJobConfiguration> >(); _developmentOptions = new Mock <IOptionsSnapshot <AzureSearchJobDevelopmentConfiguration> >(); _telemetryService = new Mock <IAzureSearchTelemetryService>(); _searchIndexClientWrapper.Setup(x => x.IndexName).Returns("search"); _searchIndexClientWrapper.Setup(x => x.Documents).Returns(() => _searchDocumentsWrapper.Object); _hijackIndexClientWrapper.Setup(x => x.IndexName).Returns("hijack"); _hijackIndexClientWrapper.Setup(x => x.Documents).Returns(() => _hijackDocumentsWrapper.Object); _versionListDataClient .Setup(x => x.TryReplaceAsync(It.IsAny <string>(), It.IsAny <VersionListData>(), It.IsAny <IAccessCondition>())) .ReturnsAsync(true); _options.Setup(x => x.Value).Returns(() => _config); _developmentOptions.Setup(x => x.Value).Returns(() => _developmentConfig); _searchBatches = new List <IndexBatch <KeyedDocument> >(); _hijackBatches = new List <IndexBatch <KeyedDocument> >(); _searchDocumentsWrapper .Setup(x => x.IndexAsync(It.IsAny <IndexBatch <KeyedDocument> >())) .ReturnsAsync(() => new DocumentIndexResult(new List <IndexingResult>())) .Callback <IndexBatch <KeyedDocument> >(b => _searchBatches.Add(b)); _hijackDocumentsWrapper .Setup(x => x.IndexAsync(It.IsAny <IndexBatch <KeyedDocument> >())) .ReturnsAsync(() => new DocumentIndexResult(new List <IndexingResult>())) .Callback <IndexBatch <KeyedDocument> >(b => _hijackBatches.Add(b)); _config.AzureSearchBatchSize = 2; _config.MaxConcurrentVersionListWriters = 1; _searchDocumentA = IndexAction.Upload(new KeyedDocument()); _searchDocumentB = IndexAction.Upload(new KeyedDocument()); _searchDocumentC = IndexAction.Upload(new KeyedDocument()); _searchDocuments = new List <IndexAction <KeyedDocument> > { _searchDocumentA, _searchDocumentB, _searchDocumentC, }; _hijackDocumentA = IndexAction.Upload(new KeyedDocument()); _hijackDocumentB = IndexAction.Upload(new KeyedDocument()); _hijackDocumentC = IndexAction.Upload(new KeyedDocument()); _hijackDocumentD = IndexAction.Upload(new KeyedDocument()); _hijackDocumentE = IndexAction.Upload(new KeyedDocument()); _hijackDocuments = new List <IndexAction <KeyedDocument> > { _hijackDocumentA, _hijackDocumentB, _hijackDocumentC, _hijackDocumentD, _hijackDocumentE, }; _indexActions = new IndexActions( _searchDocuments, _hijackDocuments, new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), AccessConditionWrapper.GenerateEmptyCondition())); _target = new BatchPusher( _searchIndexClientWrapper.Object, _hijackIndexClientWrapper.Object, _versionListDataClient.Object, _options.Object, _developmentOptions.Object, _telemetryService.Object, _logger); }