public async Task EventuallyFailsIfBatchPusherNeverSucceeds() { Config.MaxConcurrentVersionListWriters = 1; Changes["PackageA"] = 1; Changes["PackageB"] = 2; BatchPusher .Setup(x => x.TryFinishAsync()) .ReturnsAsync(new BatchPusherResult(new[] { "PackageB" })); var ex = await Assert.ThrowsAsync <InvalidOperationException>(() => Target.ExecuteAsync()); Assert.Equal("The index operations for the following package IDs failed due to version list concurrency: PackageB", ex.Message); VerifyCompletedTelemetry(JobOutcome.Failure); VerifyAllIdsAreProcessed(new[] { "PackageA", "PackageB", "PackageB", "PackageB" }); IndexActionBuilder.Verify( x => x.UpdateAsync( "PackageA", It.IsAny <Func <SearchFilters, KeyedDocument> >()), Times.Once); IndexActionBuilder.Verify( x => x.UpdateAsync( "PackageB", It.IsAny <Func <SearchFilters, KeyedDocument> >()), Times.Exactly(3)); BatchPusher.Verify( x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()), Times.Exactly(4)); BatchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(3)); BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never); }
public async Task RetriesFailedPackageIds() { Config.MaxConcurrentVersionListWriters = 1; Changes["PackageA"] = 1; Changes["PackageB"] = 2; BatchPusher .SetupSequence(x => x.TryFinishAsync()) .ReturnsAsync(new BatchPusherResult(new[] { "PackageB" })) .ReturnsAsync(new BatchPusherResult()); await Target.ExecuteAsync(); VerifyCompletedTelemetry(JobOutcome.Success); VerifyAllIdsAreProcessed(new[] { "PackageA", "PackageB", "PackageB" }); IndexActionBuilder.Verify( x => x.UpdateAsync( "PackageA", It.IsAny <Func <SearchFilters, KeyedDocument> >()), Times.Once); IndexActionBuilder.Verify( x => x.UpdateAsync( "PackageB", It.IsAny <Func <SearchFilters, KeyedDocument> >()), Times.Exactly(2)); BatchPusher.Verify( x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()), Times.Exactly(3)); BatchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(2)); BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never); }
[InlineData(4, 8, 15, 0)] // 4, 8 + 4 = 12 is greater than 10 so 8 is the batch size. public async Task RespectsAzureSearchBatchSize(int documentsPerId, int batchSize, int fullPushes, int partialPushes) { var changeCount = 30; var expectedPushes = fullPushes + partialPushes; Config.AzureSearchBatchSize = 10; IndexActions = new IndexActions( new List <IndexAction <KeyedDocument> >( Enumerable .Range(0, documentsPerId) .Select(x => IndexAction.Merge(new KeyedDocument()))), new List <IndexAction <KeyedDocument> >(), new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), new Mock <IAccessCondition>().Object)); AddChanges(changeCount); await Target.ExecuteAsync(); VerifyCompletedTelemetry(JobOutcome.Success); VerifyAllIdsAreProcessed(changeCount); IndexActionBuilder.Verify( x => x.UpdateAsync( It.IsAny <string>(), It.IsAny <Func <SearchFilters, KeyedDocument> >()), Times.Exactly(changeCount)); BatchPusher.Verify( x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()), Times.Exactly(changeCount)); BatchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(expectedPushes)); BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never); SystemTime.Verify(x => x.Delay(It.IsAny <TimeSpan>()), Times.Exactly(expectedPushes - 1)); DownloadDataClient.Verify( x => x.ReplaceLatestIndexedAsync( NewDownloadData, It.Is <IAccessCondition>(a => a.IfMatchETag == OldDownloadResult.Metadata.ETag)), Times.Once); Assert.Equal( fullPushes, FinishedBatches.Count(b => b.Sum(ia => ia.Search.Count) == batchSize)); Assert.Equal( partialPushes, FinishedBatches.Count(b => b.Sum(ia => ia.Search.Count) != batchSize)); Assert.Empty(CurrentBatch); }
public async Task PushesNothingWhenThereAreNoChanges() { await Target.ExecuteAsync(); VerifyCompletedTelemetry(JobOutcome.NoOp); VerifyAllIdsAreProcessed(changeCount: 0); IndexActionBuilder.Verify( x => x.UpdateAsync( It.IsAny <string>(), It.IsAny <Func <SearchFilters, KeyedDocument> >()), Times.Never); BatchPusher.Verify(x => x.FinishAsync(), Times.Never); BatchPusher.Verify(x => x.PushFullBatchesAsync(), Times.Never); DownloadDataClient.Verify( x => x.ReplaceLatestIndexedAsync(It.IsAny <DownloadData>(), It.IsAny <IAccessCondition>()), Times.Never); }
public async Task CanProcessInParallel() { var changeCount = 1000; Config.AzureSearchBatchSize = 5; Config.MaxConcurrentBatches = 4; Config.MaxConcurrentVersionListWriters = 8; AddChanges(changeCount); await Target.ExecuteAsync(); VerifyCompletedTelemetry(JobOutcome.Success); VerifyAllIdsAreProcessed(changeCount); IndexActionBuilder.Verify( x => x.UpdateAsync( It.IsAny <string>(), It.IsAny <Func <SearchFilters, KeyedDocument> >()), Times.Exactly(changeCount)); BatchPusher.Verify( x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()), Times.Exactly(changeCount)); BatchPusher.Verify(x => x.TryFinishAsync(), Times.AtLeastOnce); BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never); }
public PopularityTransferIntegrationTests(ITestOutputHelper output) { _featureFlags = new Mock <IFeatureFlagService>(); _telemetry = new Mock <IAzureSearchTelemetryService>(); _config = new Auxiliary2AzureSearchConfiguration { AuxiliaryDataStorageContainer = "auxiliary-container", EnablePopularityTransfers = true, StorageContainer = "storage-container", Scoring = new AzureSearchScoringConfiguration() }; var options = new Mock <IOptionsSnapshot <Auxiliary2AzureSearchConfiguration> >(); options .Setup(x => x.Value) .Returns(_config); _developmentConfig = new AzureSearchJobDevelopmentConfiguration(); var developmentOptions = new Mock <IOptionsSnapshot <AzureSearchJobDevelopmentConfiguration> >(); developmentOptions .Setup(x => x.Value) .Returns(_developmentConfig); var auxiliaryConfig = new AuxiliaryDataStorageConfiguration { AuxiliaryDataStorageContainer = "auxiliary-container", AuxiliaryDataStorageDownloadsPath = "downloads.json", AuxiliaryDataStorageExcludedPackagesPath = "excludedPackages.json", }; var auxiliaryOptions = new Mock <IOptionsSnapshot <AuxiliaryDataStorageConfiguration> >(); auxiliaryOptions .Setup(x => x.Value) .Returns(auxiliaryConfig); _auxilliaryContainer = new InMemoryCloudBlobContainer(); _storageContainer = new InMemoryCloudBlobContainer(); _blobClient = new InMemoryCloudBlobClient(); _blobClient.Containers["auxiliary-container"] = _auxilliaryContainer; _blobClient.Containers["storage-container"] = _storageContainer; var auxiliaryFileClient = new AuxiliaryFileClient( _blobClient, auxiliaryOptions.Object, _telemetry.Object, output.GetLogger <AuxiliaryFileClient>()); _newPopularityTransfers = new PopularityTransferData(); var databaseFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>(); databaseFetcher .Setup(x => x.GetPopularityTransfersAsync()) .ReturnsAsync(_newPopularityTransfers); var downloadDataClient = new DownloadDataClient( _blobClient, options.Object, _telemetry.Object, output.GetLogger <DownloadDataClient>()); var popularityTransferDataClient = new PopularityTransferDataClient( _blobClient, options.Object, _telemetry.Object, output.GetLogger <PopularityTransferDataClient>()); var versionListDataClient = new VersionListDataClient( _blobClient, options.Object, output.GetLogger <VersionListDataClient>()); var downloadComparer = new DownloadSetComparer( _telemetry.Object, options.Object, output.GetLogger <DownloadSetComparer>()); var dataComparer = new DataSetComparer( _telemetry.Object, output.GetLogger <DataSetComparer>()); var downloadTransferrer = new DownloadTransferrer( dataComparer, options.Object, output.GetLogger <DownloadTransferrer>()); var baseDocumentBuilder = new BaseDocumentBuilder(options.Object); var searchDocumentBuilder = new SearchDocumentBuilder(baseDocumentBuilder); var searchIndexActionBuilder = new SearchIndexActionBuilder( versionListDataClient, output.GetLogger <SearchIndexActionBuilder>()); _searchOperations = new Mock <IDocumentsOperationsWrapper>(); _searchOperations .Setup(x => x.IndexAsync(It.IsAny <IndexBatch <KeyedDocument> >())) .Callback <IndexBatch <KeyedDocument> >(batch => { _indexedBatch = batch; }) .ReturnsAsync(new DocumentIndexResult()); var hijackIndexClient = new Mock <ISearchIndexClientWrapper>(); var searchIndexClient = new Mock <ISearchIndexClientWrapper>(); searchIndexClient .Setup(x => x.Documents) .Returns(_searchOperations.Object); var batchPusher = new BatchPusher( searchIndexClient.Object, hijackIndexClient.Object, versionListDataClient, options.Object, developmentOptions.Object, _telemetry.Object, output.GetLogger <BatchPusher>()); Func <IBatchPusher> batchPusherFactory = () => batchPusher; var time = new Mock <ISystemTime>(); _featureFlags.Setup(x => x.IsPopularityTransferEnabled()).Returns(true); _target = new UpdateDownloadsCommand( auxiliaryFileClient, databaseFetcher.Object, downloadDataClient, downloadComparer, downloadTransferrer, popularityTransferDataClient, searchDocumentBuilder, searchIndexActionBuilder, batchPusherFactory, time.Object, _featureFlags.Object, options.Object, _telemetry.Object, output.GetLogger <Auxiliary2AzureSearchCommand>()); }