public async Task EventuallyFailsIfBatchPusherNeverSucceeds()
{
Config.MaxConcurrentVersionListWriters = 1;
Changes["PackageA"] = 1;
Changes["PackageB"] = 2;
BatchPusher
.Setup(x => x.TryFinishAsync())
.ReturnsAsync(new BatchPusherResult(new[] { "PackageB" }));
var ex = await Assert.ThrowsAsync <InvalidOperationException>(() => Target.ExecuteAsync());
Assert.Equal("The index operations for the following package IDs failed due to version list concurrency: PackageB", ex.Message);
VerifyCompletedTelemetry(JobOutcome.Failure);
VerifyAllIdsAreProcessed(new[] { "PackageA", "PackageB", "PackageB", "PackageB" });
IndexActionBuilder.Verify(
x => x.UpdateAsync(
"PackageA",
It.IsAny <Func <SearchFilters, KeyedDocument> >()),
Times.Once);
IndexActionBuilder.Verify(
x => x.UpdateAsync(
"PackageB",
It.IsAny <Func <SearchFilters, KeyedDocument> >()),
Times.Exactly(3));
BatchPusher.Verify(
x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()),
Times.Exactly(4));
BatchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(3));
BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never);
}
public async Task RetriesFailedPackageIds()
{
Config.MaxConcurrentVersionListWriters = 1;
Changes["PackageA"] = 1;
Changes["PackageB"] = 2;
BatchPusher
.SetupSequence(x => x.TryFinishAsync())
.ReturnsAsync(new BatchPusherResult(new[] { "PackageB" }))
.ReturnsAsync(new BatchPusherResult());
await Target.ExecuteAsync();
VerifyCompletedTelemetry(JobOutcome.Success);
VerifyAllIdsAreProcessed(new[] { "PackageA", "PackageB", "PackageB" });
IndexActionBuilder.Verify(
x => x.UpdateAsync(
"PackageA",
It.IsAny <Func <SearchFilters, KeyedDocument> >()),
Times.Once);
IndexActionBuilder.Verify(
x => x.UpdateAsync(
"PackageB",
It.IsAny <Func <SearchFilters, KeyedDocument> >()),
Times.Exactly(2));
BatchPusher.Verify(
x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()),
Times.Exactly(3));
BatchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(2));
BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never);
}
public RebuildSearchCommand(
NuGetClientFactory clientFactory,
ICursor cursor,
IPackageService packages,
IndexActionBuilder actionBuilder,
AzureSearchBatchIndexer indexer,
ILogger <RebuildSearchCommand> logger)
{
_clientFactory = clientFactory;
_cursor = cursor;
_packages = packages;
_actionBuilder = actionBuilder;
_indexer = indexer;
_logger = logger;
}
[InlineData(4, 8, 15, 0)] // 4, 8 + 4 = 12 is greater than 10 so 8 is the batch size.
public async Task RespectsAzureSearchBatchSize(int documentsPerId, int batchSize, int fullPushes, int partialPushes)
{
var changeCount = 30;
var expectedPushes = fullPushes + partialPushes;
Config.AzureSearchBatchSize = 10;
IndexActions = new IndexActions(
new List <IndexAction <KeyedDocument> >(
Enumerable
.Range(0, documentsPerId)
.Select(x => IndexAction.Merge(new KeyedDocument()))),
new List <IndexAction <KeyedDocument> >(),
new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
new Mock <IAccessCondition>().Object));
AddChanges(changeCount);
await Target.ExecuteAsync();
VerifyCompletedTelemetry(JobOutcome.Success);
VerifyAllIdsAreProcessed(changeCount);
IndexActionBuilder.Verify(
x => x.UpdateAsync(
It.IsAny <string>(),
It.IsAny <Func <SearchFilters, KeyedDocument> >()),
Times.Exactly(changeCount));
BatchPusher.Verify(
x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()),
Times.Exactly(changeCount));
BatchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(expectedPushes));
BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never);
SystemTime.Verify(x => x.Delay(It.IsAny <TimeSpan>()), Times.Exactly(expectedPushes - 1));
DownloadDataClient.Verify(
x => x.ReplaceLatestIndexedAsync(
NewDownloadData,
It.Is <IAccessCondition>(a => a.IfMatchETag == OldDownloadResult.Metadata.ETag)),
Times.Once);
Assert.Equal(
fullPushes,
FinishedBatches.Count(b => b.Sum(ia => ia.Search.Count) == batchSize));
Assert.Equal(
partialPushes,
FinishedBatches.Count(b => b.Sum(ia => ia.Search.Count) != batchSize));
Assert.Empty(CurrentBatch);
}
public PackageIndexer(
IPackageService packages,
IUrlGenerator url,
CloudBlobContainer blobContainer,
RegistrationBuilder registrationBuilder,
IndexActionBuilder actionBuilder,
AzureSearchBatchIndexer search,
ILogger <PackageIndexer> logger)
{
_packages = packages;
_url = url;
_blobContainer = blobContainer;
_registrationBuilder = registrationBuilder;
_actionBuilder = actionBuilder;
_search = search;
_logger = logger;
}
public async Task PushesNothingWhenThereAreNoChanges()
{
await Target.ExecuteAsync();
VerifyCompletedTelemetry(JobOutcome.NoOp);
VerifyAllIdsAreProcessed(changeCount: 0);
IndexActionBuilder.Verify(
x => x.UpdateAsync(
It.IsAny <string>(),
It.IsAny <Func <SearchFilters, KeyedDocument> >()),
Times.Never);
BatchPusher.Verify(x => x.FinishAsync(), Times.Never);
BatchPusher.Verify(x => x.PushFullBatchesAsync(), Times.Never);
DownloadDataClient.Verify(
x => x.ReplaceLatestIndexedAsync(It.IsAny <DownloadData>(), It.IsAny <IAccessCondition>()),
Times.Never);
}
public async Task CanProcessInParallel()
{
var changeCount = 1000;
Config.AzureSearchBatchSize = 5;
Config.MaxConcurrentBatches = 4;
Config.MaxConcurrentVersionListWriters = 8;
AddChanges(changeCount);
await Target.ExecuteAsync();
VerifyCompletedTelemetry(JobOutcome.Success);
VerifyAllIdsAreProcessed(changeCount);
IndexActionBuilder.Verify(
x => x.UpdateAsync(
It.IsAny <string>(),
It.IsAny <Func <SearchFilters, KeyedDocument> >()),
Times.Exactly(changeCount));
BatchPusher.Verify(
x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()),
Times.Exactly(changeCount));
BatchPusher.Verify(x => x.TryFinishAsync(), Times.AtLeastOnce);
BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never);
}