public async Task OverridesDownloadCounts()
{
DownloadSetComparer
.Setup(c => c.Compare(It.IsAny <DownloadData>(), It.IsAny <DownloadData>()))
.Returns <DownloadData, DownloadData>((oldData, newData) =>
{
return(new SortedDictionary <string, long>(
newData.ToDictionary(d => d.Key, d => d.Value.Total),
StringComparer.OrdinalIgnoreCase));
});
NewDownloadData.SetDownloadCount("A", "1.0.0", 12);
NewDownloadData.SetDownloadCount("A", "2.0.0", 34);
NewDownloadData.SetDownloadCount("B", "3.0.0", 5);
NewDownloadData.SetDownloadCount("B", "4.0.0", 4);
NewDownloadData.SetDownloadCount("C", "5.0.0", 2);
NewDownloadData.SetDownloadCount("C", "6.0.0", 3);
DownloadOverrides["A"] = 55;
DownloadOverrides["b"] = 66;
await Target.ExecuteAsync();
// Documents should have new data with overriden downloads.
SearchDocumentBuilder
.Verify(
b => b.UpdateDownloadCount("A", SearchFilters.IncludePrereleaseAndSemVer2, 55),
Times.Once);
SearchDocumentBuilder
.Verify(
b => b.UpdateDownloadCount("B", SearchFilters.IncludePrereleaseAndSemVer2, 66),
Times.Once);
SearchDocumentBuilder
.Verify(
b => b.UpdateDownloadCount("C", SearchFilters.IncludePrereleaseAndSemVer2, 5),
Times.Once);
// Downloads auxiliary file should have new data without overriden downloads.
DownloadDataClient.Verify(
c => c.ReplaceLatestIndexedAsync(
It.Is <DownloadData>(d =>
d["A"].Total == 46 &&
d["A"]["1.0.0"] == 12 &&
d["A"]["2.0.0"] == 34 &&
d["B"].Total == 9 &&
d["B"]["3.0.0"] == 5 &&
d["B"]["4.0.0"] == 4 &&
d["C"].Total == 5 &&
d["C"]["5.0.0"] == 2 &&
d["C"]["6.0.0"] == 3),
It.IsAny <IAccessCondition>()),
Times.Once);
}
public async Task FailureIsRecordedInTelemetry()
{
var expected = new InvalidOperationException("Something bad!");
DownloadDataClient
.Setup(x => x.ReadLatestIndexedAsync(It.IsAny <IAccessCondition>(), It.IsAny <StringCache>()))
.ThrowsAsync(expected);
var actual = await Assert.ThrowsAsync <InvalidOperationException>(() => Target.ExecuteAsync());
VerifyCompletedTelemetry(JobOutcome.Failure);
Assert.Same(expected, actual);
}
[InlineData(4, 8, 15, 0)] // 4, 8 + 4 = 12 is greater than 10 so 8 is the batch size.
public async Task RespectsAzureSearchBatchSize(int documentsPerId, int batchSize, int fullPushes, int partialPushes)
{
var changeCount = 30;
var expectedPushes = fullPushes + partialPushes;
Config.AzureSearchBatchSize = 10;
IndexActions = new IndexActions(
new List <IndexAction <KeyedDocument> >(
Enumerable
.Range(0, documentsPerId)
.Select(x => IndexAction.Merge(new KeyedDocument()))),
new List <IndexAction <KeyedDocument> >(),
new ResultAndAccessCondition <VersionListData>(
new VersionListData(new Dictionary <string, VersionPropertiesData>()),
new Mock <IAccessCondition>().Object));
AddChanges(changeCount);
await Target.ExecuteAsync();
VerifyCompletedTelemetry(JobOutcome.Success);
VerifyAllIdsAreProcessed(changeCount);
IndexActionBuilder.Verify(
x => x.UpdateAsync(
It.IsAny <string>(),
It.IsAny <Func <SearchFilters, KeyedDocument> >()),
Times.Exactly(changeCount));
BatchPusher.Verify(
x => x.EnqueueIndexActions(It.IsAny <string>(), It.IsAny <IndexActions>()),
Times.Exactly(changeCount));
BatchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(expectedPushes));
BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never);
SystemTime.Verify(x => x.Delay(It.IsAny <TimeSpan>()), Times.Exactly(expectedPushes - 1));
DownloadDataClient.Verify(
x => x.ReplaceLatestIndexedAsync(
NewDownloadData,
It.Is <IAccessCondition>(a => a.IfMatchETag == OldDownloadResult.Metadata.ETag)),
Times.Once);
Assert.Equal(
fullPushes,
FinishedBatches.Count(b => b.Sum(ia => ia.Search.Count) == batchSize));
Assert.Equal(
partialPushes,
FinishedBatches.Count(b => b.Sum(ia => ia.Search.Count) != batchSize));
Assert.Empty(CurrentBatch);
}
public async Task PushesNothingWhenThereAreNoChanges()
{
await Target.ExecuteAsync();
VerifyCompletedTelemetry(JobOutcome.NoOp);
VerifyAllIdsAreProcessed(changeCount: 0);
IndexActionBuilder.Verify(
x => x.UpdateAsync(
It.IsAny <string>(),
It.IsAny <Func <SearchFilters, KeyedDocument> >()),
Times.Never);
BatchPusher.Verify(x => x.FinishAsync(), Times.Never);
BatchPusher.Verify(x => x.PushFullBatchesAsync(), Times.Never);
DownloadDataClient.Verify(
x => x.ReplaceLatestIndexedAsync(It.IsAny <DownloadData>(), It.IsAny <IAccessCondition>()),
Times.Never);
}
public async Task AlwaysAppliesDownloadOverrides()
{
DownloadSetComparer
.Setup(c => c.Compare(It.IsAny <DownloadData>(), It.IsAny <DownloadData>()))
.Returns <DownloadData, DownloadData>((oldData, newData) =>
{
var config = new Auxiliary2AzureSearchConfiguration();
var telemetry = Mock.Of <IAzureSearchTelemetryService>();
var logger = Mock.Of <ILogger <DownloadSetComparer> >();
var options = new Mock <IOptionsSnapshot <Auxiliary2AzureSearchConfiguration> >();
options.Setup(o => o.Value).Returns(config);
return(new DownloadSetComparer(telemetry, options.Object, logger)
.Compare(oldData, newData));
});
// Download override should be applied even if the package's downloads haven't changed.
OldDownloadData.SetDownloadCount("A", "1.0.0", 1);
NewDownloadData.SetDownloadCount("A", "1.0.0", 1);
DownloadOverrides["A"] = 2;
await Target.ExecuteAsync();
// Documents should have new data with overriden downloads.
SearchDocumentBuilder
.Verify(
b => b.UpdateDownloadCount("A", SearchFilters.IncludePrereleaseAndSemVer2, 2),
Times.Once);
// Downloads auxiliary file should have new data without overriden downloads.
DownloadDataClient.Verify(
c => c.ReplaceLatestIndexedAsync(
It.Is <DownloadData>(d =>
d["A"].Total == 1 &&
d["A"]["1.0.0"] == 1),
It.IsAny <IAccessCondition>()),
Times.Once);
}
public PopularityTransferIntegrationTests(ITestOutputHelper output)
{
_featureFlags = new Mock <IFeatureFlagService>();
_telemetry = new Mock <IAzureSearchTelemetryService>();
_config = new Auxiliary2AzureSearchConfiguration
{
AuxiliaryDataStorageContainer = "auxiliary-container",
EnablePopularityTransfers = true,
StorageContainer = "storage-container",
Scoring = new AzureSearchScoringConfiguration()
};
var options = new Mock <IOptionsSnapshot <Auxiliary2AzureSearchConfiguration> >();
options
.Setup(x => x.Value)
.Returns(_config);
_developmentConfig = new AzureSearchJobDevelopmentConfiguration();
var developmentOptions = new Mock <IOptionsSnapshot <AzureSearchJobDevelopmentConfiguration> >();
developmentOptions
.Setup(x => x.Value)
.Returns(_developmentConfig);
var auxiliaryConfig = new AuxiliaryDataStorageConfiguration
{
AuxiliaryDataStorageContainer = "auxiliary-container",
AuxiliaryDataStorageDownloadsPath = "downloads.json",
AuxiliaryDataStorageExcludedPackagesPath = "excludedPackages.json",
};
var auxiliaryOptions = new Mock <IOptionsSnapshot <AuxiliaryDataStorageConfiguration> >();
auxiliaryOptions
.Setup(x => x.Value)
.Returns(auxiliaryConfig);
_auxilliaryContainer = new InMemoryCloudBlobContainer();
_storageContainer = new InMemoryCloudBlobContainer();
_blobClient = new InMemoryCloudBlobClient();
_blobClient.Containers["auxiliary-container"] = _auxilliaryContainer;
_blobClient.Containers["storage-container"] = _storageContainer;
var auxiliaryFileClient = new AuxiliaryFileClient(
_blobClient,
auxiliaryOptions.Object,
_telemetry.Object,
output.GetLogger <AuxiliaryFileClient>());
_newPopularityTransfers = new PopularityTransferData();
var databaseFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>();
databaseFetcher
.Setup(x => x.GetPopularityTransfersAsync())
.ReturnsAsync(_newPopularityTransfers);
var downloadDataClient = new DownloadDataClient(
_blobClient,
options.Object,
_telemetry.Object,
output.GetLogger <DownloadDataClient>());
var popularityTransferDataClient = new PopularityTransferDataClient(
_blobClient,
options.Object,
_telemetry.Object,
output.GetLogger <PopularityTransferDataClient>());
var versionListDataClient = new VersionListDataClient(
_blobClient,
options.Object,
output.GetLogger <VersionListDataClient>());
var downloadComparer = new DownloadSetComparer(
_telemetry.Object,
options.Object,
output.GetLogger <DownloadSetComparer>());
var dataComparer = new DataSetComparer(
_telemetry.Object,
output.GetLogger <DataSetComparer>());
var downloadTransferrer = new DownloadTransferrer(
dataComparer,
options.Object,
output.GetLogger <DownloadTransferrer>());
var baseDocumentBuilder = new BaseDocumentBuilder(options.Object);
var searchDocumentBuilder = new SearchDocumentBuilder(baseDocumentBuilder);
var searchIndexActionBuilder = new SearchIndexActionBuilder(
versionListDataClient,
output.GetLogger <SearchIndexActionBuilder>());
_searchOperations = new Mock <IDocumentsOperationsWrapper>();
_searchOperations
.Setup(x => x.IndexAsync(It.IsAny <IndexBatch <KeyedDocument> >()))
.Callback <IndexBatch <KeyedDocument> >(batch =>
{
_indexedBatch = batch;
})
.ReturnsAsync(new DocumentIndexResult());
var hijackIndexClient = new Mock <ISearchIndexClientWrapper>();
var searchIndexClient = new Mock <ISearchIndexClientWrapper>();
searchIndexClient
.Setup(x => x.Documents)
.Returns(_searchOperations.Object);
var batchPusher = new BatchPusher(
searchIndexClient.Object,
hijackIndexClient.Object,
versionListDataClient,
options.Object,
developmentOptions.Object,
_telemetry.Object,
output.GetLogger <BatchPusher>());
Func <IBatchPusher> batchPusherFactory = () => batchPusher;
var time = new Mock <ISystemTime>();
_featureFlags.Setup(x => x.IsPopularityTransferEnabled()).Returns(true);
_target = new UpdateDownloadsCommand(
auxiliaryFileClient,
databaseFetcher.Object,
downloadDataClient,
downloadComparer,
downloadTransferrer,
popularityTransferDataClient,
searchDocumentBuilder,
searchIndexActionBuilder,
batchPusherFactory,
time.Object,
_featureFlags.Object,
options.Object,
_telemetry.Object,
output.GetLogger <Auxiliary2AzureSearchCommand>());
}
public async Task AppliesTransferChanges()
{
var downloadChanges = new SortedDictionary <string, long>(StringComparer.OrdinalIgnoreCase);
DownloadSetComparer
.Setup(c => c.Compare(It.IsAny <DownloadData>(), It.IsAny <DownloadData>()))
.Returns <DownloadData, DownloadData>((oldData, newData) =>
{
return(downloadChanges);
});
TransferChanges["Package1"] = 100;
TransferChanges["Package2"] = 200;
NewTransfers.AddTransfer("Package1", "Package2");
await Target.ExecuteAsync();
PopularityTransferDataClient
.Verify(
c => c.ReadLatestIndexedAsync(
It.Is <IAccessCondition>(x => x.IfMatchETag == null && x.IfNoneMatchETag == null),
It.IsAny <StringCache>()),
Times.Once);
DatabaseFetcher
.Verify(
d => d.GetPopularityTransfersAsync(),
Times.Once);
DownloadTransferrer
.Verify(
x => x.UpdateDownloadTransfers(
NewDownloadData,
downloadChanges,
OldTransfers,
NewTransfers),
Times.Once);
// Documents should be updated.
SearchDocumentBuilder
.Verify(
b => b.UpdateDownloadCount("Package1", SearchFilters.IncludePrereleaseAndSemVer2, 100),
Times.Once);
SearchDocumentBuilder
.Verify(
b => b.UpdateDownloadCount("Package2", SearchFilters.IncludePrereleaseAndSemVer2, 200),
Times.Once);
// Downloads auxiliary file should not include transfer changes.
DownloadDataClient.Verify(
c => c.ReplaceLatestIndexedAsync(
It.Is <DownloadData>(d => d.Count == 0),
It.IsAny <IAccessCondition>()),
Times.Once);
// Popularity transfers auxiliary file should have new data.
PopularityTransferDataClient.Verify(
c => c.ReplaceLatestIndexedAsync(
It.Is <PopularityTransferData>(d =>
d.Count == 1 &&
d["Package1"].Count == 1 &&
d["Package1"].Contains("Package2")),
It.IsAny <IAccessCondition>()),
Times.Once);
}
public async Task DoesNotOverrideIfDownloadsGreaterOrPackageHasNoDownloads()
{
DownloadSetComparer
.Setup(c => c.Compare(It.IsAny <DownloadData>(), It.IsAny <DownloadData>()))
.Returns <DownloadData, DownloadData>((oldData, newData) =>
{
return(new SortedDictionary <string, long>(
newData.ToDictionary(d => d.Key, d => d.Value.Total),
StringComparer.OrdinalIgnoreCase));
});
NewDownloadData.SetDownloadCount("A", "1.0.0", 100);
NewDownloadData.SetDownloadCount("A", "2.0.0", 200);
NewDownloadData.SetDownloadCount("B", "3.0.0", 5);
NewDownloadData.SetDownloadCount("B", "4.0.0", 4);
NewDownloadData.SetDownloadCount("C", "5.0.0", 0);
DownloadOverrides["A"] = 55;
DownloadOverrides["C"] = 66;
DownloadOverrides["D"] = 77;
await Target.ExecuteAsync();
// Documents should have new data with overriden downloads.
SearchDocumentBuilder
.Verify(
b => b.UpdateDownloadCount("A", SearchFilters.IncludePrereleaseAndSemVer2, 300),
Times.Once);
SearchDocumentBuilder
.Verify(
b => b.UpdateDownloadCount("B", SearchFilters.IncludePrereleaseAndSemVer2, 9),
Times.Once);
SearchDocumentBuilder
.Verify(
b => b.UpdateDownloadCount("B", SearchFilters.IncludePrereleaseAndSemVer2, 9),
Times.Once);
SearchDocumentBuilder
.Verify(
b => b.UpdateDownloadCount("C", It.IsAny <SearchFilters>(), It.IsAny <long>()),
Times.Never);
SearchDocumentBuilder
.Verify(
b => b.UpdateDownloadCount("D", It.IsAny <SearchFilters>(), It.IsAny <long>()),
Times.Never);
// Downloads auxiliary file should have new data without overriden downloads.
DownloadDataClient.Verify(
c => c.ReplaceLatestIndexedAsync(
It.Is <DownloadData>(d =>
d.Keys.Count() == 2 &&
d["A"].Total == 300 &&
d["A"]["1.0.0"] == 100 &&
d["A"]["2.0.0"] == 200 &&
d["B"].Total == 9 &&
d["B"]["3.0.0"] == 5 &&
d["B"]["4.0.0"] == 4),
It.IsAny <IAccessCondition>()),
Times.Once);
}