public AzureSearchCollectorLogicIntegrationTests(ITestOutputHelper output)
{
_utilityConfig = new CommitCollectorConfiguration
{
MaxConcurrentCatalogLeafDownloads = 1,
};
_utilityOptions = new Mock <IOptionsSnapshot <CommitCollectorConfiguration> >();
_utilityOptions.Setup(x => x.Value).Returns(() => _utilityConfig);
_config = new Catalog2AzureSearchConfiguration
{
MaxConcurrentBatches = 1,
MaxConcurrentVersionListWriters = 1,
StorageContainer = "integration-tests-container",
StoragePath = "integration-tests-path",
RegistrationsBaseUrl = "https://example/registrations/",
GalleryBaseUrl = Data.GalleryBaseUrl,
FlatContainerBaseUrl = Data.FlatContainerBaseUrl,
FlatContainerContainerName = Data.FlatContainerContainerName,
Scoring = new AzureSearchScoringConfiguration()
};
_options = new Mock <IOptionsSnapshot <Catalog2AzureSearchConfiguration> >();
_options.Setup(x => x.Value).Returns(() => _config);
_developmentConfig = new AzureSearchJobDevelopmentConfiguration();
_developmentOptions = new Mock <IOptionsSnapshot <AzureSearchJobDevelopmentConfiguration> >();
_developmentOptions.Setup(x => x.Value).Returns(() => _developmentConfig);
_telemetryClient = new Mock <ITelemetryClient>();
_telemetryService = new AzureSearchTelemetryService(_telemetryClient.Object);
_v3TelemetryService = new V3TelemetryService(_telemetryClient.Object);
// Mock the database that is used for fetching owner information. The product code only reads
// from the database so it is less important to have a realistic, stateful implementation.
_entitiesContextFactory = new Mock <IEntitiesContextFactory>();
_entitiesContext = new Mock <IEntitiesContext>();
_entitiesContextFactory.Setup(x => x.CreateAsync(It.IsAny <bool>())).ReturnsAsync(() => _entitiesContext.Object);
_entitiesContext.Setup(x => x.PackageRegistrations).Returns(DbSetMockFactory.Create <PackageRegistration>());
_ownerFetcher = new DatabaseAuxiliaryDataFetcher(
new Mock <ISqlConnectionFactory <GalleryDbConfiguration> >().Object,
_entitiesContextFactory.Object,
_telemetryService,
output.GetLogger <DatabaseAuxiliaryDataFetcher>());
_cloudBlobClient = new InMemoryCloudBlobClient();
_versionListDataClient = new VersionListDataClient(
_cloudBlobClient,
_options.Object,
output.GetLogger <VersionListDataClient>());
_registrationClient = new InMemoryRegistrationClient();
_catalogClient = new InMemoryCatalogClient();
_leafFetcher = new CatalogLeafFetcher(
_registrationClient,
_catalogClient,
_options.Object,
_telemetryService,
output.GetLogger <CatalogLeafFetcher>());
_baseDocumentBuilder = new BaseDocumentBuilder(_options.Object);
_search = new SearchDocumentBuilder(_baseDocumentBuilder);
_hijack = new HijackDocumentBuilder(_baseDocumentBuilder);
_builder = new CatalogIndexActionBuilder(
_versionListDataClient,
_leafFetcher,
_ownerFetcher,
_search,
_hijack,
output.GetLogger <CatalogIndexActionBuilder>());
_searchIndex = new Mock <ISearchIndexClientWrapper>();
_searchDocuments = new InMemoryDocumentsOperations();
_searchIndex.Setup(x => x.Documents).Returns(() => _searchDocuments);
_hijackIndex = new Mock <ISearchIndexClientWrapper>();
_hijackDocuments = new InMemoryDocumentsOperations();
_hijackIndex.Setup(x => x.Documents).Returns(() => _hijackDocuments);
_fixUpEvaluator = new DocumentFixUpEvaluator(
_versionListDataClient,
_leafFetcher,
output.GetLogger <DocumentFixUpEvaluator>());
_commitCollectorUtility = new CommitCollectorUtility(
_catalogClient,
_v3TelemetryService,
_utilityOptions.Object,
output.GetLogger <CommitCollectorUtility>());
_collector = new AzureSearchCollectorLogic(
_builder,
() => new BatchPusher(
_searchIndex.Object,
_hijackIndex.Object,
_versionListDataClient,
_options.Object,
_developmentOptions.Object,
_telemetryService,
output.GetLogger <BatchPusher>()),
_fixUpEvaluator,
_commitCollectorUtility,
_options.Object,
_telemetryService,
output.GetLogger <AzureSearchCollectorLogic>());
}
public PopularityTransferIntegrationTests(ITestOutputHelper output)
{
_featureFlags = new Mock <IFeatureFlagService>();
_telemetry = new Mock <IAzureSearchTelemetryService>();
_config = new Auxiliary2AzureSearchConfiguration
{
AuxiliaryDataStorageContainer = "auxiliary-container",
EnablePopularityTransfers = true,
StorageContainer = "storage-container",
Scoring = new AzureSearchScoringConfiguration()
};
var options = new Mock <IOptionsSnapshot <Auxiliary2AzureSearchConfiguration> >();
options
.Setup(x => x.Value)
.Returns(_config);
_developmentConfig = new AzureSearchJobDevelopmentConfiguration();
var developmentOptions = new Mock <IOptionsSnapshot <AzureSearchJobDevelopmentConfiguration> >();
developmentOptions
.Setup(x => x.Value)
.Returns(_developmentConfig);
var auxiliaryConfig = new AuxiliaryDataStorageConfiguration
{
AuxiliaryDataStorageContainer = "auxiliary-container",
AuxiliaryDataStorageDownloadsPath = "downloads.json",
AuxiliaryDataStorageExcludedPackagesPath = "excludedPackages.json",
};
var auxiliaryOptions = new Mock <IOptionsSnapshot <AuxiliaryDataStorageConfiguration> >();
auxiliaryOptions
.Setup(x => x.Value)
.Returns(auxiliaryConfig);
_auxilliaryContainer = new InMemoryCloudBlobContainer();
_storageContainer = new InMemoryCloudBlobContainer();
_blobClient = new InMemoryCloudBlobClient();
_blobClient.Containers["auxiliary-container"] = _auxilliaryContainer;
_blobClient.Containers["storage-container"] = _storageContainer;
var auxiliaryFileClient = new AuxiliaryFileClient(
_blobClient,
auxiliaryOptions.Object,
_telemetry.Object,
output.GetLogger <AuxiliaryFileClient>());
_newPopularityTransfers = new PopularityTransferData();
var databaseFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>();
databaseFetcher
.Setup(x => x.GetPopularityTransfersAsync())
.ReturnsAsync(_newPopularityTransfers);
var downloadDataClient = new DownloadDataClient(
_blobClient,
options.Object,
_telemetry.Object,
output.GetLogger <DownloadDataClient>());
var popularityTransferDataClient = new PopularityTransferDataClient(
_blobClient,
options.Object,
_telemetry.Object,
output.GetLogger <PopularityTransferDataClient>());
var versionListDataClient = new VersionListDataClient(
_blobClient,
options.Object,
output.GetLogger <VersionListDataClient>());
var downloadComparer = new DownloadSetComparer(
_telemetry.Object,
options.Object,
output.GetLogger <DownloadSetComparer>());
var dataComparer = new DataSetComparer(
_telemetry.Object,
output.GetLogger <DataSetComparer>());
var downloadTransferrer = new DownloadTransferrer(
dataComparer,
options.Object,
output.GetLogger <DownloadTransferrer>());
var baseDocumentBuilder = new BaseDocumentBuilder(options.Object);
var searchDocumentBuilder = new SearchDocumentBuilder(baseDocumentBuilder);
var searchIndexActionBuilder = new SearchIndexActionBuilder(
versionListDataClient,
output.GetLogger <SearchIndexActionBuilder>());
_searchOperations = new Mock <IDocumentsOperationsWrapper>();
_searchOperations
.Setup(x => x.IndexAsync(It.IsAny <IndexBatch <KeyedDocument> >()))
.Callback <IndexBatch <KeyedDocument> >(batch =>
{
_indexedBatch = batch;
})
.ReturnsAsync(new DocumentIndexResult());
var hijackIndexClient = new Mock <ISearchIndexClientWrapper>();
var searchIndexClient = new Mock <ISearchIndexClientWrapper>();
searchIndexClient
.Setup(x => x.Documents)
.Returns(_searchOperations.Object);
var batchPusher = new BatchPusher(
searchIndexClient.Object,
hijackIndexClient.Object,
versionListDataClient,
options.Object,
developmentOptions.Object,
_telemetry.Object,
output.GetLogger <BatchPusher>());
Func <IBatchPusher> batchPusherFactory = () => batchPusher;
var time = new Mock <ISystemTime>();
_featureFlags.Setup(x => x.IsPopularityTransferEnabled()).Returns(true);
_target = new UpdateDownloadsCommand(
auxiliaryFileClient,
databaseFetcher.Object,
downloadDataClient,
downloadComparer,
downloadTransferrer,
popularityTransferDataClient,
searchDocumentBuilder,
searchIndexActionBuilder,
batchPusherFactory,
time.Object,
_featureFlags.Object,
options.Object,
_telemetry.Object,
output.GetLogger <Auxiliary2AzureSearchCommand>());
}
