public UpdateAsync(ITestOutputHelper output)
{
_output = output;
_testDirectory = TestDirectory.Create();
_batches = new List <List <PackageDownloads> >();
_client = new Mock <IPackageDownloadsClient>();
_service = new Mock <IPackageService>();
_etagService = new Mock <IETagService>();
_settings = new ExplorePackagesSettings
{
DownloadsV1Path = Path.Combine(_testDirectory, "downloads.txt"),
};
_service
.Setup(x => x.AddOrUpdatePackagesAsync(It.IsAny <IEnumerable <PackageDownloads> >()))
.Returns(Task.CompletedTask)
.Callback <IEnumerable <PackageDownloads> >(b => _batches.Add(b.ToList()));
_target = new PackageDownloadsToDatabaseProcessor(
_client.Object,
_service.Object,
_etagService.Object,
_settings,
output.GetLogger <PackageDownloadsToDatabaseProcessor>());
}
public PackageDownloadsClient(
HttpClient httpClient,
ExplorePackagesSettings settings)
{
_httpClient = httpClient;
_settings = settings;
}
public PackagesContainerConsistencyService(
PackagesContainerClient client,
ExplorePackagesSettings settings)
{
_client = client;
_settings = settings;
}
public GalleryConsistencyService(
GalleryClient client,
ExplorePackagesSettings settings)
{
_client = client;
_settings = settings;
}
public V2ToDatabaseProcessor(
CursorService cursorService,
V2Client v2Client,
PackageService service,
ExplorePackagesSettings settings)
{
_cursorService = cursorService;
_v2Client = v2Client;
_service = service;
_settings = settings;
}
public CatalogToDatabaseProcessor(
IPackageService packageService,
CatalogService catalogService,
V2Client v2Client,
ExplorePackagesSettings settings)
{
_packageService = packageService;
_catalogService = catalogService;
_v2Client = v2Client;
_settings = settings;
}
public PackageDownloadsToDatabaseProcessor(
IPackageDownloadsClient client,
IPackageService service,
IETagService etagService,
ExplorePackagesSettings settings,
ILogger <PackageDownloadsToDatabaseProcessor> logger)
{
_client = client;
_service = service;
_etagService = etagService;
_settings = settings;
_logger = logger;
}
public BlobStorageService(ExplorePackagesSettings settings, ILogger <BlobStorageService> logger)
{
_settings = settings;
_logger = logger;
_lazyContainer = new Lazy <CloudBlobContainer>(() =>
{
var account = CloudStorageAccount.Parse(_settings.StorageConnectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(_settings.StorageContainerName);
return(container);
});
}
public PackageQueryContextBuilder(
NuspecStore nuspecStore,
PackageService packageService,
GalleryConsistencyService galleryConsistencyService,
ExplorePackagesSettings settings,
ILogger <PackageQueryContextBuilder> logger)
{
_nuspecStore = nuspecStore;
_packageService = packageService;
_galleryConsistencyService = galleryConsistencyService;
_settings = settings;
_logger = logger;
}
public FileStorageServiceTest(ITestOutputHelper output)
{
_directory = TestDirectory.Create();
_output = output;
_settings = new ExplorePackagesSettings
{
PackagePath = _directory,
StorageConnectionString = "UseDevelopmentStorage=true",
StorageContainerName = Guid.NewGuid().ToString("N"),
};
_blobNameProvider = new PackageBlobNameProvider();
_blobStorageService = new Mock <BlobStorageService>(_settings, _output.GetLogger <BlobStorageService>())
{
CallBase = true,
};
_memoryCache = new MemoryCache(Options.Create(new MemoryCacheOptions()));
_target = new FileStorageService(
_blobNameProvider,
_blobStorageService.Object,
_memoryCache,
_output.GetLogger <FileStorageService>());
}
public LatestV2PackageFetcher(V2Client client, ExplorePackagesSettings settings)
{
_client = client;
_settings = settings;
}
public static IServiceCollection AddExplorePackages(this IServiceCollection serviceCollection, ExplorePackagesSettings settings)
{
serviceCollection.AddMemoryCache();
serviceCollection.AddSingleton <UrlReporterProvider>();
serviceCollection.AddTransient <UrlReporterHandler>();
serviceCollection.AddTransient <LoggingHandler>();
serviceCollection.AddSingleton(
x => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
});
serviceCollection.AddTransient(
x => new InitializeServicePointHandler(
connectionLeaseTimeout: TimeSpan.FromMinutes(1)));
serviceCollection.AddTransient <HttpMessageHandler>(
x =>
{
var httpClientHandler = x.GetRequiredService <HttpClientHandler>();
var initializeServicePointerHander = x.GetRequiredService <InitializeServicePointHandler>();
var urlReporterHandler = x.GetRequiredService <UrlReporterHandler>();
initializeServicePointerHander.InnerHandler = httpClientHandler;
urlReporterHandler.InnerHandler = initializeServicePointerHander;
return(urlReporterHandler);
});
serviceCollection.AddSingleton(
x =>
{
var httpMessageHandler = x.GetRequiredService <HttpMessageHandler>();
var loggingHandler = x.GetRequiredService <LoggingHandler>();
loggingHandler.InnerHandler = httpMessageHandler;
var httpClient = new HttpClient(loggingHandler);
UserAgent.SetUserAgent(httpClient);
httpClient.DefaultRequestHeaders.TryAddWithoutValidation("x-ms-version", "2017-04-17");
return(httpClient);
});
serviceCollection.AddSingleton(
x => new HttpSource(
new PackageSource(settings.V3ServiceIndex),
() =>
{
var httpClientHandler = x.GetRequiredService <HttpClientHandler>();
var httpMessageHandler = x.GetRequiredService <HttpMessageHandler>();
return(Task.FromResult <HttpHandlerResource>(new HttpHandlerResourceV3(
httpClientHandler,
httpMessageHandler)));
},
NullThrottle.Instance));
var searchServiceUrlCache = new SearchServiceUrlCache();
serviceCollection.AddSingleton(searchServiceUrlCache);
serviceCollection.AddSingleton <ISearchServiceUrlCacheInvalidator>(searchServiceUrlCache);
serviceCollection.AddSingleton(
x => new CatalogReader(
new Uri(x.GetRequiredService <ExplorePackagesSettings>().V3ServiceIndex, UriKind.Absolute),
x.GetRequiredService <HttpSource>(),
cacheContext: null,
cacheTimeout: TimeSpan.Zero,
log: x.GetRequiredService <ILogger <CatalogReader> >().ToNuGetLogger()));
serviceCollection.AddTransient(
x => new HttpZipProvider(
x.GetRequiredService <HttpClient>())
{
FirstBufferSize = 4096,
SecondBufferSize = 4096,
BufferGrowthExponent = 2,
});
serviceCollection.AddTransient <MZipFormat>();
serviceCollection.AddTransient(x => settings.Clone());
serviceCollection.AddTransient <NuspecStore>();
serviceCollection.AddTransient <MZipStore>();
serviceCollection.AddTransient <RemoteCursorService>();
serviceCollection.AddTransient <IPortTester, PortTester>();
serviceCollection.AddTransient <IPortDiscoverer, SimplePortDiscoverer>();
serviceCollection.AddTransient <SearchServiceUrlDiscoverer>();
serviceCollection.AddTransient <SearchServiceCursorReader>();
serviceCollection.AddTransient <PackageQueryContextBuilder>();
serviceCollection.AddTransient <IProgressReporter, NullProgressReporter>();
serviceCollection.AddTransient <LatestV2PackageFetcher>();
serviceCollection.AddTransient <LatestCatalogCommitFetcher>();
serviceCollection.AddTransient <PackageBlobNameProvider>();
serviceCollection.AddTransient <IFileStorageService, FileStorageService>();
serviceCollection.AddTransient <IBlobStorageService, BlobStorageService>();
serviceCollection.AddTransient <PackageQueryProcessor>();
serviceCollection.AddTransient <CatalogToDatabaseProcessor>();
serviceCollection.AddTransient <CatalogToNuspecsProcessor>();
serviceCollection.AddTransient <V2ToDatabaseProcessor>();
serviceCollection.AddTransient <PackageDownloadsToDatabaseProcessor>();
serviceCollection.AddTransient <MZipCommitProcessor>();
serviceCollection.AddTransient <MZipCommitCollector>();
serviceCollection.AddTransient <MZipToDatabaseCommitProcessor>();
serviceCollection.AddTransient <MZipToDatabaseCommitCollector>();
serviceCollection.AddTransient <DependenciesToDatabaseCommitProcessor>();
serviceCollection.AddTransient <DependenciesToDatabaseCommitCollector>();
serviceCollection.AddTransient <DependencyPackagesToDatabaseCommitProcessor>();
serviceCollection.AddTransient <DependencyPackagesToDatabaseCommitCollector>();
serviceCollection.AddTransient <PackageCommitEnumerator>();
serviceCollection.AddTransient <ICommitEnumerator <PackageEntity>, PackageCommitEnumerator>();
serviceCollection.AddTransient <ICommitEnumerator <PackageRegistrationEntity>, PackageRegistrationCommitEnumerator>();
serviceCollection.AddTransient <CursorService>();
serviceCollection.AddTransient <IETagService, ETagService>();
serviceCollection.AddTransient <PackageService>();
serviceCollection.AddTransient <IPackageService, PackageService>();
serviceCollection.AddTransient <PackageQueryService>();
serviceCollection.AddTransient <CatalogService>();
serviceCollection.AddTransient <PackageDependencyService>();
serviceCollection.AddTransient <ProblemService>();
serviceCollection.AddTransient <V2Parser>();
serviceCollection.AddSingleton <ServiceIndexCache>();
serviceCollection.AddTransient <GalleryClient>();
serviceCollection.AddTransient <V2Client>();
serviceCollection.AddTransient <PackagesContainerClient>();
serviceCollection.AddTransient <FlatContainerClient>();
serviceCollection.AddTransient <RegistrationClient>();
serviceCollection.AddTransient <SearchClient>();
serviceCollection.AddTransient <AutocompleteClient>();
serviceCollection.AddTransient <IPackageDownloadsClient, PackageDownloadsClient>();
serviceCollection.AddTransient <GalleryConsistencyService>();
serviceCollection.AddTransient <V2ConsistencyService>();
serviceCollection.AddTransient <FlatContainerConsistencyService>();
serviceCollection.AddTransient <PackagesContainerConsistencyService>();
serviceCollection.AddTransient <RegistrationOriginalConsistencyService>();
serviceCollection.AddTransient <RegistrationGzippedConsistencyService>();
serviceCollection.AddTransient <RegistrationSemVer2ConsistencyService>();
serviceCollection.AddTransient <SearchLoadBalancerConsistencyService>();
serviceCollection.AddTransient <SearchSpecificInstancesConsistencyService>();
serviceCollection.AddTransient <PackageConsistencyService>();
serviceCollection.AddTransient <CrossCheckConsistencyService>();
serviceCollection.AddTransient <FindIdsEndingInDotNumberNuspecQuery>();
serviceCollection.AddTransient <FindRepositoriesNuspecQuery>();
serviceCollection.AddTransient <FindInvalidDependencyVersionsNuspecQuery>();
serviceCollection.AddTransient <FindMissingDependencyIdsNuspecQuery>();
serviceCollection.AddTransient <FindPackageTypesNuspecQuery>();
serviceCollection.AddTransient <FindSemVer2PackageVersionsNuspecQuery>();
serviceCollection.AddTransient <FindSemVer2DependencyVersionsNuspecQuery>();
serviceCollection.AddTransient <FindFloatingDependencyVersionsNuspecQuery>();
serviceCollection.AddTransient <FindNonAsciiIdsNuspecQuery>();
serviceCollection.AddTransient <FindInvalidPackageIdsNuspecQuery>();
serviceCollection.AddTransient <FindInvalidPackageVersionsNuspecQuery>();
serviceCollection.AddTransient <FindPackageVersionsContainingWhitespaceNuspecQuery>();
serviceCollection.AddTransient <FindInvalidDependencyIdNuspecQuery>();
serviceCollection.AddTransient <FindInvalidDependencyTargetFrameworkNuspecQuery>();
serviceCollection.AddTransient <FindMixedDependencyGroupStylesNuspecQuery>();
serviceCollection.AddTransient <FindWhitespaceDependencyTargetFrameworkNuspecQuery>();
serviceCollection.AddTransient <FindUnsupportedDependencyTargetFrameworkNuspecQuery>();
serviceCollection.AddTransient <FindDuplicateDependencyTargetFrameworksNuspecQuery>();
serviceCollection.AddTransient <FindDuplicateNormalizedDependencyTargetFrameworksNuspecQuery>();
serviceCollection.AddTransient <FindEmptyDependencyIdsNuspecQuery>();
serviceCollection.AddTransient <FindWhitespaceDependencyIdsNuspecQuery>();
serviceCollection.AddTransient <FindWhitespaceDependencyVersionsNuspecQuery>();
serviceCollection.AddTransient <FindDuplicateDependenciesNuspecQuery>();
serviceCollection.AddTransient <FindCaseSensitiveDuplicateMetadataElementsNuspecQuery>();
serviceCollection.AddTransient <FindCaseInsensitiveDuplicateMetadataElementsNuspecQuery>();
serviceCollection.AddTransient <FindNonAlphabetMetadataElementsNuspecQuery>();
serviceCollection.AddTransient <FindCollidingMetadataElementsNuspecQuery>();
serviceCollection.AddTransient <FindUnexpectedValuesForBooleanMetadataNuspecQuery>();
if (settings.RunBoringQueries)
{
serviceCollection.AddTransient <FindNonNormalizedPackageVersionsNuspecQuery>();
serviceCollection.AddTransient <FindMissingDependencyVersionsNuspecQuery>();
serviceCollection.AddTransient <FindEmptyDependencyVersionsNuspecQuery>();
}
if (settings.RunConsistencyChecks)
{
serviceCollection.AddTransient <IPackageQuery, HasV2DiscrepancyPackageQuery>();
serviceCollection.AddTransient <IPackageQuery, HasPackagesContainerDiscrepancyPackageQuery>();
serviceCollection.AddTransient <IPackageQuery, HasFlatContainerDiscrepancyPackageQuery>();
serviceCollection.AddTransient <IPackageQuery, HasRegistrationDiscrepancyInOriginalHivePackageQuery>();
serviceCollection.AddTransient <IPackageQuery, HasRegistrationDiscrepancyInGzippedHivePackageQuery>();
serviceCollection.AddTransient <IPackageQuery, HasRegistrationDiscrepancyInSemVer2HivePackageQuery>();
serviceCollection.AddTransient <IPackageQuery, HasSearchDiscrepancyPackageQuery>();
serviceCollection.AddTransient <IPackageQuery, HasCrossCheckDiscrepancyPackageQuery>();
}
serviceCollection.AddTransient <IPackageQuery, HasMissingNuspecPackageQuery>();
// Add all of the .nuspec queries as package queries.
var nuspecQueryDescriptors = serviceCollection
.Where(x => typeof(INuspecQuery).IsAssignableFrom(x.ServiceType))
.ToList();
foreach (var nuspecQueryDescriptor in nuspecQueryDescriptors)
{
serviceCollection.AddTransient <IPackageQuery>(x =>
{
var nuspecQuery = (INuspecQuery)x.GetRequiredService(nuspecQueryDescriptor.ImplementationType);
return(new NuspecPackageQuery(nuspecQuery));
});
}
return(serviceCollection);
}
