public DnxCatalogCollector(
Uri index,
StorageFactory storageFactory,
IAzureStorage preferredPackageSourceStorage,
Uri contentBaseAddress,
ITelemetryService telemetryService,
ILogger logger,
int maxDegreeOfParallelism,
Func <HttpMessageHandler> handlerFunc = null,
TimeSpan?httpClientTimeout = null)
: base(index, telemetryService, handlerFunc, httpClientTimeout)
{
_storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory));
_sourceStorage = preferredPackageSourceStorage;
_contentBaseAddress = contentBaseAddress;
_dnxMaker = new DnxMaker(storageFactory);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (maxDegreeOfParallelism < 1)
{
throw new ArgumentOutOfRangeException(
nameof(maxDegreeOfParallelism),
string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue));
}
// Find two factors which are close or equal to each other.
var squareRoot = Math.Sqrt(maxDegreeOfParallelism);
// If the max degree of parallelism is a perfect square, great.
// Otherwise, prefer a greater degree of parallelism in batches than commit items within a batch.
_maxConcurrentBatches = Convert.ToInt32(Math.Ceiling(squareRoot));
_maxConcurrentCommitItemsWithinBatch = Convert.ToInt32(maxDegreeOfParallelism / _maxConcurrentBatches);
ServicePointManager.DefaultConnectionLimit = _maxConcurrentBatches * _maxConcurrentCommitItemsWithinBatch;
}
public DnxCatalogCollector(
Uri index,
StorageFactory storageFactory,
IAzureStorage preferredPackageSourceStorage,
Uri contentBaseAddress,
ITelemetryService telemetryService,
ILogger logger,
int maxDegreeOfParallelism,
Func <HttpMessageHandler> handlerFunc = null,
TimeSpan?httpClientTimeout = null)
: base(index, telemetryService, handlerFunc, httpClientTimeout)
{
_storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory));
_sourceStorage = preferredPackageSourceStorage;
_contentBaseAddress = contentBaseAddress;
_dnxMaker = new DnxMaker(storageFactory);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (maxDegreeOfParallelism < 1)
{
throw new ArgumentOutOfRangeException(
nameof(maxDegreeOfParallelism),
string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue));
}
_maxDegreeOfParallelism = maxDegreeOfParallelism;
}
public DnxCatalogCollector(Uri index, StorageFactory storageFactory, Func<HttpMessageHandler> handlerFunc = null)
: base(index, handlerFunc)
{
_dnxMaker = new DnxMaker(storageFactory);
}
private async Task <IEnumerable <CatalogEntry> > ProcessCatalogEntriesAsync(
CollectorHttpClient client,
IEnumerable <CatalogEntry> catalogEntries,
CancellationToken cancellationToken)
{
var processedCatalogEntries = new ConcurrentBag <CatalogEntry>();
await catalogEntries.ForEachAsync(_maxConcurrentCommitItemsWithinBatch, async catalogEntry =>
{
var packageId = catalogEntry.PackageId;
var normalizedPackageVersion = catalogEntry.NormalizedPackageVersion;
if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDetails.AbsoluteUri)
{
var telemetryProperties = GetTelemetryProperties(catalogEntry);
using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDetailsSeconds, telemetryProperties))
{
var packageFileName = PackageUtility.GetPackageFileName(
packageId,
normalizedPackageVersion);
var sourceUri = new Uri(_contentBaseAddress, packageFileName);
var destinationStorage = _storageFactory.Create(packageId);
var destinationRelativeUri = DnxMaker.GetRelativeAddressNupkg(
packageId,
normalizedPackageVersion);
var destinationUri = destinationStorage.GetUri(destinationRelativeUri);
var isNupkgSynchronized = await destinationStorage.AreSynchronized(sourceUri, destinationUri);
var isPackageInIndex = await _dnxMaker.HasPackageInIndexAsync(
destinationStorage,
packageId,
normalizedPackageVersion,
cancellationToken);
var areRequiredPropertiesPresent = await AreRequiredPropertiesPresentAsync(destinationStorage, destinationUri);
if (isNupkgSynchronized && isPackageInIndex && areRequiredPropertiesPresent)
{
_logger.LogInformation("No changes detected: {Id}/{Version}", packageId, normalizedPackageVersion);
return;
}
if ((isNupkgSynchronized && areRequiredPropertiesPresent) ||
await ProcessPackageDetailsAsync(
client,
packageId,
normalizedPackageVersion,
sourceUri,
telemetryProperties,
cancellationToken))
{
processedCatalogEntries.Add(catalogEntry);
}
}
}
else if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDelete.AbsoluteUri)
{
var properties = GetTelemetryProperties(catalogEntry);
using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDeleteSeconds, properties))
{
await ProcessPackageDeleteAsync(packageId, normalizedPackageVersion, cancellationToken);
processedCatalogEntries.Add(catalogEntry);
}
}
});
return(processedCatalogEntries);
}