Exemplo n.º 1
0
        public DnxCatalogCollector(
            Uri index,
            StorageFactory storageFactory,
            IAzureStorage preferredPackageSourceStorage,
            Uri contentBaseAddress,
            ITelemetryService telemetryService,
            ILogger logger,
            int maxDegreeOfParallelism,
            Func <HttpMessageHandler> handlerFunc = null,
            TimeSpan?httpClientTimeout            = null)
            : base(index, telemetryService, handlerFunc, httpClientTimeout)
        {
            _storageFactory     = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory));
            _sourceStorage      = preferredPackageSourceStorage;
            _contentBaseAddress = contentBaseAddress;
            _dnxMaker           = new DnxMaker(storageFactory);
            _logger             = logger ?? throw new ArgumentNullException(nameof(logger));

            if (maxDegreeOfParallelism < 1)
            {
                throw new ArgumentOutOfRangeException(
                          nameof(maxDegreeOfParallelism),
                          string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue));
            }

            // Find two factors which are close or equal to each other.
            var squareRoot = Math.Sqrt(maxDegreeOfParallelism);

            // If the max degree of parallelism is a perfect square, great.
            // Otherwise, prefer a greater degree of parallelism in batches than commit items within a batch.
            _maxConcurrentBatches = Convert.ToInt32(Math.Ceiling(squareRoot));
            _maxConcurrentCommitItemsWithinBatch = Convert.ToInt32(maxDegreeOfParallelism / _maxConcurrentBatches);

            ServicePointManager.DefaultConnectionLimit = _maxConcurrentBatches * _maxConcurrentCommitItemsWithinBatch;
        }
Exemplo n.º 2
0
        public DnxCatalogCollector(
            Uri index,
            StorageFactory storageFactory,
            IAzureStorage preferredPackageSourceStorage,
            Uri contentBaseAddress,
            ITelemetryService telemetryService,
            ILogger logger,
            int maxDegreeOfParallelism,
            Func <HttpMessageHandler> handlerFunc = null,
            TimeSpan?httpClientTimeout            = null)
            : base(index, telemetryService, handlerFunc, httpClientTimeout)
        {
            _storageFactory     = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory));
            _sourceStorage      = preferredPackageSourceStorage;
            _contentBaseAddress = contentBaseAddress;
            _dnxMaker           = new DnxMaker(storageFactory);
            _logger             = logger ?? throw new ArgumentNullException(nameof(logger));

            if (maxDegreeOfParallelism < 1)
            {
                throw new ArgumentOutOfRangeException(
                          nameof(maxDegreeOfParallelism),
                          string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue));
            }

            _maxDegreeOfParallelism = maxDegreeOfParallelism;
        }
 public DnxCatalogCollector(Uri index, StorageFactory storageFactory, Func<HttpMessageHandler> handlerFunc = null)
     : base(index, handlerFunc)
 {
     _dnxMaker = new DnxMaker(storageFactory);
 }
Exemplo n.º 4
0
        private async Task <IEnumerable <CatalogEntry> > ProcessCatalogEntriesAsync(
            CollectorHttpClient client,
            IEnumerable <CatalogEntry> catalogEntries,
            CancellationToken cancellationToken)
        {
            var processedCatalogEntries = new ConcurrentBag <CatalogEntry>();

            await catalogEntries.ForEachAsync(_maxConcurrentCommitItemsWithinBatch, async catalogEntry =>
            {
                var packageId = catalogEntry.PackageId;
                var normalizedPackageVersion = catalogEntry.NormalizedPackageVersion;

                if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDetails.AbsoluteUri)
                {
                    var telemetryProperties = GetTelemetryProperties(catalogEntry);

                    using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDetailsSeconds, telemetryProperties))
                    {
                        var packageFileName = PackageUtility.GetPackageFileName(
                            packageId,
                            normalizedPackageVersion);
                        var sourceUri              = new Uri(_contentBaseAddress, packageFileName);
                        var destinationStorage     = _storageFactory.Create(packageId);
                        var destinationRelativeUri = DnxMaker.GetRelativeAddressNupkg(
                            packageId,
                            normalizedPackageVersion);
                        var destinationUri = destinationStorage.GetUri(destinationRelativeUri);

                        var isNupkgSynchronized = await destinationStorage.AreSynchronized(sourceUri, destinationUri);
                        var isPackageInIndex    = await _dnxMaker.HasPackageInIndexAsync(
                            destinationStorage,
                            packageId,
                            normalizedPackageVersion,
                            cancellationToken);
                        var areRequiredPropertiesPresent = await AreRequiredPropertiesPresentAsync(destinationStorage, destinationUri);

                        if (isNupkgSynchronized && isPackageInIndex && areRequiredPropertiesPresent)
                        {
                            _logger.LogInformation("No changes detected: {Id}/{Version}", packageId, normalizedPackageVersion);

                            return;
                        }

                        if ((isNupkgSynchronized && areRequiredPropertiesPresent) ||
                            await ProcessPackageDetailsAsync(
                                client,
                                packageId,
                                normalizedPackageVersion,
                                sourceUri,
                                telemetryProperties,
                                cancellationToken))
                        {
                            processedCatalogEntries.Add(catalogEntry);
                        }
                    }
                }
                else if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDelete.AbsoluteUri)
                {
                    var properties = GetTelemetryProperties(catalogEntry);

                    using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDeleteSeconds, properties))
                    {
                        await ProcessPackageDeleteAsync(packageId, normalizedPackageVersion, cancellationToken);

                        processedCatalogEntries.Add(catalogEntry);
                    }
                }
            });

            return(processedCatalogEntries);
        }