Inheritance: NuGet.Services.Metadata.Catalog.AppendOnlyCatalogItem
        public async Task <PackageCatalogItem> CreateAsync(
            FeedPackageDetails packageItem,
            DateTime timestamp,
            CancellationToken cancellationToken)
        {
            if (packageItem == null)
            {
                throw new ArgumentNullException(nameof(packageItem));
            }

            cancellationToken.ThrowIfCancellationRequested();

            PackageCatalogItem item = null;

            if (_storage != null)
            {
                item = await GetPackageViaStorageAsync(packageItem, cancellationToken);
            }

            if (item == null)
            {
                item = await GetPackageViaHttpAsync(packageItem, timestamp, item, cancellationToken);
            }

            return(item);
        }
コード例 #2
0
        public async Task <PackageCatalogItem> CreateAsync(
            FeedPackageDetails packageItem,
            DateTime timestamp,
            CancellationToken cancellationToken)
        {
            if (packageItem == null)
            {
                throw new ArgumentNullException(nameof(packageItem));
            }

            cancellationToken.ThrowIfCancellationRequested();

            PackageCatalogItem item = null;

            _logger.LogInformation(
                "Creating package catalog item for {Id} {Version}",
                packageItem.PackageId,
                packageItem.PackageNormalizedVersion);

            if (_storage != null)
            {
                item = await GetPackageViaStorageAsync(packageItem, cancellationToken);
            }

            if (item == null)
            {
                item = await GetPackageViaHttpAsync(packageItem, timestamp, item, cancellationToken);
            }

            _logger.LogInformation(
                "Finished creating package catalog item for {Id} {Version}",
                packageItem.PackageId,
                packageItem.PackageNormalizedVersion);

            return(item);
        }
コード例 #3
0
        private async Task <PackageCatalogItem> GetPackageViaStorageAsync(
            FeedPackageDetails packageItem,
            CancellationToken cancellationToken)
        {
            PackageCatalogItem item      = null;
            var packageId                = packageItem.PackageId.ToLowerInvariant();
            var packageNormalizedVersion = packageItem.PackageNormalizedVersion.ToLowerInvariant();
            var packageFileName          = PackageUtility.GetPackageFileName(packageId, packageNormalizedVersion);
            var blobUri = _storage.ResolveUri(packageFileName);
            var blob    = await _storage.GetCloudBlockBlobReferenceAsync(blobUri);

            if (!await blob.ExistsAsync(cancellationToken))
            {
                _telemetryService.TrackMetric(
                    TelemetryConstants.NonExistentBlob,
                    metric: 1,
                    properties: GetProperties(packageId, packageNormalizedVersion, blob));

                return(item);
            }

            using (_telemetryService.TrackDuration(
                       TelemetryConstants.PackageBlobReadSeconds,
                       GetProperties(packageId, packageNormalizedVersion, blob: null)))
            {
                await blob.FetchAttributesAsync(cancellationToken);

                string packageHash = null;
                var    etag        = blob.ETag;

                var metadata = await blob.GetMetadataAsync(cancellationToken);

                if (metadata.TryGetValue(Constants.Sha512, out packageHash))
                {
                    using (var stream = await blob.GetStreamAsync(cancellationToken))
                    {
                        item = Utils.CreateCatalogItem(
                            packageItem.ContentUri.ToString(),
                            stream,
                            packageItem.CreatedDate,
                            packageItem.LastEditedDate,
                            packageItem.PublishedDate,
                            licenseNames: null,
                            licenseReportUrl: null,
                            packageHash: packageHash,
                            deprecationItem: packageItem.DeprecationInfo);

                        if (item == null)
                        {
                            _logger.LogWarning("Unable to extract metadata from: {PackageDetailsContentUri}", packageItem.ContentUri);
                        }
                    }

                    if (item != null)
                    {
                        // Since obtaining the ETag the first time, it's possible (though unlikely) that the blob may
                        // have changed.  Although reading a blob with a single GET request should return the whole
                        // blob in a consistent state, we're reading the blob using ZipArchive and a seekable stream,
                        // which results in many GET requests.  To guard against the blob having changed since we
                        // obtained the package hash, we check the ETag one more time.  If this check fails, we'll
                        // fallback to using a single HTTP GET request.
                        await blob.FetchAttributesAsync(cancellationToken);

                        if (etag != blob.ETag)
                        {
                            item = null;

                            _telemetryService.TrackMetric(
                                TelemetryConstants.BlobModified,
                                metric: 1,
                                properties: GetProperties(packageId, packageNormalizedVersion, blob));
                        }
                    }
                }
                else
                {
                    _telemetryService.TrackMetric(
                        TelemetryConstants.NonExistentPackageHash,
                        metric: 1,
                        properties: GetProperties(packageId, packageNormalizedVersion, blob));
                }
            }

            return(item);
        }
コード例 #4
0
        private async Task <PackageCatalogItem> GetPackageViaHttpAsync(FeedPackageDetails packageItem, DateTime timestamp, PackageCatalogItem item, CancellationToken cancellationToken)
        {
            // When downloading the package binary, add a query string parameter
            // that corresponds to the operation's timestamp.
            // This query string will ensure the package is not cached
            // (e.g. on the CDN) and returns the "latest and greatest" package metadata.
            var packageUri = Utilities.GetNugetCacheBustingUri(packageItem.ContentUri, timestamp.ToString("O"));
            HttpResponseMessage response = null;

            try
            {
                using (_telemetryService.TrackDuration(
                           TelemetryConstants.PackageDownloadSeconds,
                           GetProperties(packageItem, blob: null)))
                {
                    response = await _httpClient.GetAsync(packageUri, cancellationToken);
                }
            }
            catch (TaskCanceledException tce)
            {
                // If the HTTP request timed out, a TaskCanceledException will be thrown.
                throw new HttpClientTimeoutException($"HttpClient request timed out in {nameof(PackageCatalogItemCreator.GetPackageViaHttpAsync)}.", tce);
            }

            if (response.IsSuccessStatusCode)
            {
                using (var stream = await response.Content.ReadAsStreamAsync())
                {
                    item = Utils.CreateCatalogItem(
                        packageItem.ContentUri.ToString(),
                        stream,
                        packageItem.CreatedDate,
                        packageItem.LastEditedDate,
                        packageItem.PublishedDate,
                        deprecationItem: packageItem.DeprecationInfo);

                    if (item == null)
                    {
                        _logger.LogWarning("Unable to extract metadata from: {PackageDetailsContentUri}", packageItem.ContentUri);
                    }
                }
            }
            else
            {
                if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
                {
                    //  the feed is out of sync with the actual package storage - if we don't have the package there is nothing to be done we might as well move onto the next package
                    _logger?.LogWarning("Unable to download: {PackageDetailsContentUri}. Http status: {HttpStatusCode}", packageItem.ContentUri, response.StatusCode);
                }
                else
                {
                    //  this should trigger a restart - of this program - and not move the cursor forward
                    _logger?.LogError("Unable to download: {PackageDetailsContentUri}. Http status: {HttpStatusCode}", packageItem.ContentUri, response.StatusCode);
                    throw new Exception(
                              $"Unable to download: {packageItem.ContentUri} http status: {response.StatusCode}");
                }
            }

            return(item);
        }