private async Task IndexPackageAsync( PackageDetailsCatalogLeaf catalogLeaf, CancellationToken cancellationToken) { var packageId = catalogLeaf.PackageId; var packageVersion = catalogLeaf.ParsePackageVersion(); Package package; Stream packageStream = null; Stream readmeStream = null; try { var contentClient = _clientFactory.CreatePackageContentClient(); using (var stream = await contentClient.GetPackageContentStreamOrNullAsync(packageId, packageVersion, cancellationToken)) { packageStream = await stream.AsTemporaryFileStreamAsync(cancellationToken); } _logger.LogInformation( "Downloaded package {PackageId} {PackageVersion}, building metadata...", packageId, packageVersion); using (var reader = new PackageArchiveReader(packageStream, leaveStreamOpen: true)) { package = reader.GetPackageMetadata(); package.Listed = catalogLeaf.IsListed(); package.Published = catalogLeaf.Published.UtcDateTime; if (package.HasReadme) { using (var stream = await reader.GetReadmeAsync(cancellationToken)) { readmeStream = await stream.AsTemporaryFileStreamAsync(cancellationToken); } } await IndexPackageAsync(package, readmeStream, cancellationToken); } } catch (Exception e) { _logger.LogError( e, "Failed to process package {PackageId} {PackageVersion}", packageId, packageVersion); throw; } finally { packageStream?.Dispose(); readmeStream?.Dispose(); } }
public HijackDocument.Full FullFromCatalog( string normalizedVersion, HijackDocumentChanges changes, PackageDetailsCatalogLeaf leaf) { var document = new HijackDocument.Full(); PopulateLatest( document, leaf.PackageId, normalizedVersion, lastUpdatedFromCatalog: true, lastCommitTimestamp: leaf.CommitTimestamp, lastCommitId: leaf.CommitId, changes: changes); _baseDocumentBuilder.PopulateMetadata(document, normalizedVersion, leaf); document.Listed = leaf.IsListed(); return(document); }
private void UpdateCatalogEntry( HiveType hive, string id, Protocol.Registration.RegistrationCatalogEntry catalogEntry, PackageDetailsCatalogLeaf packageDetails, NuGetVersion parsedVersion) { catalogEntry.Url = packageDetails.Url; catalogEntry.Type = JsonLdConstants.RegistrationLeafItemCatalogEntryType; catalogEntry.Authors = packageDetails.Authors ?? string.Empty; // Add the "registration" property to each package dependency. if (packageDetails.DependencyGroups != null) { catalogEntry.DependencyGroups = new List <RegistrationPackageDependencyGroup>(); foreach (var group in packageDetails.DependencyGroups) { var registrationGroup = new RegistrationPackageDependencyGroup { Url = group.Url, Type = group.Type, TargetFramework = group.TargetFramework, }; catalogEntry.DependencyGroups.Add(registrationGroup); if (group.Dependencies == null) { continue; } registrationGroup.Dependencies = new List <RegistrationPackageDependency>(); for (int i = 0; i < group.Dependencies.Count; i++) { var catalogDependency = group.Dependencies[i]; var registrationDependency = new RegistrationPackageDependency { Url = catalogDependency.Url, Type = catalogDependency.Type, Id = catalogDependency.Id, Range = catalogDependency.Range, Registration = _urlBuilder.GetIndexUrl(hive, catalogDependency.Id), }; registrationGroup.Dependencies.Add(registrationDependency); } } } else { catalogEntry.DependencyGroups = null; } // Add the types to the deprecation. if (hive == HiveType.Legacy || hive == HiveType.Gzipped) { catalogEntry.Deprecation = null; } else { catalogEntry.Deprecation = packageDetails.Deprecation; if (catalogEntry.Deprecation != null) { catalogEntry.Deprecation.Type = JsonLdConstants.PackageDeprecationType; if (catalogEntry.Deprecation.AlternatePackage != null) { catalogEntry.Deprecation.AlternatePackage.Type = JsonLdConstants.AlternatePackageType; } } } catalogEntry.Description = packageDetails.Description ?? string.Empty; if (!string.IsNullOrWhiteSpace(packageDetails.IconUrl) || !string.IsNullOrWhiteSpace(packageDetails.IconFile)) { catalogEntry.IconUrl = GetPackageIconUrl(id, packageDetails); } else { catalogEntry.IconUrl = string.Empty; } catalogEntry.PackageId = packageDetails.PackageId ?? id; catalogEntry.Language = packageDetails.Language ?? string.Empty; catalogEntry.LicenseExpression = packageDetails.LicenseExpression ?? string.Empty; if (!string.IsNullOrWhiteSpace(packageDetails.LicenseFile) || !string.IsNullOrWhiteSpace(packageDetails.LicenseExpression)) { // Use the package ID casing from this specific version, since license URLs do not exclusively use // lowercase package ID like icon URL. This is legacy behavior that can be revisited later. Gallery // supports case insensitive package IDs so it doesn't matter too much. catalogEntry.LicenseUrl = LicenseHelper.GetGalleryLicenseUrl( catalogEntry.PackageId, parsedVersion.ToNormalizedString(), _galleryBaseUrl); } else { catalogEntry.LicenseUrl = packageDetails.LicenseUrl ?? string.Empty; } catalogEntry.Listed = packageDetails.IsListed(); catalogEntry.MinClientVersion = packageDetails.MinClientVersion ?? string.Empty; catalogEntry.PackageContent = GetPackageContentUrl(id, packageDetails); catalogEntry.ProjectUrl = packageDetails.ProjectUrl ?? string.Empty; catalogEntry.Published = packageDetails.Published; catalogEntry.RequireLicenseAcceptance = packageDetails.RequireLicenseAcceptance ?? false; catalogEntry.Summary = packageDetails.Summary ?? string.Empty; if (packageDetails.Tags != null && packageDetails.Tags.Count > 0) { catalogEntry.Tags = packageDetails.Tags; } else { catalogEntry.Tags = new List <string> { string.Empty }; } catalogEntry.Title = packageDetails.Title ?? string.Empty; catalogEntry.Version = parsedVersion.ToFullString(); if (hive == HiveType.SemVer2 && packageDetails.Vulnerabilities != null && packageDetails.Vulnerabilities.Count > 0) { catalogEntry.Vulnerabilities = packageDetails.Vulnerabilities.Select(v => new RegistrationPackageVulnerability() { AdvisoryUrl = v.AdvisoryUrl, Severity = v.Severity } ).ToList(); } else { catalogEntry.Vulnerabilities = null; } }
public Task <bool> ProcessPackageDetailsAsync(PackageDetailsCatalogLeaf leaf) { _packageProcessTasks.Add(ProcessPackageLagDetailsAsync(leaf, leaf.Created, leaf.LastEdited, leaf.IsListed(), isDelete: false)); return(Task.FromResult(true)); }