private static async Task <IEnumerable <VersionInfo> > GetVersions( V2FeedPackageInfo package, MetadataReferenceCache metadataCache, SearchFilter filter, V2FeedParser feedParser, Common.ILogger log, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); // apply the filters to the version list returned var packages = await feedParser.FindPackagesByIdAsync( package.Id, filter.IncludeDelisted, filter.IncludePrerelease, log, cancellationToken); var uniqueVersions = new HashSet <NuGetVersion>(); var results = new List <VersionInfo>(); foreach (var versionPackage in packages.OrderByDescending(p => p.Version)) { if (uniqueVersions.Add(versionPackage.Version)) { var versionInfo = new VersionInfo(versionPackage.Version, versionPackage.DownloadCount) { PackageSearchMetadata = new PackageSearchMetadataV2Feed(versionPackage, metadataCache) }; results.Add(versionInfo); } } return(results); }
public PackageSearchMetadataV2Feed(V2FeedPackageInfo package, MetadataReferenceCache metadataCache) { Authors = metadataCache.GetString(string.Join(", ", package.Authors)); DependencySets = package.DependencySets; Description = package.Description; IconUrl = GetUriSafe(package.IconUrl); LicenseUrl = GetUriSafe(package.LicenseUrl); Owners = metadataCache.GetString(string.Join(", ", package.Owners)); PackageId = package.Id; ProjectUrl = GetUriSafe(package.ProjectUrl); Created = package.Created; LastEdited = package.LastEdited; Published = package.Published; ReportAbuseUrl = GetUriSafe(package.ReportAbuseUrl); RequireLicenseAcceptance = package.RequireLicenseAcceptance; Summary = package.Summary; Tags = package.Tags; Title = package.Title; Version = package.Version; IsListed = package.IsListed; long count; if (long.TryParse(package.DownloadCount, out count)) { DownloadCount = count; } }
/// <summary> /// Query nuget package list from nuget server. This implementation optimized for performance so doesn't iterate whole result /// returned nuget server, so as soon as find "take" number of result packages then stop processing and return the result. /// </summary> /// <param name="searchTerm">The term we're searching for.</param> /// <param name="filter">Filter for whether to include prerelease, delisted, supportedframework flags in query.</param> /// <param name="skip">Skip how many items from beginning of list.</param> /// <param name="take">Return how many items.</param> /// <param name="log">Logger instance.</param> /// <param name="cancellationToken">Cancellation token.</param> /// <returns>List of package meta data.</returns> public override async Task <IEnumerable <IPackageSearchMetadata> > SearchAsync(string searchTerm, SearchFilter filter, int skip, int take, Common.ILogger log, CancellationToken cancellationToken) { IEnumerable <PackageSearchMetadata> searchResultMetadata; var metadataCache = new MetadataReferenceCache(); if (_client != null && _searchEndpoints != null) { searchResultMetadata = await Search( searchTerm, filter, skip, take, Common.NullLogger.Instance, cancellationToken); } else { #pragma warning disable CS0618 var searchResultJsonObjects = await _rawSearchResource.Search(searchTerm, filter, skip, take, Common.NullLogger.Instance, cancellationToken); #pragma warning restore CS0618 searchResultMetadata = searchResultJsonObjects .Select(s => s.FromJToken <PackageSearchMetadata>()); } var searchResults = searchResultMetadata .Select(m => m.WithVersions(() => GetVersions(m, filter))) .Select(m => metadataCache.GetObject((PackageSearchMetadataBuilder.ClonedPackageSearchMetadata)m)) .ToArray(); return(searchResults); }
public override async Task <IEnumerable <IPackageSearchMetadata> > GetMetadataAsync(string packageId, bool includePrerelease, bool includeUnlisted, Common.ILogger log, CancellationToken token) { var metadataCache = new MetadataReferenceCache(); var packages = (await _regResource.GetPackageMetadata(packageId, includePrerelease, includeUnlisted, log, token)) .Select(ParseMetadata) .Select(m => metadataCache.GetObject(m)) .ToArray(); return(packages); }
public override async Task <IEnumerable <IPackageSearchMetadata> > GetMetadataAsync( string packageId, bool includePrerelease, bool includeUnlisted, Common.ILogger log, CancellationToken token) { var packages = await _feedParser.FindPackagesByIdAsync(packageId, includeUnlisted, includePrerelease, log, token); var metadataCache = new MetadataReferenceCache(); return(packages.Select(p => new PackageSearchMetadataV2Feed(p, metadataCache)).ToList()); }
public static IPackageSearchMetadata CreatePackageSearchResult( V2FeedPackageInfo package, MetadataReferenceCache metadataCache, SearchFilter filter, V2FeedParser feedParser, Common.ILogger log, CancellationToken cancellationToken) { var metadata = new PackageSearchMetadataV2Feed(package, metadataCache); return(metadata .WithVersions(() => GetVersions(package, metadataCache, filter, feedParser, log, cancellationToken))); }
/// <summary> /// Finds all entries on the page and parses them /// </summary> private IEnumerable <V2FeedPackageInfo> ParsePage(XDocument doc, string id, MetadataReferenceCache metadataCache) { if (doc.Root.Name == _xnameEntry) { return(new List <V2FeedPackageInfo> { ParsePackage(id, doc.Root, metadataCache) }); } else { return(doc.Root.Elements(_xnameEntry) .Select(x => ParsePackage(id, x, metadataCache))); } }
public override async Task <IEnumerable <IPackageSearchMetadata> > SearchAsync(string searchTerm, SearchFilter filter, int skip, int take, Common.ILogger log, CancellationToken cancellationToken) { var searchResultJsonObjects = await _rawSearchResource.Search(searchTerm, filter, skip, take, Common.NullLogger.Instance, cancellationToken); var metadataCache = new MetadataReferenceCache(); var searchResults = searchResultJsonObjects .Select(s => s.FromJToken <PackageSearchMetadata>()) .Select(m => m.WithVersions(() => GetVersions(m, filter))) .Select(m => metadataCache.GetObject((PackageSearchMetadataBuilder.ClonedPackageSearchMetadata)m)) .ToArray(); return(searchResults); }
public override async Task <IEnumerable <IPackageSearchMetadata> > GetMetadataAsync( string packageId, bool includePrerelease, bool includeUnlisted, SourceCacheContext sourceCacheContext, Common.ILogger log, CancellationToken token) { var packages = await _feedParser.FindPackagesByIdAsync(packageId, includeUnlisted, includePrerelease, sourceCacheContext, log, token); var metadataCache = new MetadataReferenceCache(); var filter = new SearchFilter(includePrerelease); filter.IncludeDelisted = includeUnlisted; return(packages.Select(p => V2FeedUtilities.CreatePackageSearchResult(p, metadataCache, filter, _feedParser, log, token)).ToList()); }
public override async Task <IPackageSearchMetadata> GetMetadataAsync( PackageIdentity package, SourceCacheContext sourceCacheContext, Common.ILogger log, CancellationToken token) { var v2Package = await _feedParser.GetPackage(package, sourceCacheContext, log, token); if (v2Package != null) { var metadataCache = new MetadataReferenceCache(); var filter = new SearchFilter(v2Package.Version.IsPrerelease); filter.IncludeDelisted = !v2Package.IsListed; return(V2FeedUtilities.CreatePackageSearchResult(v2Package, metadataCache, filter, _feedParser, log, token)); } return(null); }
private static async Task <IEnumerable <VersionInfo> > GetVersions( V2FeedPackageInfo package, MetadataReferenceCache metadataCache, SearchFilter filter, V2FeedParser feedParser, Common.ILogger log, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); using (var sourceCacheContext = new SourceCacheContext()) { // Update http source cache context MaxAge so that it can always go online to fetch // latest version of packages. sourceCacheContext.MaxAge = DateTimeOffset.UtcNow; // apply the filters to the version list returned var packages = await feedParser.FindPackagesByIdAsync( package.Id, filter.IncludeDelisted, filter.IncludePrerelease, sourceCacheContext, log, cancellationToken); var uniqueVersions = new HashSet <NuGetVersion>(); var results = new List <VersionInfo>(); foreach (var versionPackage in packages.OrderByDescending(p => p.Version)) { if (uniqueVersions.Add(versionPackage.Version)) { var versionInfo = new VersionInfo(versionPackage.Version, versionPackage.DownloadCount) { PackageSearchMetadata = new PackageSearchMetadataV2Feed(versionPackage, metadataCache) }; results.Add(versionInfo); } } return(results); } }
public override Task <IEnumerable <IPackageSearchMetadata> > GetMetadataAsync( string packageId, bool includePrerelease, bool includeUnlisted, ILogger log, CancellationToken token) { // All packages are considered listed within a local repo return(Task.Run <IEnumerable <IPackageSearchMetadata> >(() => { var metadataCache = new MetadataReferenceCache(); return _localResource.FindPackagesById(packageId, log, token) .Where(p => includePrerelease || !p.Identity.Version.IsPrerelease) .Select(GetPackageMetadata) .Select(p => metadataCache.GetObject(p)) .ToList(); }, token)); }
/// <summary> /// Process RegistrationPage /// </summary> /// <param name="registrationPage">Nuget registration page.</param> /// <param name="results">Used to return nuget result.</param> /// <param name="range">Nuget version range.</param> /// <param name="includePrerelease">Whether to include PreRelease versions into result.</param> /// <param name="includeUnlisted">Whether to include Unlisted versions into result.</param> private void ProcessRegistrationPage( RegistrationPage registrationPage, List <PackageSearchMetadataRegistration> results, VersionRange range, bool includePrerelease, bool includeUnlisted, MetadataReferenceCache metadataCache) { foreach (RegistrationLeafItem registrationLeaf in registrationPage.Items) { PackageSearchMetadataRegistration catalogEntry = registrationLeaf.CatalogEntry; NuGetVersion version = catalogEntry.Version; bool listed = catalogEntry.IsListed; if (range.Satisfies(catalogEntry.Version) && (includePrerelease || !version.IsPrerelease) && (includeUnlisted || listed)) { catalogEntry.ReportAbuseUrl = _reportAbuseResource?.GetReportAbuseUrl(catalogEntry.PackageId, catalogEntry.Version); catalogEntry.PackageDetailsUrl = _packageDetailsUriResource?.GetUri(catalogEntry.PackageId, catalogEntry.Version); catalogEntry = metadataCache.GetObject(catalogEntry); results.Add(catalogEntry); } } }
public override async Task <IEnumerable <IPackageSearchMetadata> > SearchAsync( string searchTerm, SearchFilter filters, int skip, int take, Common.ILogger log, CancellationToken cancellationToken) { var query = await _feedParser.Search( searchTerm, filters, skip, take, log, cancellationToken); var metadataCache = new MetadataReferenceCache(); // NuGet.Server does not group packages by id, this resource needs to handle it. var results = query.GroupBy(p => p.Id) .Select(group => group.OrderByDescending(p => p.Version).First()) .Select(package => V2FeedUtilities.CreatePackageSearchResult(package, metadataCache, filters, _feedParser, log, cancellationToken)); return(results.ToList()); }
public async Task <V2FeedPage> QueryV2FeedAsync( string relativeUri, string id, int max, bool ignoreNotFounds, SourceCacheContext sourceCacheContext, ILogger log, CancellationToken token) { var metadataCache = new MetadataReferenceCache(); var results = new List <V2FeedPackageInfo>(); var uris = new HashSet <string>(StringComparer.OrdinalIgnoreCase); var uri = string.Format("{0}{1}", _baseAddress, relativeUri); uris.Add(uri); // page var page = 1; // http cache key var cacheKey = GetCacheKey(relativeUri, page); // first request Task <XDocument> docRequest = LoadXmlAsync(uri, cacheKey, ignoreNotFounds, sourceCacheContext, log, token); // TODO: re-implement caching at a higher level for both v2 and v3 string nextUri = null; while (!token.IsCancellationRequested && docRequest != null) { // TODO: Pages for a package Id are cached separately. // So we will get inaccurate data when a page shrinks. // However, (1) In most cases the pages grow rather than shrink; // (2) cache for pages is valid for only 30 min. // So we decide to leave current logic and observe. var doc = await docRequest; if (doc != null) { var result = ParsePage(doc, id, metadataCache); results.AddRange(result); nextUri = GetNextUrl(doc); } docRequest = null; if (max < 0 || results.Count < max) { // Request the next url in parallel to parsing the current page if (!string.IsNullOrEmpty(nextUri)) { // a bug on the server side causes the same next link to be returned // for every page. To avoid falling into an infinite loop we must // keep track of all uri and error out for any duplicate uri which means // potential bug at server side. if (!uris.Add(nextUri)) { throw new FatalProtocolException(string.Format( CultureInfo.CurrentCulture, Strings.Protocol_duplicateUri, nextUri)); } page++; cacheKey = GetCacheKey(relativeUri, page); docRequest = LoadXmlAsync(nextUri, cacheKey, ignoreNotFounds, sourceCacheContext, log, token); } } } if (max > -1 && results.Count > max) { // Remove extra results if the page contained extras results = results.Take(max).ToList(); } if (docRequest != null) { // explicitly ignore exception to prevent it from going unobserved _ = docRequest.ContinueWith(t => { _ = t.Exception; }, TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously); } return(new V2FeedPage( results, string.IsNullOrEmpty(nextUri) ? null : nextUri)); }
/// <summary> /// Parse an entry into a V2FeedPackageInfo /// </summary> private V2FeedPackageInfo ParsePackage(string id, XElement element, MetadataReferenceCache metadataCache) { var properties = element.Element(_xnameProperties); var idElement = properties.Element(_xnameId); var titleElement = element.Element(_xnameTitle); // If 'Id' element exist, use its value as accurate package Id // Otherwise, use the value of 'title' if it exist // Use the given Id as final fallback if all elements above don't exist var identityId = metadataCache.GetString(idElement?.Value ?? titleElement?.Value ?? id); var versionString = properties.Element(_xnameVersion).Value; var version = metadataCache.GetVersion(metadataCache.GetString(versionString)); var downloadUrl = metadataCache.GetString(element.Element(_xnameContent).Attribute("src").Value); var title = metadataCache.GetString(titleElement?.Value); var summary = metadataCache.GetString(GetString(element, _xnameSummary)); var description = metadataCache.GetString(GetString(properties, _xnameDescription)); var iconUrl = metadataCache.GetString(GetString(properties, _xnameIconUrl)); var licenseUrl = metadataCache.GetString(GetString(properties, _xnameLicenseUrl)); var projectUrl = metadataCache.GetString(GetString(properties, _xnameProjectUrl)); var galleryDetailsUrl = metadataCache.GetString(GetString(properties, _xnameGalleryDetailsUrl)); var reportAbuseUrl = metadataCache.GetString(GetString(properties, _xnameReportAbuseUrl)); var tags = metadataCache.GetString(GetString(properties, _xnameTags)); var dependencies = metadataCache.GetString(GetString(properties, _xnameDependencies)); var downloadCount = metadataCache.GetString(GetString(properties, _xnameDownloadCount)); var requireLicenseAcceptance = StringComparer.OrdinalIgnoreCase.Equals(bool.TrueString, GetString(properties, _xnameRequireLicenseAcceptance)); var packageHash = metadataCache.GetString(GetString(properties, _xnamePackageHash)); var packageHashAlgorithm = metadataCache.GetString(GetString(properties, _xnamePackageHashAlgorithm)); NuGetVersion minClientVersion = null; var minClientVersionString = GetString(properties, _xnameMinClientVersion); if (!string.IsNullOrEmpty(minClientVersionString)) { if (NuGetVersion.TryParse(minClientVersionString, out minClientVersion)) { minClientVersion = metadataCache.GetVersion(minClientVersionString); } } var created = GetDate(properties, _xnameCreated); var lastEdited = GetDate(properties, _xnameLastEdited); var published = GetDate(properties, _xnamePublished); IEnumerable <string> owners = null; IEnumerable <string> authors = null; var authorNode = element.Element(_xnameAuthor); if (authorNode != null) { authors = authorNode.Elements(_xnameName).Select(e => metadataCache.GetString(e.Value)); } return(new V2FeedPackageInfo(new PackageIdentity(identityId, version), title, summary, description, authors, owners, iconUrl, licenseUrl, projectUrl, reportAbuseUrl, galleryDetailsUrl, tags, created, lastEdited, published, dependencies, requireLicenseAcceptance, downloadUrl, downloadCount, packageHash, packageHashAlgorithm, minClientVersion)); }
private async Task <IEnumerable <IPackageSearchMetadata> > GetMetadataAsync( string packageId, bool includePrerelease, bool includeUnlisted, VersionRange range, SourceCacheContext sourceCacheContext, ILogger log, CancellationToken token) { var metadataCache = new MetadataReferenceCache(); var registrationUri = _regResource.GetUri(packageId); var(registrationIndex, httpSourceCacheContext) = await LoadRegistrationIndexAsync( _client, registrationUri, packageId, sourceCacheContext, httpSourceResult => DeserializeStreamDataAsync <RegistrationIndex>(httpSourceResult.Stream, token), log, token); if (registrationIndex == null) { // The server returned a 404, the package does not exist return(Enumerable.Empty <PackageSearchMetadataRegistration>()); } var results = new List <PackageSearchMetadataRegistration>(); foreach (var registrationPage in registrationIndex.Items) { if (registrationPage == null) { throw new InvalidDataException(registrationUri.AbsoluteUri); } var lower = NuGetVersion.Parse(registrationPage.Lower); var upper = NuGetVersion.Parse(registrationPage.Upper); if (range.DoesRangeSatisfy(lower, upper)) { if (registrationPage.Items == null) { var rangeUri = registrationPage.Url; var leafRegistrationPage = await GetRegistratioIndexPageAsync(_client, rangeUri, packageId, lower, upper, httpSourceCacheContext, log, token); if (registrationPage == null) { throw new InvalidDataException(registrationUri.AbsoluteUri); } ProcessRegistrationPage(leafRegistrationPage, results, range, includePrerelease, includeUnlisted, metadataCache); } else { ProcessRegistrationPage(registrationPage, results, range, includePrerelease, includeUnlisted, metadataCache); } } } return(results); }