private static async Task <IEnumerable <VersionInfo> > GetVersions(
V2FeedPackageInfo package,
MetadataReferenceCache metadataCache,
SearchFilter filter,
V2FeedParser feedParser,
Common.ILogger log,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
// apply the filters to the version list returned
var packages = await feedParser.FindPackagesByIdAsync(
package.Id,
filter.IncludeDelisted,
filter.IncludePrerelease,
log,
cancellationToken);
var uniqueVersions = new HashSet <NuGetVersion>();
var results = new List <VersionInfo>();
foreach (var versionPackage in packages.OrderByDescending(p => p.Version))
{
if (uniqueVersions.Add(versionPackage.Version))
{
var versionInfo = new VersionInfo(versionPackage.Version, versionPackage.DownloadCount)
{
PackageSearchMetadata = new PackageSearchMetadataV2Feed(versionPackage, metadataCache)
};
results.Add(versionInfo);
}
}
return(results);
}
public PackageSearchMetadataV2Feed(V2FeedPackageInfo package, MetadataReferenceCache metadataCache)
{
Authors = metadataCache.GetString(string.Join(", ", package.Authors));
DependencySets = package.DependencySets;
Description = package.Description;
IconUrl = GetUriSafe(package.IconUrl);
LicenseUrl = GetUriSafe(package.LicenseUrl);
Owners = metadataCache.GetString(string.Join(", ", package.Owners));
PackageId = package.Id;
ProjectUrl = GetUriSafe(package.ProjectUrl);
Created = package.Created;
LastEdited = package.LastEdited;
Published = package.Published;
ReportAbuseUrl = GetUriSafe(package.ReportAbuseUrl);
RequireLicenseAcceptance = package.RequireLicenseAcceptance;
Summary = package.Summary;
Tags = package.Tags;
Title = package.Title;
Version = package.Version;
IsListed = package.IsListed;
long count;
if (long.TryParse(package.DownloadCount, out count))
{
DownloadCount = count;
}
}
/// <summary>
/// Query nuget package list from nuget server. This implementation optimized for performance so doesn't iterate whole result
/// returned nuget server, so as soon as find "take" number of result packages then stop processing and return the result.
/// </summary>
/// <param name="searchTerm">The term we're searching for.</param>
/// <param name="filter">Filter for whether to include prerelease, delisted, supportedframework flags in query.</param>
/// <param name="skip">Skip how many items from beginning of list.</param>
/// <param name="take">Return how many items.</param>
/// <param name="log">Logger instance.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>List of package meta data.</returns>
public override async Task <IEnumerable <IPackageSearchMetadata> > SearchAsync(string searchTerm, SearchFilter filter, int skip, int take, Common.ILogger log, CancellationToken cancellationToken)
{
IEnumerable <PackageSearchMetadata> searchResultMetadata;
var metadataCache = new MetadataReferenceCache();
if (_client != null && _searchEndpoints != null)
{
searchResultMetadata = await Search(
searchTerm,
filter,
skip,
take,
Common.NullLogger.Instance,
cancellationToken);
}
else
{
#pragma warning disable CS0618
var searchResultJsonObjects = await _rawSearchResource.Search(searchTerm, filter, skip, take, Common.NullLogger.Instance, cancellationToken);
#pragma warning restore CS0618
searchResultMetadata = searchResultJsonObjects
.Select(s => s.FromJToken <PackageSearchMetadata>());
}
var searchResults = searchResultMetadata
.Select(m => m.WithVersions(() => GetVersions(m, filter)))
.Select(m => metadataCache.GetObject((PackageSearchMetadataBuilder.ClonedPackageSearchMetadata)m))
.ToArray();
return(searchResults);
}
public override async Task <IEnumerable <IPackageSearchMetadata> > GetMetadataAsync(string packageId, bool includePrerelease, bool includeUnlisted, Common.ILogger log, CancellationToken token)
{
var metadataCache = new MetadataReferenceCache();
var packages =
(await _regResource.GetPackageMetadata(packageId, includePrerelease, includeUnlisted, log, token))
.Select(ParseMetadata)
.Select(m => metadataCache.GetObject(m))
.ToArray();
return(packages);
}
public override async Task <IEnumerable <IPackageSearchMetadata> > GetMetadataAsync(
string packageId,
bool includePrerelease,
bool includeUnlisted,
Common.ILogger log,
CancellationToken token)
{
var packages = await _feedParser.FindPackagesByIdAsync(packageId, includeUnlisted, includePrerelease, log, token);
var metadataCache = new MetadataReferenceCache();
return(packages.Select(p => new PackageSearchMetadataV2Feed(p, metadataCache)).ToList());
}
public static IPackageSearchMetadata CreatePackageSearchResult(
V2FeedPackageInfo package,
MetadataReferenceCache metadataCache,
SearchFilter filter,
V2FeedParser feedParser,
Common.ILogger log,
CancellationToken cancellationToken)
{
var metadata = new PackageSearchMetadataV2Feed(package, metadataCache);
return(metadata
.WithVersions(() => GetVersions(package, metadataCache, filter, feedParser, log, cancellationToken)));
}
/// <summary>
/// Finds all entries on the page and parses them
/// </summary>
private IEnumerable <V2FeedPackageInfo> ParsePage(XDocument doc, string id, MetadataReferenceCache metadataCache)
{
if (doc.Root.Name == _xnameEntry)
{
return(new List <V2FeedPackageInfo> {
ParsePackage(id, doc.Root, metadataCache)
});
}
else
{
return(doc.Root.Elements(_xnameEntry)
.Select(x => ParsePackage(id, x, metadataCache)));
}
}
public override async Task <IEnumerable <IPackageSearchMetadata> > SearchAsync(string searchTerm, SearchFilter filter, int skip, int take, Common.ILogger log, CancellationToken cancellationToken)
{
var searchResultJsonObjects = await _rawSearchResource.Search(searchTerm, filter, skip, take, Common.NullLogger.Instance, cancellationToken);
var metadataCache = new MetadataReferenceCache();
var searchResults = searchResultJsonObjects
.Select(s => s.FromJToken <PackageSearchMetadata>())
.Select(m => m.WithVersions(() => GetVersions(m, filter)))
.Select(m => metadataCache.GetObject((PackageSearchMetadataBuilder.ClonedPackageSearchMetadata)m))
.ToArray();
return(searchResults);
}
public override async Task <IEnumerable <IPackageSearchMetadata> > GetMetadataAsync(
string packageId,
bool includePrerelease,
bool includeUnlisted,
SourceCacheContext sourceCacheContext,
Common.ILogger log,
CancellationToken token)
{
var packages = await _feedParser.FindPackagesByIdAsync(packageId, includeUnlisted, includePrerelease, sourceCacheContext, log, token);
var metadataCache = new MetadataReferenceCache();
var filter = new SearchFilter(includePrerelease);
filter.IncludeDelisted = includeUnlisted;
return(packages.Select(p => V2FeedUtilities.CreatePackageSearchResult(p, metadataCache, filter, _feedParser, log, token)).ToList());
}
public override async Task <IPackageSearchMetadata> GetMetadataAsync(
PackageIdentity package,
SourceCacheContext sourceCacheContext,
Common.ILogger log,
CancellationToken token)
{
var v2Package = await _feedParser.GetPackage(package, sourceCacheContext, log, token);
if (v2Package != null)
{
var metadataCache = new MetadataReferenceCache();
var filter = new SearchFilter(v2Package.Version.IsPrerelease);
filter.IncludeDelisted = !v2Package.IsListed;
return(V2FeedUtilities.CreatePackageSearchResult(v2Package, metadataCache, filter, _feedParser, log, token));
}
return(null);
}
private static async Task <IEnumerable <VersionInfo> > GetVersions(
V2FeedPackageInfo package,
MetadataReferenceCache metadataCache,
SearchFilter filter,
V2FeedParser feedParser,
Common.ILogger log,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
using (var sourceCacheContext = new SourceCacheContext())
{
// Update http source cache context MaxAge so that it can always go online to fetch
// latest version of packages.
sourceCacheContext.MaxAge = DateTimeOffset.UtcNow;
// apply the filters to the version list returned
var packages = await feedParser.FindPackagesByIdAsync(
package.Id,
filter.IncludeDelisted,
filter.IncludePrerelease,
sourceCacheContext,
log,
cancellationToken);
var uniqueVersions = new HashSet <NuGetVersion>();
var results = new List <VersionInfo>();
foreach (var versionPackage in packages.OrderByDescending(p => p.Version))
{
if (uniqueVersions.Add(versionPackage.Version))
{
var versionInfo = new VersionInfo(versionPackage.Version, versionPackage.DownloadCount)
{
PackageSearchMetadata = new PackageSearchMetadataV2Feed(versionPackage, metadataCache)
};
results.Add(versionInfo);
}
}
return(results);
}
}
public override Task <IEnumerable <IPackageSearchMetadata> > GetMetadataAsync(
string packageId,
bool includePrerelease,
bool includeUnlisted,
ILogger log,
CancellationToken token)
{
// All packages are considered listed within a local repo
return(Task.Run <IEnumerable <IPackageSearchMetadata> >(() =>
{
var metadataCache = new MetadataReferenceCache();
return _localResource.FindPackagesById(packageId, log, token)
.Where(p => includePrerelease || !p.Identity.Version.IsPrerelease)
.Select(GetPackageMetadata)
.Select(p => metadataCache.GetObject(p))
.ToList();
},
token));
}
/// <summary>
/// Process RegistrationPage
/// </summary>
/// <param name="registrationPage">Nuget registration page.</param>
/// <param name="results">Used to return nuget result.</param>
/// <param name="range">Nuget version range.</param>
/// <param name="includePrerelease">Whether to include PreRelease versions into result.</param>
/// <param name="includeUnlisted">Whether to include Unlisted versions into result.</param>
private void ProcessRegistrationPage(
RegistrationPage registrationPage,
List <PackageSearchMetadataRegistration> results,
VersionRange range, bool includePrerelease,
bool includeUnlisted,
MetadataReferenceCache metadataCache)
{
foreach (RegistrationLeafItem registrationLeaf in registrationPage.Items)
{
PackageSearchMetadataRegistration catalogEntry = registrationLeaf.CatalogEntry;
NuGetVersion version = catalogEntry.Version;
bool listed = catalogEntry.IsListed;
if (range.Satisfies(catalogEntry.Version) &&
(includePrerelease || !version.IsPrerelease) &&
(includeUnlisted || listed))
{
catalogEntry.ReportAbuseUrl = _reportAbuseResource?.GetReportAbuseUrl(catalogEntry.PackageId, catalogEntry.Version);
catalogEntry.PackageDetailsUrl = _packageDetailsUriResource?.GetUri(catalogEntry.PackageId, catalogEntry.Version);
catalogEntry = metadataCache.GetObject(catalogEntry);
results.Add(catalogEntry);
}
}
}
public override async Task <IEnumerable <IPackageSearchMetadata> > SearchAsync(
string searchTerm,
SearchFilter filters,
int skip,
int take,
Common.ILogger log,
CancellationToken cancellationToken)
{
var query = await _feedParser.Search(
searchTerm,
filters,
skip,
take,
log,
cancellationToken);
var metadataCache = new MetadataReferenceCache();
// NuGet.Server does not group packages by id, this resource needs to handle it.
var results = query.GroupBy(p => p.Id)
.Select(group => group.OrderByDescending(p => p.Version).First())
.Select(package => V2FeedUtilities.CreatePackageSearchResult(package, metadataCache, filters, _feedParser, log, cancellationToken));
return(results.ToList());
}
public async Task <V2FeedPage> QueryV2FeedAsync(
string relativeUri,
string id,
int max,
bool ignoreNotFounds,
SourceCacheContext sourceCacheContext,
ILogger log,
CancellationToken token)
{
var metadataCache = new MetadataReferenceCache();
var results = new List <V2FeedPackageInfo>();
var uris = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
var uri = string.Format("{0}{1}", _baseAddress, relativeUri);
uris.Add(uri);
// page
var page = 1;
// http cache key
var cacheKey = GetCacheKey(relativeUri, page);
// first request
Task <XDocument> docRequest = LoadXmlAsync(uri, cacheKey, ignoreNotFounds, sourceCacheContext, log, token);
// TODO: re-implement caching at a higher level for both v2 and v3
string nextUri = null;
while (!token.IsCancellationRequested && docRequest != null)
{
// TODO: Pages for a package Id are cached separately.
// So we will get inaccurate data when a page shrinks.
// However, (1) In most cases the pages grow rather than shrink;
// (2) cache for pages is valid for only 30 min.
// So we decide to leave current logic and observe.
var doc = await docRequest;
if (doc != null)
{
var result = ParsePage(doc, id, metadataCache);
results.AddRange(result);
nextUri = GetNextUrl(doc);
}
docRequest = null;
if (max < 0 || results.Count < max)
{
// Request the next url in parallel to parsing the current page
if (!string.IsNullOrEmpty(nextUri))
{
// a bug on the server side causes the same next link to be returned
// for every page. To avoid falling into an infinite loop we must
// keep track of all uri and error out for any duplicate uri which means
// potential bug at server side.
if (!uris.Add(nextUri))
{
throw new FatalProtocolException(string.Format(
CultureInfo.CurrentCulture,
Strings.Protocol_duplicateUri,
nextUri));
}
page++;
cacheKey = GetCacheKey(relativeUri, page);
docRequest = LoadXmlAsync(nextUri, cacheKey, ignoreNotFounds, sourceCacheContext, log, token);
}
}
}
if (max > -1 && results.Count > max)
{
// Remove extra results if the page contained extras
results = results.Take(max).ToList();
}
if (docRequest != null)
{
// explicitly ignore exception to prevent it from going unobserved
_ = docRequest.ContinueWith(t => { _ = t.Exception; },
TaskContinuationOptions.OnlyOnFaulted |
TaskContinuationOptions.ExecuteSynchronously);
}
return(new V2FeedPage(
results,
string.IsNullOrEmpty(nextUri) ? null : nextUri));
}
/// <summary>
/// Parse an entry into a V2FeedPackageInfo
/// </summary>
private V2FeedPackageInfo ParsePackage(string id, XElement element, MetadataReferenceCache metadataCache)
{
var properties = element.Element(_xnameProperties);
var idElement = properties.Element(_xnameId);
var titleElement = element.Element(_xnameTitle);
// If 'Id' element exist, use its value as accurate package Id
// Otherwise, use the value of 'title' if it exist
// Use the given Id as final fallback if all elements above don't exist
var identityId = metadataCache.GetString(idElement?.Value ?? titleElement?.Value ?? id);
var versionString = properties.Element(_xnameVersion).Value;
var version = metadataCache.GetVersion(metadataCache.GetString(versionString));
var downloadUrl = metadataCache.GetString(element.Element(_xnameContent).Attribute("src").Value);
var title = metadataCache.GetString(titleElement?.Value);
var summary = metadataCache.GetString(GetString(element, _xnameSummary));
var description = metadataCache.GetString(GetString(properties, _xnameDescription));
var iconUrl = metadataCache.GetString(GetString(properties, _xnameIconUrl));
var licenseUrl = metadataCache.GetString(GetString(properties, _xnameLicenseUrl));
var projectUrl = metadataCache.GetString(GetString(properties, _xnameProjectUrl));
var galleryDetailsUrl = metadataCache.GetString(GetString(properties, _xnameGalleryDetailsUrl));
var reportAbuseUrl = metadataCache.GetString(GetString(properties, _xnameReportAbuseUrl));
var tags = metadataCache.GetString(GetString(properties, _xnameTags));
var dependencies = metadataCache.GetString(GetString(properties, _xnameDependencies));
var downloadCount = metadataCache.GetString(GetString(properties, _xnameDownloadCount));
var requireLicenseAcceptance = StringComparer.OrdinalIgnoreCase.Equals(bool.TrueString, GetString(properties, _xnameRequireLicenseAcceptance));
var packageHash = metadataCache.GetString(GetString(properties, _xnamePackageHash));
var packageHashAlgorithm = metadataCache.GetString(GetString(properties, _xnamePackageHashAlgorithm));
NuGetVersion minClientVersion = null;
var minClientVersionString = GetString(properties, _xnameMinClientVersion);
if (!string.IsNullOrEmpty(minClientVersionString))
{
if (NuGetVersion.TryParse(minClientVersionString, out minClientVersion))
{
minClientVersion = metadataCache.GetVersion(minClientVersionString);
}
}
var created = GetDate(properties, _xnameCreated);
var lastEdited = GetDate(properties, _xnameLastEdited);
var published = GetDate(properties, _xnamePublished);
IEnumerable <string> owners = null;
IEnumerable <string> authors = null;
var authorNode = element.Element(_xnameAuthor);
if (authorNode != null)
{
authors = authorNode.Elements(_xnameName).Select(e => metadataCache.GetString(e.Value));
}
return(new V2FeedPackageInfo(new PackageIdentity(identityId, version), title, summary, description, authors,
owners, iconUrl, licenseUrl, projectUrl, reportAbuseUrl, galleryDetailsUrl, tags, created, lastEdited,
published, dependencies, requireLicenseAcceptance, downloadUrl, downloadCount, packageHash,
packageHashAlgorithm, minClientVersion));
}
private async Task <IEnumerable <IPackageSearchMetadata> > GetMetadataAsync(
string packageId,
bool includePrerelease,
bool includeUnlisted,
VersionRange range,
SourceCacheContext sourceCacheContext,
ILogger log,
CancellationToken token)
{
var metadataCache = new MetadataReferenceCache();
var registrationUri = _regResource.GetUri(packageId);
var(registrationIndex, httpSourceCacheContext) = await LoadRegistrationIndexAsync(
_client,
registrationUri,
packageId,
sourceCacheContext,
httpSourceResult => DeserializeStreamDataAsync <RegistrationIndex>(httpSourceResult.Stream, token),
log,
token);
if (registrationIndex == null)
{
// The server returned a 404, the package does not exist
return(Enumerable.Empty <PackageSearchMetadataRegistration>());
}
var results = new List <PackageSearchMetadataRegistration>();
foreach (var registrationPage in registrationIndex.Items)
{
if (registrationPage == null)
{
throw new InvalidDataException(registrationUri.AbsoluteUri);
}
var lower = NuGetVersion.Parse(registrationPage.Lower);
var upper = NuGetVersion.Parse(registrationPage.Upper);
if (range.DoesRangeSatisfy(lower, upper))
{
if (registrationPage.Items == null)
{
var rangeUri = registrationPage.Url;
var leafRegistrationPage = await GetRegistratioIndexPageAsync(_client, rangeUri, packageId, lower, upper, httpSourceCacheContext, log, token);
if (registrationPage == null)
{
throw new InvalidDataException(registrationUri.AbsoluteUri);
}
ProcessRegistrationPage(leafRegistrationPage, results, range, includePrerelease, includeUnlisted, metadataCache);
}
else
{
ProcessRegistrationPage(registrationPage, results, range, includePrerelease, includeUnlisted, metadataCache);
}
}
}
return(results);
}
