internal static IEnumerable<PackageBase> ProcessPackagesFeed(Stream stream)
{
if (stream == null) {
throw new ArgumentNullException("stream");
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry");
if (entries == null)
{
var message = string.Format(Messages.ManifestRequiredXmlElementMissing, "entry");
throw new InvalidDataException(message);
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
ReadEntryElement(ref package, entry);
yield return package;
}
}
/// <summary>
/// Parse the 'entry' xml element.
/// </summary>
/// <param name="package"></param>
/// <param name="xElement"></param>
internal static void ReadEntryElement(ref PackageBase package, XElement xElement)
{
var node = xElement.FirstNode;
//iterate each <entry> element from the feed, e.g.
//http://www.nuget.org/api/v2/FindPackagesById()?id='Jquery'
//
while (node != null)
{
var element = node as XElement;
if (element != null)
{
ReadEntryChildNode(ref package, element);
}
node = node.NextNode;
}
}
/// <summary>
/// A helper method for processing the children of the 'metadata' tag in the .nuspec file
/// </summary>
/// <param name="package"></param>
/// <param name="element"></param>
private static void ReadPackageMetadataElement(ref PackageBase package, XElement element)
{
var value = element.Value.SafeTrim();
// since we put all the name to lower case, each word in case should all be in lower case
switch (element.Name.LocalName.ToLowerInvariant())
{
case "id":
package.Id = value;
break;
case "version":
package.Version = value;
break;
case "authors":
package.Authors = value;
break;
case "owners":
package.Owners = value;
break;
case "licenseurl":
package.LicenseUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "projecturl":
package.ProjectUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "iconurl":
package.IconUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "requirelicenseacceptance":
package.RequireLicenseAcceptance = XmlConvert.ToBoolean(value);
break;
case "developmentdependency":
package.DevelopmentDependency = XmlConvert.ToBoolean(value);
break;
case "description":
package.Description = value;
break;
case "summary":
package.Summary = value;
break;
case "releasenotes":
package.ReleaseNotes = value;
break;
case "copyright":
package.Copyright = value;
break;
case "language":
package.Language = value;
break;
case "title":
package.Title = value;
break;
case "tags":
package.Tags = value;
break;
case "dependencies":
package.DependencySetList = ReadDependencyList(element);
break;
/* case "frameworkAssemblies":
* manifestMetadata.FrameworkAssemblies = ReadFrameworkAssemblies(element);
* break;
* case "references":
* manifestMetadata.ReferenceSets = ReadReferenceSets(element);
* break;*/
}
}
/// <summary>
/// Parse the children of the 'entry' tag.
/// </summary>
/// <param name="package"></param>
/// <param name="element"></param>
private static void ReadEntryChildNode(ref PackageBase package, XElement element)
{
var value = element.Value.SafeTrim();
switch (element.Name.LocalName.ToLowerInvariant())
{
case "id":
//In manifest, <id>http://www.nuget.org/api/v2/Packages(Id='jQuery',Version='1.10.1')</id>
//we here only need Id=Jquery
package.Id = ExtractId(value);
break;
case "version":
package.Version = value;
break;
case "minclientversion":
package.MinClientVersion = string.IsNullOrWhiteSpace(value) ? null : new Version(value);
break;
case "author":
package.Authors = value;
break;
case "owners":
package.Owners = value;
break;
case "licenseurl":
package.LicenseUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "projecturl":
package.ProjectUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "iconurl":
package.IconUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "gallerydetailsurl":
package.GalleryDetailsUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "requirelicenseacceptance":
package.RequireLicenseAcceptance = XmlConvert.ToBoolean(value);
break;
case "developmentdependency":
package.DevelopmentDependency = XmlConvert.ToBoolean(value);
break;
case "description":
package.Description = value;
break;
case "summary":
package.Summary = value;
break;
case "content":
var srcAttribute = element.Attributes(XName.Get("src")).FirstOrDefault();
if (srcAttribute != null)
{
package.ContentSrcUrl = srcAttribute.Value;
}
break;
case "releasenotes":
package.ReleaseNotes = value;
break;
case "copyright":
package.Copyright = value;
break;
case "language":
package.Language = value;
break;
case "title":
package.Title = value;
break;
case "tags":
package.Tags = value;
break;
case "islatestversion":
package.IsLatestVersion = Convert.ToBoolean(value, CultureInfo.InvariantCulture);
break;
case "isabsolutelatestversion":
package.IsAbsoluteLatestVersion = Convert.ToBoolean(value, CultureInfo.InvariantCulture);
break;
case "published":
package.Published = GetDateTime(value);
break;
case "created":
package.Created = GetDateTime(value);
break;
case "lastupdated":
package.LastUpdated = GetDateTime(value);
break;
case "lastedited":
package.LastEdited = GetDateTime(value);
break;
case "licensereporturl":
package.LicenseReportUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "reportabuseurl":
package.ReportAbuseUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "downloadcount":
package.DownloadCount = XmlConvert.ToInt64(value);
break;
case "versiondownloadcount":
package.VersionDownloadCount = XmlConvert.ToInt64(value);
break;
case "packagesize":
package.PackageSize = XmlConvert.ToInt64(value);
break;
case "properties":
var node = element.FirstNode;
while (node != null)
{
var propertyElement = node as XElement;
if (propertyElement != null)
{
ReadEntryChildNode(ref package, propertyElement);
}
node = node.NextNode;
}
break;
case "dependencies":
package.DependencySetList = ParseDependencySet(value);
break;
case "packagehashalgorithm":
package.PackageHashAlgorithm = value;
break;
case "packagehash":
package.PackageHash = value;
break;
/* case "frameworkAssemblies":
* package.FrameworkAssemblies = ReadFrameworkAssemblies(element);
* break;
* case "references":
* package.ReferenceSets = ReadReferenceSets(element);
* break;*/
default:
if (!String.IsNullOrWhiteSpace(value) && !String.IsNullOrWhiteSpace(element.Name.LocalName))
{
package.AdditionalProperties.AddOrSet(element.Name.LocalName, value);
}
break;
}
}
/// <summary>
/// Send the request to the server with buffer size to account for the case where there are more data
/// that we need to fetch
/// </summary>
/// <param name="query"></param>
/// <param name="request"></param>
/// <returns></returns>
public static IEnumerable<PackageBase> SendRequest(string query, NuGetRequest request)
{
const int bufferSize = 40;
// number of threads sending the requests
const int numberOfSenders = 4;
var startPoint = 0;
var tasks = new List<Task<Stream>>();
bool stopSending = false;
object stopLock = new Object();
// Send one request first
// this initial query is of the form http://www.nuget.org/api/v2/FindPackagesById()?id='jquery'&$skip={0}&$top={1}
UriBuilder initialQuery = new UriBuilder(query.InsertSkipAndTop());
PackageBase firstPackage = null;
// Send out an initial request
// we send out 1 initial request first to check for redirection and check whether repository supports odata
using (Stream stream = NuGetClient.InitialDownloadDataToStream(initialQuery, startPoint, bufferSize, request))
{
if (stream == null)
{
yield break;
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry").ToList();
// If the initial request has different number of entries than the buffer size, return it because this means the server
// does not understand odata request or there is no more data. in the former case, we have to stop to prevent infinite loop
if (entries.Count != bufferSize)
{
request.Debug(Messages.PackagesReceived, entries.Count);
stopSending = true;
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
// set the first package of the request. this is used later to verify that the case when the number of packages in the repository
// is the same as the buffer size and the repository does not support odata query. in that case, we want to check whether the first package
// exists anywhere in the second call. if it is, then we cancel the request (this is to prevent infinite loop)
if (firstPackage == null)
{
firstPackage = package;
}
PackageUtility.ReadEntryElement(ref package, entry);
yield return package;
}
}
if (stopSending || request.IsCanceled)
{
yield break;
}
// To avoid more redirection (for example, if the initial query is nuget.org, it will be changed to www.nuget.org
query = initialQuery.Uri.ToString();
// Sending the initial requests
for (var i = 0; i < numberOfSenders; i++)
{
// Update the start point to fetch the packages
startPoint += bufferSize;
// Get the query
var newQuery = string.Format(query, startPoint, bufferSize);
// Send it
tasks.Add(Task.Factory.StartNew(() =>
{
Stream items = NuGetClient.DownloadDataToStream(newQuery, request);
return items;
}));
}
//Wait for the responses, parse the data, and send to the user
while (tasks.Count > 0)
{
//Cast because the compiler warning: Co-variant array conversion from Task[] to Task[] can cause run-time exception on write operation.
var index = Task.WaitAny(tasks.Cast<Task>().ToArray());
using (Stream stream = tasks[index].Result)
{
if (stream == null)
{
yield break;
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry").ToList();
if (entries.Count < bufferSize)
{
request.Debug(Messages.PackagesReceived, entries.Count);
lock (stopLock)
{
stopSending = true;
}
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
PackageUtility.ReadEntryElement(ref package, entry);
if (firstPackage != null)
{
// check whether first package in the first request exists anywhere in the second request
if (string.Equals(firstPackage.GetFullName(), package.GetFullName(), StringComparison.OrdinalIgnoreCase)
&& string.Equals(firstPackage.Version, package.Version, StringComparison.OrdinalIgnoreCase))
{
lock (stopLock)
{
stopSending = true;
}
break;
}
}
yield return package;
}
// we only needs to check for the existence of the first package in the second request. don't need to do for subsequent request
if (firstPackage != null)
{
firstPackage = null;
}
}
// checks whether we should stop sending requests
if (!stopSending && !request.IsCanceled)
{
// Make sure nobody else is updating the startPoint
lock (stopLock)
{
// update the startPoint
startPoint += bufferSize;
}
// Make a new request with the new startPoint
var newQuery = string.Format(query, startPoint, bufferSize);
//Keep sending a request
tasks[index] = (Task.Factory.StartNew(searchQuery =>
{
var items = NuGetClient.DownloadDataToStream(searchQuery.ToStringSafe(), request);
return items;
}, newQuery));
}
else
{
if (request.IsCanceled)
{
request.Warning(Messages.RequestCanceled, "HttpClientPackageRepository", "SendRequest");
//stop sending request to the remote server
stopSending = true;
}
tasks.RemoveAt(index);
}
}
}
/// <summary>
/// Send the request to the server with buffer size to account for the case where there are more data
/// that we need to fetch
/// </summary>
/// <param name="query"></param>
/// <param name="request"></param>
/// <returns></returns>
public static IEnumerable <PackageBase> SendRequest(string query, NuGetRequest request)
{
const int bufferSize = 40;
// number of threads sending the requests
const int numberOfSenders = 4;
var startPoint = 0;
var tasks = new List <Task <Stream> >();
bool stopSending = false;
object stopLock = new Object();
// Send one request first
// this initial query is of the form http://www.nuget.org/api/v2/FindPackagesById()?id='jquery'&$skip={0}&$top={1}
UriBuilder initialQuery = new UriBuilder(query.InsertSkipAndTop());
PackageBase firstPackage = null;
// Send out an initial request
// we send out 1 initial request first to check for redirection and check whether repository supports odata
using (Stream stream = NuGetClient.InitialDownloadDataToStream(initialQuery, startPoint, bufferSize, request))
{
if (stream == null)
{
yield break;
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry").ToList();
// If the initial request has different number of entries than the buffer size, return it because this means the server
// does not understand odata request or there is no more data. in the former case, we have to stop to prevent infinite loop
if (entries.Count != bufferSize)
{
request.Debug(Messages.PackagesReceived, entries.Count);
stopSending = true;
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
// set the first package of the request. this is used later to verify that the case when the number of packages in the repository
// is the same as the buffer size and the repository does not support odata query. in that case, we want to check whether the first package
// exists anywhere in the second call. if it is, then we cancel the request (this is to prevent infinite loop)
if (firstPackage == null)
{
firstPackage = package;
}
PackageUtility.ReadEntryElement(ref package, entry);
yield return(package);
}
}
if (stopSending || request.IsCanceled)
{
yield break;
}
// To avoid more redirection (for example, if the initial query is nuget.org, it will be changed to www.nuget.org
query = initialQuery.Uri.ToString();
// Sending the initial requests
for (var i = 0; i < numberOfSenders; i++)
{
// Update the start point to fetch the packages
startPoint += bufferSize;
// Get the query
var newQuery = string.Format(query, startPoint, bufferSize);
// Send it
tasks.Add(Task.Factory.StartNew(() =>
{
Stream items = NuGetClient.DownloadDataToStream(newQuery, request);
return(items);
}));
}
//Wait for the responses, parse the data, and send to the user
while (tasks.Count > 0)
{
//Cast because the compiler warning: Co-variant array conversion from Task[] to Task[] can cause run-time exception on write operation.
var index = Task.WaitAny(tasks.Cast <Task>().ToArray());
using (Stream stream = tasks[index].Result)
{
if (stream == null)
{
yield break;
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry").ToList();
if (entries.Count < bufferSize)
{
request.Debug(Messages.PackagesReceived, entries.Count);
lock (stopLock)
{
stopSending = true;
}
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
PackageUtility.ReadEntryElement(ref package, entry);
if (firstPackage != null)
{
// check whether first package in the first request exists anywhere in the second request
if (string.Equals(firstPackage.GetFullName(), package.GetFullName(), StringComparison.OrdinalIgnoreCase) &&
string.Equals(firstPackage.Version, package.Version, StringComparison.OrdinalIgnoreCase))
{
lock (stopLock)
{
stopSending = true;
}
break;
}
}
yield return(package);
}
// we only needs to check for the existence of the first package in the second request. don't need to do for subsequent request
if (firstPackage != null)
{
firstPackage = null;
}
}
// checks whether we should stop sending requests
if (!stopSending && !request.IsCanceled)
{
// Make sure nobody else is updating the startPoint
lock (stopLock)
{
// update the startPoint
startPoint += bufferSize;
}
// Make a new request with the new startPoint
var newQuery = string.Format(query, startPoint, bufferSize);
//Keep sending a request
tasks[index] = (Task.Factory.StartNew(searchQuery =>
{
var items = NuGetClient.DownloadDataToStream(searchQuery.ToStringSafe(), request);
return(items);
}, newQuery));
}
else
{
if (request.IsCanceled)
{
request.Warning(Messages.RequestCanceled, "HttpClientPackageRepository", "SendRequest");
//stop sending request to the remote server
stopSending = true;
}
tasks.RemoveAt(index);
}
}
}
private IEnumerable <PackageBase> GetPackagesForPackageEntry(dynamic packageEntry, NuGetSearchContext searchContext, RequestWrapper request)
{
PackageEntryInfo packageEntryInfo = new PackageEntryInfo(packageEntry.id);
if (!PackageFilterUtility.IsValidByName(packageEntryInfo, searchContext))
{
yield break;
}
// This will help us take packageEntryInfo.LatestVersion and packageEntryInfo.AbsoluteLatestVersion and get the matching package easily
// We're not setting isLatestVersion here so we don't have to deal with "is it latest or absolute latest"
Dictionary <SemanticVersion, PackageBase> versionToPackageTable = new Dictionary <SemanticVersion, PackageBase>();
NuGetSearchContext individualPackageSearchContext = new NuGetSearchContext()
{
PackageInfo = packageEntryInfo,
AllVersions = searchContext.AllVersions,
AllowPrerelease = searchContext.AllowPrerelease,
RequiredVersion = searchContext.RequiredVersion,
MinimumVersion = searchContext.MinimumVersion,
MaximumVersion = searchContext.MaximumVersion,
EnableDeepMetadataBypass = searchContext.EnableDeepMetadataBypass
};
bool latestVersionRequired = !searchContext.AllVersions && searchContext.RequiredVersion == null && searchContext.MinimumVersion == null && searchContext.MaximumVersion == null;
if (searchContext.EnableDeepMetadataBypass)
{
if (latestVersionRequired)
{
// Use the search result page to get the metadata for the latest version
SemanticVersion individualPackageVersion = new SemanticVersion(packageEntry.version);
packageEntryInfo.AddVersion(individualPackageVersion);
PackageBase pb = this.ResourcesCollection.PackageConverter.Make(packageEntry);
if (pb != null)
{
yield return(pb);
}
}
else
{
// Go to the registration index of this package first. This allows us to bypass "deep" (but required) metadata in certain cases.
NuGetPackageFeed3 packageFeed3 = (NuGetPackageFeed3)this.ResourcesCollection.PackagesFeed;
NuGetSearchResult result = packageFeed3.Find(individualPackageSearchContext, request);
foreach (PackageBase pb in result.Result.Cast <PackageBase>())
{
yield return(pb);
}
}
}
else
{
// Either we want a specific version or we want all metadata for any packages
foreach (dynamic packageVersionEntry in packageEntry.versions)
{
if (packageEntry.version.Equals(packageVersionEntry.version) || searchContext.AllVersions)
{
if (packageVersionEntry.Metadata.HasProperty("id"))
{
// Collect all versions from the search results so we can manually set isLatestVersion and isAbsoluteLatestVersion later
SemanticVersion individualPackageVersion = new SemanticVersion(packageVersionEntry.version);
packageEntryInfo.AddVersion(individualPackageVersion);
// Skip prerelease versions if AllowPrereleaseVersions is not specified
if (!String.IsNullOrEmpty(individualPackageVersion.SpecialVersion) && !searchContext.AllowPrerelease)
{
continue;
}
long?versionDownloadCount = null;
if (packageVersionEntry.HasProperty("downloads"))
{
versionDownloadCount = packageVersionEntry.downloads;
}
string registrationUrl = packageVersionEntry.Metadata.id;
// This should be PackageFeed3
// There should be a better way to reuse this function
NuGetPackageFeed3 packageFeed3 = (NuGetPackageFeed3)this.ResourcesCollection.PackagesFeed;
PackageBase packageVersionPackage = packageFeed3.Find(registrationUrl, individualPackageSearchContext, request, true).FirstOrDefault();
if (packageVersionPackage != null)
{
if (versionDownloadCount.HasValue)
{
packageVersionPackage.VersionDownloadCount = versionDownloadCount.Value;
}
// Reset these so we haven't collected all versions yet, so this is wrong
packageVersionPackage.IsLatestVersion = false;
packageVersionPackage.IsAbsoluteLatestVersion = false;
versionToPackageTable[individualPackageVersion] = packageVersionPackage;
}
}
}
}
// Now manually set the latest versions
if (packageEntryInfo.LatestVersion != null && versionToPackageTable.ContainsKey(packageEntryInfo.LatestVersion))
{
versionToPackageTable[packageEntryInfo.LatestVersion].IsLatestVersion = true;
}
if (packageEntryInfo.AbsoluteLatestVersion != null && versionToPackageTable.ContainsKey(packageEntryInfo.AbsoluteLatestVersion))
{
versionToPackageTable[packageEntryInfo.AbsoluteLatestVersion].IsAbsoluteLatestVersion = true;
}
// I think this is the best we can do for enumeration (reads all versions of a package before yielding anything)
foreach (PackageBase package in versionToPackageTable.Values)
{
yield return(package);
}
}
}
public string MakeDownloadUri(PackageBase package)
{
// v2 doesn't really build this dynamically, so assume ContentSrcUrl is already set. This shouldn't be called for v2 as it doesn't make sense.
return(package.ContentSrcUrl);
}
/// <summary>
/// A help for processing the metatadata tag
/// </summary>
/// <param name="package"></param>
/// <param name="xElement"></param>
private static void ReadPackageMetadata(ref PackageBase package, XElement xElement)
{
var node = xElement.FirstNode;
if (node == null) {
//Check for the tags within metadata tag
throw new InvalidDataException(string.Format(CultureInfo.InvariantCulture, Messages.NuspecRequiredXmlElementMissing, "id, version, authors, or description"));
}
while (node != null)
{
var element = node as XElement;
if (element != null)
{
ReadPackageMetadataElement(ref package, element);
}
node = node.NextNode;
}
// now check for required elements, which include <id>, <version>, <authors> and <description>
EnsureRequiredXmlTags(package);
}
/// <summary>
/// A helper method for processing the children of the 'metadata' tag in the .nuspec file
/// </summary>
/// <param name="package"></param>
/// <param name="element"></param>
private static void ReadPackageMetadataElement(ref PackageBase package, XElement element)
{
var value = element.Value.SafeTrim();
// since we put all the name to lower case, each word in case should all be in lower case
switch (element.Name.LocalName.ToLowerInvariant())
{
case "id":
package.Id = value;
break;
case "version":
package.Version = value;
break;
case "authors":
package.Authors = value;
break;
case "owners":
package.Owners = value;
break;
case "licenseurl":
package.LicenseUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "projecturl":
package.ProjectUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "iconurl":
package.IconUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "requirelicenseacceptance":
package.RequireLicenseAcceptance = XmlConvert.ToBoolean(value);
break;
case "developmentdependency":
package.DevelopmentDependency = XmlConvert.ToBoolean(value);
break;
case "description":
package.Description = value;
break;
case "summary":
package.Summary = value;
break;
case "releasenotes":
package.ReleaseNotes = value;
break;
case "copyright":
package.Copyright = value;
break;
case "language":
package.Language = value;
break;
case "title":
package.Title = value;
break;
case "tags":
package.Tags = value;
break;
case "dependencies":
package.DependencySetList = ReadDependencyList(element);
break;
/* case "frameworkAssemblies":
manifestMetadata.FrameworkAssemblies = ReadFrameworkAssemblies(element);
break;
case "references":
manifestMetadata.ReferenceSets = ReadReferenceSets(element);
break;*/
}
}
/// <summary>
/// Utility to parse the nuspec file and produces the package object.
/// </summary>
/// <param name="rootDocument">XDocument object pointing to a .nuspec xml file </param>
/// <returns></returns>
private static PackageBase ProcessNuspec(XDocument rootDocument)
{
if (rootDocument == null)
{
throw new ArgumentNullException("rootDocument");
}
var pkg = new PackageBase();
var metadataElement = rootDocument.Root.ElementsNoNamespace("metadata").FirstOrDefault();
if (metadataElement == null)
{
var message = string.Format(CultureInfo.InvariantCulture, Messages.NuspecRequiredXmlElementMissing, "metadata");
throw new InvalidDataException(message);
}
ReadPackageMetadata(ref pkg, metadataElement);
return pkg;
}
/// <summary>
/// Parse the children of the 'entry' tag.
/// </summary>
/// <param name="package"></param>
/// <param name="element"></param>
private static void ReadEntryChildNode(ref PackageBase package, XElement element)
{
var value = element.Value.SafeTrim();
switch (element.Name.LocalName.ToLowerInvariant())
{
case "id":
//In manifest, <id>http://www.nuget.org/api/v2/Packages(Id='jQuery',Version='1.10.1')</id>
//we here only need Id=Jquery
package.Id = ExtractId(value);
break;
case "version":
package.Version = value;
break;
case "minclientversion":
package.MinClientVersion = string.IsNullOrWhiteSpace(value) ? null : new Version(value);
break;
case "author":
package.Authors = value;
break;
case "owners":
package.Owners = value;
break;
case "licenseurl":
package.LicenseUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "projecturl":
package.ProjectUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "iconurl":
package.IconUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "gallerydetailsurl":
package.GalleryDetailsUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "requirelicenseacceptance":
package.RequireLicenseAcceptance = XmlConvert.ToBoolean(value);
break;
case "developmentdependency":
package.DevelopmentDependency = XmlConvert.ToBoolean(value);
break;
case "description":
package.Description = value;
break;
case "summary":
package.Summary = value;
break;
case "content":
var srcAttribute = element.Attributes(XName.Get("src")).FirstOrDefault();
if (srcAttribute != null) {
package.ContentSrcUrl = srcAttribute.Value;
}
break;
case "releasenotes":
package.ReleaseNotes = value;
break;
case "copyright":
package.Copyright = value;
break;
case "language":
package.Language = value;
break;
case "title":
package.Title = value;
break;
case "tags":
package.Tags = value;
break;
case "islatestversion":
package.IsLatestVersion = Convert.ToBoolean(value, CultureInfo.InvariantCulture);
break;
case "isabsolutelatestversion":
package.IsAbsoluteLatestVersion = Convert.ToBoolean(value, CultureInfo.InvariantCulture);
break;
case "published":
package.Published = string.IsNullOrWhiteSpace(value) ? (null as DateTimeOffset?) : Convert.ToDateTime(value, CultureInfo.InvariantCulture);
break;
case "created":
package.Created = string.IsNullOrWhiteSpace(value) ? (null as DateTimeOffset?) : Convert.ToDateTime(value, CultureInfo.InvariantCulture);
break;
case "lastupdated":
package.LastUpdated = string.IsNullOrWhiteSpace(value) ? (null as DateTimeOffset?) : Convert.ToDateTime(value, CultureInfo.InvariantCulture);
break;
case "lastedited":
package.LastEdited = string.IsNullOrWhiteSpace(value) ? (null as DateTimeOffset?) : Convert.ToDateTime(value, CultureInfo.InvariantCulture);
break;
case "licensereporturl":
package.LicenseReportUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "reportabuseurl":
package.ReportAbuseUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value);
break;
case "downloadcount":
package.DownloadCount = XmlConvert.ToInt64(value);
break;
case "versiondownloadcount":
package.VersionDownloadCount = XmlConvert.ToInt64(value);
break;
case "packagesize":
package.PackageSize = XmlConvert.ToInt64(value);
break;
case "properties":
var node = element.FirstNode;
while (node != null)
{
var propertyElement = node as XElement;
if (propertyElement != null)
{
ReadEntryChildNode(ref package, propertyElement);
}
node = node.NextNode;
}
break;
case "dependencies":
package.DependencySetList = ParseDependencySet(value);
break;
case "packagehashalgorithm":
package.PackageHashAlgorithm = value;
break;
case "packagehash":
package.PackageHash = value;
break;
/* case "frameworkAssemblies":
package.FrameworkAssemblies = ReadFrameworkAssemblies(element);
break;
case "references":
package.ReferenceSets = ReadReferenceSets(element);
break;*/
default:
if (!String.IsNullOrWhiteSpace(value) && !String.IsNullOrWhiteSpace(element.Name.LocalName))
{
package.AdditionalProperties.AddOrSet(element.Name.LocalName, value);
}
break;
}
}
/// <summary>
/// No download URI.
/// </summary>
public string MakeDownloadUri(PackageBase package)
{
throw new NotImplementedException();
}
/// <summary>
/// Find packages when the registration URL is already known.
/// </summary>
/// <param name="registrationUrl"></param>
/// <param name="request"></param>
/// <returns></returns>
internal IEnumerable <PackageBase> Find(string registrationUrl, NuGetSearchContext context, RequestWrapper request, bool finalAttempt)
{
request.Debug(Messages.DebugInfoCallMethod3, "NuGetPackageFeed3", "Find", registrationUrl);
List <PackageBase> packages = null;
PackageBase cachedPackage;
if (!registrationUrl.Contains("index.json") && ConcurrentInMemoryCache.Instance.TryGet <PackageBase>(registrationUrl, out cachedPackage))
{
if (cachedPackage == null)
{
return(packages);
}
else
{
packages = new List <PackageBase>()
{
cachedPackage
};
return(packages);
}
}
Stream s = NuGetClient.DownloadDataToStream(registrationUrl, request, ignoreNullResponse: true, tries: 1);
if (s != null)
{
packages = new List <PackageBase>();
// Now we can get the catalog entry
dynamic root = DynamicJsonParser.Parse(new StreamReader(s).ReadToEnd());
if (root.Metadata.HasProperty("type"))
{
// First check if this is a Package or PackageRegistration type
// Package is from packageId + version
// PackageRegistration is from packageId
bool isRegistrationType = false;
foreach (string t in root.Metadata.type)
{
if (t.Equals("PackageRegistration", StringComparison.OrdinalIgnoreCase))
{
isRegistrationType = true;
break;
}
}
if (context.PackageInfo.AllVersions.Count == 0)
{
// Only when the version list is restricted is this method usually faster
this.ResourcesCollection.FilesFeed.GetVersionInfo(context.PackageInfo, request);
}
HashSet <SemanticVersion> packageSemanticVersions = null;
if (context.PackageInfo.AllVersions.Count > 0)
{
packageSemanticVersions = FilterVersionsByRequirements(context, context.PackageInfo);
}
if (isRegistrationType)
{
// This is a registration index, like: "https://api.nuget.org/v3/registration3/json/index.json"
// Get all versions from files service
// In addition, when DeepMetadataBypass is enabled, we MUST use the registration index to get package info
// If a call to -Name -RequiredVersion is done, DeepMetadataBypass will never be enabled (for now)
// If we wanted, we could enable this by checking if !isRegistrationType && context.EnableDeepMetadataBypass, then call into Find with the registration index URL
if (!context.AllVersions && packageSemanticVersions != null && !context.EnableDeepMetadataBypass)
{
foreach (SemanticVersion packageVersion in packageSemanticVersions)
{
NuGetSearchResult result = this.ResourcesCollection.PackagesFeed.Find(new NuGetSearchContext()
{
PackageInfo = context.PackageInfo,
RequiredVersion = packageVersion,
EnableDeepMetadataBypass = context.EnableDeepMetadataBypass
}, request);
PackageBase package = result.Result == null ? null : result.Result.FirstOrDefault() as PackageBase;
if (package != null)
{
packages.Add(package);
}
}
}
else
{
// Going to collect versions from the registration index in here
// Map of package version -> either PackageBase (if context.EnableDeepMetadataBypass) or catalog URL
Dictionary <SemanticVersion, object> catalogObjects = new Dictionary <SemanticVersion, object>();
// If the version list hasn't been built yet, we can build it from the registration page instead of using FilesFeed
bool buildPackageInfoVersions = context.PackageInfo.AllVersions.Count == 0;
// Fallback to catalog crawling in these cases:
// - Bypass deep metadata
// - Getting version list failed
// - All versions required
foreach (dynamic catalogPage in root.items)
{
dynamic actualCatalogPage = catalogPage;
if (!actualCatalogPage.HasProperty("items"))
{
// Sometimes the catalog page on the PackageRegistration entry doesn't have the actual page contents
// In this case, the ID metadata tag points to the full catalog entry
Stream fullCatalogPageResponseStream = NuGetClient.DownloadDataToStream(actualCatalogPage.Metadata.id, request);
if (fullCatalogPageResponseStream != null)
{
actualCatalogPage = DynamicJsonParser.Parse(new StreamReader(fullCatalogPageResponseStream).ReadToEnd());
}
}
foreach (dynamic packageEntry in actualCatalogPage.items)
{
SemanticVersion version = new SemanticVersion(packageEntry.catalogentry.version);
if (buildPackageInfoVersions)
{
context.PackageInfo.AddVersion(version);
}
bool hasCatalogEntry = packageEntry.HasProperty("catalogentry");
if (context.EnableDeepMetadataBypass || !hasCatalogEntry)
{
// Bypass retrieving "deep" (but required) metadata like packageHash
// Also do this if there's no catalog entry
PackageBase pb = null;
if (packageEntry.HasProperty("catalogentry"))
{
pb = this.ResourcesCollection.PackageConverter.Make(packageEntry.catalogentry, context.PackageInfo);
}
else
{
// In some implementations (lookin' at you MyGet) there's no catalogEntry object
pb = this.ResourcesCollection.PackageConverter.Make(packageEntry, context.PackageInfo);
}
if (pb != null)
{
catalogObjects[version] = pb;
}
}
else
{
catalogObjects[version] = this.ResourcesCollection.CatalogUrlConverter.Make(packageEntry);
}
}
}
packageSemanticVersions = FilterVersionsByRequirements(context, context.PackageInfo);
foreach (SemanticVersion version in packageSemanticVersions)
{
if (!catalogObjects.ContainsKey(version))
{
continue;
}
if (context.EnableDeepMetadataBypass)
{
packages.Add((PackageBase)catalogObjects[version]);
}
else
{
PackageBase pb = GetPackageFromCatalogUrl((string)catalogObjects[version], request, packageSemanticVersions, context);
if (pb != null)
{
packages.Add(pb);
}
}
}
}
}
else
{
// In some implementations (lookin' at you MyGet) there's no catalogEntry object
PackageBase pb = ConcurrentInMemoryCache.Instance.GetOrAdd <PackageBase>(registrationUrl, () =>
{
if (!root.HasProperty("catalogentry"))
{
if ((packageSemanticVersions == null || packageSemanticVersions.Contains(new SemanticVersion(root.version))))
{
return(this.ResourcesCollection.PackageConverter.Make(root));
}
else
{
return(null);
}
}
else
{
return(GetPackageFromCatalogUrl(this.ResourcesCollection.CatalogUrlConverter.Make(root), request, packageSemanticVersions, context));
}
});
if (pb != null)
{
packages.Add(pb);
}
}
}
else
{
request.Warning(Messages.JsonSchemaMismatch, "type");
request.Debug(Messages.JsonObjectDump, DynamicJsonParser.Serialize(root));
}
if (context.RequiredVersion == null && context.MinimumVersion == null && context.MaximumVersion == null && packages != null)
{
PackageBase absoluteLatestPackage = packages.Where(p => p.IsPrerelease).OrderByDescending(pb => ((IPackage)pb).Version).FirstOrDefault();
if (absoluteLatestPackage != null)
{
absoluteLatestPackage.IsAbsoluteLatestVersion = true;
}
PackageBase latestPackage = packages.Where(p => !p.IsPrerelease).OrderByDescending(pb => ((IPackage)pb).Version).FirstOrDefault();
if (latestPackage != null)
{
latestPackage.IsLatestVersion = true;
}
}
}
else if (finalAttempt)
{
// This is the last retry of this URL. It's definitely not a good one.
ConcurrentInMemoryCache.Instance.GetOrAdd <PackageBase>(registrationUrl, () => null);
}
return(packages);
}
/// <summary>
/// Make sure the Nuget required xml tags exist witin the metadata tag in the nuspec file.
/// </summary>
/// <param name="package"></param>
private static void EnsureRequiredXmlTags(PackageBase package)
{
//As a .nupec, id", "version", "authors", "description" tags are required.
//Check if they exists
if (string.IsNullOrWhiteSpace(package.Version) ||
string.IsNullOrWhiteSpace(package.Id) ||
string.IsNullOrWhiteSpace(package.Authors) ||
string.IsNullOrWhiteSpace(package.Description))
{
//bad .nuspec xml format
string message = string.Format(CultureInfo.InvariantCulture, "{0}: '{1}', '{2}, '{3}', '{4}'", Messages.NuspecRequiredXmlElementMissing, "Version", "Id", "authors", "description");
throw new InvalidDataException(message);
}
}
// parse the list of installed packages
private IEnumerable<PackageBase> GetInstalledPackagesOptionValue()
{
// get possible installed packages
var installedPackages = GetOptionValues("InstalledPackages") ?? new string[0];
// parse the installed package options passed in
foreach (var installedPackage in installedPackages)
{
// we assume that the name passed in is something like jquery#1.2.5
string[] nameAndVersion = installedPackage.Split(new[] { "!#!" }, StringSplitOptions.None);
var package = new PackageBase();
// only name passed in, no version
if (nameAndVersion.Count() == 1)
{
package.Id = nameAndVersion[0];
yield return package;
}
else if (nameAndVersion.Count() == 2)
{
// this means there is version
SemanticVersion semVers = null;
try
{
// convert version to semvers
semVers = new SemanticVersion(nameAndVersion[1]);
}
catch
{
// not a valid version, ignores this entry
continue;
}
// set name and version of this installed packages
package.Id = nameAndVersion[0];
package.Version = semVers.Version.ToStringSafe();
yield return package;
}
}
}
/// <summary>
/// Send the request to the server with buffer size to account for the case where there are more data
/// that we need to fetch
/// </summary>
/// <param name="query"></param>
/// <param name="request"></param>
/// <returns></returns>
public static IEnumerable<PackageBase> SendRequest(string query, Request request)
{
const int bufferSize = 40;
// number of threads sending the requests
const int numberOfSenders = 4;
var startPoint = 0;
var tasks = new List<Task<Stream>>();
bool stopSending = false;
object stopLock = new Object();
// Send one request first
// this initial query is of the form http://www.nuget.org/api/v2/FindPackagesById()?id='jquery'&$skip={0}&$top={1}
UriBuilder initialQuery = new UriBuilder(query.InsertSkipAndTop());
// Send out an initial request
using (Stream stream = NuGetClient.InitialDownloadDataToStream(initialQuery, startPoint, bufferSize, request))
{
if (stream == null)
{
yield break;
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry").ToList();
// If the initial request has less entries than the buffer size, return it because there won't be any more data
if (entries.Count < bufferSize)
{
request.Debug(Messages.PackagesReceived, entries.Count);
stopSending = true;
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
PackageUtility.ReadEntryElement(ref package, entry);
yield return package;
}
}
if (stopSending || request.IsCanceled)
{
yield break;
}
// To avoid more redirection (for example, if the initial query is nuget.org, it will be changed to www.nuget.org
query = initialQuery.Uri.ToString();
// Sending the initial requests
for (var i = 0; i < numberOfSenders; i++)
{
// Update the start point to fetch the packages
startPoint += bufferSize;
// Get the query
var newQuery = string.Format(query, startPoint, bufferSize);
// Send it
tasks.Add(Task.Factory.StartNew(() =>
{
Stream items = NuGetClient.DownloadDataToStream(newQuery, request);
return items;
}));
}
//Wait for the responses, parse the data, and send to the user
while (tasks.Count > 0)
{
//Cast because the compiler warning: Co-variant array conversion from Task[] to Task[] can cause run-time exception on write operation.
var index = Task.WaitAny(tasks.Cast<Task>().ToArray());
using (Stream stream = tasks[index].Result)
{
if (stream == null)
{
yield break;
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry").ToList();
if (entries.Count < bufferSize)
{
request.Debug(Messages.PackagesReceived, entries.Count);
lock (stopLock)
{
stopSending = true;
}
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
PackageUtility.ReadEntryElement(ref package, entry);
yield return package;
}
}
// checks whether we should stop sending requests
if (!stopSending && !request.IsCanceled)
{
// Make sure nobody else is updating the startPoint
lock (stopLock)
{
// update the startPoint
startPoint += bufferSize;
}
// Make a new request with the new startPoint
var newQuery = string.Format(query, startPoint, bufferSize);
//Keep sending a request
tasks[index] = (Task.Factory.StartNew(searchQuery =>
{
var items = NuGetClient.DownloadDataToStream(searchQuery.ToStringSafe(), request);
return items;
}, newQuery));
}
else
{
if (request.IsCanceled)
{
request.Warning(Messages.RequestCanceled, "HttpClientPackageRepository", "SendRequest");
//stop sending request to the remote server
stopSending = true;
}
tasks.RemoveAt(index);
}
}
}
