/// <summary>
/// Ctor's
/// </summary>
/// <param name="request">The nuget request object</param>
/// <param name="queryUrl">Packagesource location</param>
internal HttpClientPackageRepository(string queryUrl, Request request)
{
// Validate the url
Uri newUri;
Uri validatedUri = null;
if (Uri.TryCreate(queryUrl, UriKind.Absolute, out newUri))
{
validatedUri = PathUtility.ValidateUri(newUri, request);
}
if (validatedUri == null)
{
throw new ArgumentException(String.Format(CultureInfo.CurrentCulture, Messages.InvalidQueryUrl, queryUrl));
}
queryUrl = validatedUri.AbsoluteUri;
// if a query is http://www.nuget.org/api/v2 then we add / to the end
if (!queryUrl.EndsWith("/", StringComparison.OrdinalIgnoreCase))
{
queryUrl = String.Concat(queryUrl, "/");
}
_queryUri = queryUrl;
//we are constructing the url query like http://www.nuget.org/api/v2/FindPackagesById()?id='JQuery'
_nugetFindPackageIdQueryFormat = PathUtility.UriCombine(_queryUri, NuGetConstant.FindPackagesById);
}
/// <summary>
/// Find the package via the given uri query.
/// </summary>
/// <param name="query">A full Uri. A sample Uri looks like "http://www.nuget.org/api/v2/FindPackagesById()?id='Jquery'" </param>
/// <param name="request">An object passed in from the PackageManagement that contains functions that can be used to interact with its Provider</param>
/// <returns>Package objects</returns>
internal static IEnumerable<PackageBase> FindPackage(string query, Request request) {
request.Debug(Messages.DebugInfoCallMethod, "NuGetClient", "FindPackage");
request.Verbose(Messages.SearchingRepository, query, "");
return HttpClientPackageRepository.SendRequest(query, request);
}
/// <summary>
/// Finding the packages in the file repository
/// </summary>
/// <param name="openPackage">Delegate function which is actually finding a package</param>
/// <param name="packageId">Package Id</param>
/// <param name="packagePaths">File repository path</param>
/// <param name="request"></param>
/// <returns></returns>
private static IEnumerable<IPackage> GetPackages(Func<string, Request, IPackage> openPackage,
string packageId,
IEnumerable<string> packagePaths,
Request request)
{
request.Debug(Resources.Messages.DebugInfoCallMethod3, "LocalPackageRepository", "GetPackages", packageId);
foreach (var path in packagePaths)
{
IPackage package = null;
try {
package = GetPackage(openPackage, path, request);
} catch (InvalidOperationException ex) {
// ignore error for unzipped packages (nuspec files).
if (!string.Equals(NuGetConstant.ManifestExtension, Path.GetExtension(path), StringComparison.OrdinalIgnoreCase)) {
request.Verbose(ex.Message);
throw;
}
}
if (package != null && package.Id.Equals(packageId, StringComparison.OrdinalIgnoreCase)) {
yield return package;
}
}
}
/// <summary>
/// Finding the package in the file repository
/// </summary>
/// <param name="openPackage">Delegate function which is actually finding a package</param>
/// <param name="path">File repository path</param>
/// <param name="request"></param>
/// <returns></returns>
private static IPackage GetPackage(Func<string, Request, IPackage> openPackage, string path, Request request)
{
//Create the package
IPackage package = openPackage(path, request);
return package;
}
public virtual IPackageRepository CreateRepository(string packageSource, Request request)
{
if (packageSource == null)
{
throw new ArgumentNullException("packageSource");
}
Uri uri = new Uri(packageSource);
if (uri.IsFile)
{
return new LocalPackageRepository(uri.LocalPath, request);
}
return new HttpClientPackageRepository(packageSource, request);
}
/// <summary>
/// Find-Package
/// </summary>
/// <param name="packageId">package Id</param>
/// <param name="version">package version</param>
/// <param name="request"></param>
/// <returns></returns>
public IPackage FindPackage(string packageId, SemanticVersion version, Request request)
{
if (string.IsNullOrWhiteSpace(packageId)) {
return null;
}
request.Debug(Messages.DebugInfoCallMethod3, "HttpClientPackageRepository", "FindPackage", packageId);
var query = packageId.MakeFindPackageByIdQuery(_nugetFindPackageIdQueryFormat);
var packages = NuGetClient.FindPackage(query, request);
//Usually versions has a limited number, ToArray should be ok.
var versions = version.GetComparableVersionStrings().ToArray();
//Will only enumerate oackages once
return packages.FirstOrDefault(package => packageId.Equals(package.Id, StringComparison.OrdinalIgnoreCase) && versions.Contains(package.Version,StringComparer.OrdinalIgnoreCase));
}
private IEnumerable<IPackage> SearchImpl(string searchTerm, Request request) {
var nugetRequest = request as NuGetRequest;
if (nugetRequest == null) {
yield break;
}
nugetRequest.Debug(Resources.Messages.SearchingRepository, "LocalPackageRepository", Source);
var files = Directory.GetFiles(Source);
foreach (var package in nugetRequest.FilterOnTags(files.Select(nugetRequest.GetPackageByFilePath).Where(pkgItem => pkgItem != null).Select(pkg => pkg.Package)))
{
yield return package;
}
// look in the package source location for directories that contain nupkg files.
var subdirs = Directory.EnumerateDirectories(Source, "*", SearchOption.AllDirectories);
foreach (var subdir in subdirs) {
var nupkgs = Directory.EnumerateFileSystemEntries(subdir, "*.nupkg", SearchOption.TopDirectoryOnly);
foreach (var package in nugetRequest.FilterOnTags(nupkgs.Select(nugetRequest.GetPackageByFilePath).Where(pkgItem => pkgItem != null).Select(pkg => pkg.Package)))
{
yield return package;
}
}
}
/// <summary>
/// Search the entire repository for the case when a user does not provider package name or uses wildcards in the name.
/// </summary>
/// <param name="searchTerm">The Searchterm</param>
/// <param name="request"></param>
/// <returns></returns>
public IEnumerable<IPackage> Search(string searchTerm, Request request) {
var packages = SearchImpl(searchTerm, request);
if (packages == null) {
return Enumerable.Empty<IPackage>();
}
var nugetRequest = request as NuGetRequest;
if (nugetRequest != null && nugetRequest.AllVersions.Value) {
//return whatever we can find
return packages;
}
//return the lastest version
return packages.GroupBy(p => p.Id).Select(each => each.OrderByDescending(pp => pp.Version).FirstOrDefault());
}
/// <summary>
/// Find-Package based the given Id
/// </summary>
/// <param name="packageId">Package Id</param>
/// <param name="request"></param>
/// <returns></returns>
public IEnumerable<IPackage> FindPackagesById(string packageId, Request request) {
return FindPackagesById(OpenPackage, packageId, request);
}
/// <summary>
/// Ctor's
/// </summary>
/// <param name="request"></param>
/// <param name="physicalPath"></param>
public LocalPackageRepository(string physicalPath, Request request) {
_path = physicalPath;
}
/// <summary>
/// Find-Package
/// </summary>
/// <param name="openPackage">Delegate function which is actually finding a package</param>
/// <param name="packageId">Package Id</param>
/// <param name="version">Package version</param>
/// <param name="request"></param>
/// <returns></returns>
private IPackage FindPackage(Func<string, Request, IPackage> openPackage, string packageId, SemanticVersion version, Request request)
{
var nugetRequest = request as NuGetRequest;
if (nugetRequest == null) {
return null;
}
nugetRequest.Debug(Resources.Messages.SearchingRepository, "FindPackage", packageId);
var lookupPackageName = new PackageName(packageId, version);
//handle file cache here if we want to support local package cache later
// Lookup files which start with the name "<Id>." and attempt to match it with all possible version string combinations (e.g. 1.2.0, 1.2.0.0)
// before opening the package. To avoid creating file name strings, we attempt to specifically match everything after the last path separator
// which would be the file name and extension.
return (from path in GetPackageLookupPaths(packageId, version)
let package = GetPackage(openPackage, path, request)
where lookupPackageName.Equals(new PackageName(package.Id, package.Version))
select package).FirstOrDefault();
}
/// <summary>
/// Search the entire repository for the case when a user does not provider package name or uses wildcards in the name.
/// </summary>
/// <param name="searchTerm"></param>
/// <param name="request"></param>
/// <returns></returns>
public IEnumerable<IPackage> Search(string searchTerm, Request request)
{
var nugetRequest = request as NuGetRequest;
if (nugetRequest == null)
{
yield break;
}
nugetRequest.Debug(Messages.DebugInfoCallMethod3, "HttpClientPackageRepository", "Search", searchTerm);
var searchQuery = searchTerm.MakeSearchQuery(_queryUri, nugetRequest.AllowPrereleaseVersions.Value, nugetRequest.AllVersions.Value);
foreach (var pkg in SendRequest(searchQuery, request))
{
yield return pkg;
}
}
/// <summary>
/// Returns a collection of strings to the client advertizing features this provider supports.
/// </summary>
/// <param name="request">An object passed in from the PackageManagement that contains functions that can be used to interact with its Provider</param>
public void GetFeatures(Request request)
{
if (request == null){
throw new ArgumentNullException("request");
}
request.Debug(Resources.Messages.DebugInfoCallMethod, PackageProviderName, "GetFeatures");
request.Yield(Features);
}
/// <summary>
/// A delegate used for finding the package.
/// </summary>
/// <param name="path">File repository path</param>
/// <param name="request"></param>
/// <returns></returns>
private static IPackage OpenPackage(string path, Request request)
{
request.Debug(Resources.Messages.DebugInfoCallMethod3, "LocalPackageRepository", "OpenPackage", path);
if (!File.Exists(path)) {
request.Warning(Resources.Messages.FileNotFound, path,"LocalPackageRepository::OpenPackage");
return null;
}
//deal with .nupkg
if (string.Equals(Path.GetExtension(path), NuGetConstant.PackageExtension, StringComparison.OrdinalIgnoreCase)) {
PackageBase package;
try {
package = ProcessZipPackage(path);
} catch (Exception ex) {
request.Verbose(ex.Message);
throw;
}
// Set the last modified date on the package
package.Published = FileUtility.GetLastModified(path);
// We assume local files in the local repository are all latest version
package.IsAbsoluteLatestVersion = true;
package.IsLatestVersion = true;
package.FullFilePath = path;
return package;
}
return null;
}
/// <summary>
/// Send the request to the server with buffer size to account for the case where there are more data
/// that we need to fetch
/// </summary>
/// <param name="query"></param>
/// <param name="request"></param>
/// <returns></returns>
public static IEnumerable<PackageBase> SendRequest(string query, Request request)
{
const int bufferSize = 40;
// number of threads sending the requests
const int numberOfSenders = 4;
var startPoint = 0;
var tasks = new List<Task<Stream>>();
bool stopSending = false;
object stopLock = new Object();
// Send one request first
// this initial query is of the form http://www.nuget.org/api/v2/FindPackagesById()?id='jquery'&$skip={0}&$top={1}
UriBuilder initialQuery = new UriBuilder(query.InsertSkipAndTop());
// Send out an initial request
using (Stream stream = NuGetClient.InitialDownloadDataToStream(initialQuery, startPoint, bufferSize, request))
{
if (stream == null)
{
yield break;
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry").ToList();
// If the initial request has less entries than the buffer size, return it because there won't be any more data
if (entries.Count < bufferSize)
{
request.Debug(Messages.PackagesReceived, entries.Count);
stopSending = true;
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
PackageUtility.ReadEntryElement(ref package, entry);
yield return package;
}
}
if (stopSending || request.IsCanceled)
{
yield break;
}
// To avoid more redirection (for example, if the initial query is nuget.org, it will be changed to www.nuget.org
query = initialQuery.Uri.ToString();
// Sending the initial requests
for (var i = 0; i < numberOfSenders; i++)
{
// Update the start point to fetch the packages
startPoint += bufferSize;
// Get the query
var newQuery = string.Format(query, startPoint, bufferSize);
// Send it
tasks.Add(Task.Factory.StartNew(() =>
{
Stream items = NuGetClient.DownloadDataToStream(newQuery, request);
return items;
}));
}
//Wait for the responses, parse the data, and send to the user
while (tasks.Count > 0)
{
//Cast because the compiler warning: Co-variant array conversion from Task[] to Task[] can cause run-time exception on write operation.
var index = Task.WaitAny(tasks.Cast<Task>().ToArray());
using (Stream stream = tasks[index].Result)
{
if (stream == null)
{
yield break;
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry").ToList();
if (entries.Count < bufferSize)
{
request.Debug(Messages.PackagesReceived, entries.Count);
lock (stopLock)
{
stopSending = true;
}
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
PackageUtility.ReadEntryElement(ref package, entry);
yield return package;
}
}
// checks whether we should stop sending requests
if (!stopSending && !request.IsCanceled)
{
// Make sure nobody else is updating the startPoint
lock (stopLock)
{
// update the startPoint
startPoint += bufferSize;
}
// Make a new request with the new startPoint
var newQuery = string.Format(query, startPoint, bufferSize);
//Keep sending a request
tasks[index] = (Task.Factory.StartNew(searchQuery =>
{
var items = NuGetClient.DownloadDataToStream(searchQuery.ToStringSafe(), request);
return items;
}, newQuery));
}
else
{
if (request.IsCanceled)
{
request.Warning(Messages.RequestCanceled, "HttpClientPackageRepository", "SendRequest");
//stop sending request to the remote server
stopSending = true;
}
tasks.RemoveAt(index);
}
}
}
/// <summary>
/// Send an initial request to download data from the server.
/// From the initial request, we may change the host of subsequent calls (if a redirection happens in this initial request)
/// Also, if the initial request sends us less data than the amount we request, then we do not
/// need to issue more requests
/// </summary>
/// <param name="query"></param>
/// <param name="startPoint"></param>
/// <param name="bufferSize"></param>
/// <param name="request"></param>
/// <returns></returns>
internal static Stream InitialDownloadDataToStream(UriBuilder query, int startPoint, int bufferSize, Request request)
{
var uri = String.Format(CultureInfo.CurrentCulture, query.Uri.ToString(), startPoint, bufferSize);
request.Debug(Messages.DownloadingPackage, uri);
var client = GetHttpClient(request);
var response = PathUtility.GetHttpResponse(client, uri, request);
// Check that response was successful or write error
if (response == null || !response.IsSuccessStatusCode)
{
request.Warning(Resources.Messages.CouldNotGetResponseFromQuery, uri);
return null;
}
// Read response and write out a stream
var stream = GetStreamBasedOnEncoding(response);
request.Debug(Messages.CompletedDownload, uri);
// If the host from the response is different, change the host of the original query
if (!String.Equals(response.RequestMessage.RequestUri.Host, query.Host, StringComparison.OrdinalIgnoreCase))
{
query.Host = response.RequestMessage.RequestUri.Host;
}
return stream;
}
/// <summary>
/// Download data from remote via uri query.
/// </summary>
/// <param name="query">Uri query</param>
/// <param name="request">An object passed in from the PackageManagement platform that contains APIs that can be used to interact with it </param>
/// <returns></returns>
internal static Stream DownloadDataToStream(string query, Request request)
{
request.Debug(Messages.DownloadingPackage, query);
var client = GetHttpClient(request);
var response = PathUtility.GetHttpResponse(client, query, request);
// Check that response was successful or throw exception
if (response == null || !response.IsSuccessStatusCode)
{
request.Warning(Resources.Messages.CouldNotGetResponseFromQuery, query);
return null;
}
// Read response and write out a stream
var stream = GetStreamBasedOnEncoding(response);
request.Debug(Messages.CompletedDownload, query);
return stream;
}
/// <summary>
/// Returns a httpclient with the headers set.
/// </summary>
/// <returns></returns>
private static HttpClient GetHttpClient(Request request) {
HttpClient client = new HttpClient(new HttpClientHandler() { UseDefaultCredentials = true });
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Charset", "UTF-8");
// Request for gzip and deflate encoding to make the response lighter.
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip,deflate");
client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "NuGet");
if (request is NuGetRequest)
{
var nuGetRequest = request as NuGetRequest;
if (nuGetRequest != null && nuGetRequest.Headers != null)
{
foreach (var header in nuGetRequest.Headers.Value)
{
// header is in the format "A=B" because OneGet doesn't support Dictionary parameters
if (!String.IsNullOrEmpty(header))
{
var headerSplit = header.Split(new string[] { "=" }, 2, StringSplitOptions.RemoveEmptyEntries);
// ignore wrong entries
if (headerSplit.Count() == 2)
{
client.DefaultRequestHeaders.TryAddWithoutValidation(headerSplit[0], headerSplit[1]);
}
else
{
request.Warning(Messages.HeaderIgnored, header);
}
}
}
}
}
return client;
}
private IEnumerable<PackageItem> GetPackageById(PackageSource source,
string name,
Request request,
string requiredVersion = null,
string minimumVersion = null,
string maximumVersion = null,
bool minInclusive = true,
bool maxInclusive = true) {
try {
Debug(Resources.Messages.DebugInfoCallMethod3, "NuGetRequest", "GetPackageById", name);
// otherwise fall back to traditional behavior
var pkgs = source.Repository.FindPackagesById(name, request);
// if required version not specified then don't use unlisted version
// unlisted version is the one that has published year as 1900
if (string.IsNullOrWhiteSpace(requiredVersion))
{
pkgs = pkgs.Where(pkg => pkg.Published.HasValue && pkg.Published.Value.Year > 1900);
}
if (AllVersions.Value){
//Display versions from lastest to oldest
pkgs = (from p in pkgs select p).OrderByDescending(x => x.Version);
}
//A user does not provide version info, we choose the latest
if (!AllVersions.Value && (String.IsNullOrWhiteSpace(requiredVersion) && String.IsNullOrWhiteSpace(minimumVersion) && String.IsNullOrWhiteSpace(maximumVersion)))
{
if (AllowPrereleaseVersions.Value || source.Repository.IsFile) {
//Handling something like Json.NET 7.0.1-beta3 as well as local repository
pkgs = from p in pkgs
group p by p.Id
into newGroup
select newGroup.Aggregate((current, next) => (next.Version > current.Version) ? next : current);
} else {
pkgs = from p in pkgs where p.IsLatestVersion select p;
}
}
pkgs = FilterOnContains(pkgs);
pkgs = FilterOnTags(pkgs);
var results = FilterOnVersion(pkgs, requiredVersion, minimumVersion, maximumVersion, minInclusive, maxInclusive)
.Select(pkg => new PackageItem {
Package = pkg,
PackageSource = source,
FastPath = MakeFastPath(source, pkg.Id, pkg.Version.ToString())
});
return results;
} catch (Exception e) {
e.Dump(this);
return Enumerable.Empty<PackageItem>();
}
}
/// <summary>
/// Get the package based on given package id
/// </summary>
/// <param name="name">Package id or name</param>
/// <param name="requiredVersion"></param>
/// <param name="minimumVersion"></param>
/// <param name="maximumVersion"></param>
/// <param name="maxInclusive"></param>
/// <param name="minInclusive"></param>
/// <param name="request"></param>
/// <returns></returns>
internal IEnumerable<PackageItem> GetPackageById(string name, Request request, string requiredVersion = null,
string minimumVersion = null, string maximumVersion = null, bool minInclusive = true, bool maxInclusive = true) {
if (String.IsNullOrWhiteSpace(name))
{
return Enumerable.Empty<PackageItem>();
}
return SelectedSources.AsParallel().WithMergeOptions(ParallelMergeOptions.NotBuffered).SelectMany(source => GetPackageById(source, name, request, requiredVersion, minimumVersion, maximumVersion, minInclusive, maxInclusive));
}
/// <summary>
/// Find-Package based the given Id
/// </summary>
/// <param name="openPackage">Delegate function which is actually finding a package</param>
/// <param name="packageId">Package version</param>
/// <param name="request"></param>
/// <returns></returns>
private IEnumerable<IPackage> FindPackagesById(Func<string, Request, IPackage> openPackage, string packageId, Request request)
{
request.Debug(Resources.Messages.DebugInfoCallMethod3, "LocalPackageRepository", "FindPackagesById", packageId);
// get packages in .nupkg or .nuspec files
// add "*" to avoid parsing the version (id: packageName+Version, e.g. jQuery.1.10.0)
return GetPackages(
openPackage,
packageId,
GetPackageFiles(packageId + "*" + NuGetConstant.PackageExtension),
request).Union(
GetPackages(
openPackage,
packageId,
GetPackageFiles(packageId + "*" + NuGetConstant.ManifestExtension),
request));
}
/// <summary>
/// Find-Package
/// </summary>
/// <param name="packageId">Package id</param>
/// <param name="version">Package Name</param>
/// <param name="request"></param>
/// <returns></returns>
public virtual IPackage FindPackage(string packageId, SemanticVersion version, Request request) {
return FindPackage(OpenPackage, packageId, version, request);
}
/// <summary>
/// Returns dynamic option definitions to the HOST
///
/// example response:
/// request.YieldDynamicOption( "MySwitch", OptionType.String.ToString(), false);
///
/// </summary>
/// <param name="category">The category of dynamic options that the HOST is interested in</param>
/// <param name="request">An object passed in from the PackageManagement that contains functions that can be used to interact with its Provider</param>
public void GetDynamicOptions(string category, Request request)
{
if (request == null){
throw new ArgumentNullException("request");
}
if (category == null)
{
request.Warning(Resources.Messages.UnkownCategory, PackageProviderName, "GetDynamicOptions", category);
return;
}
switch ((category ?? string.Empty).ToLowerInvariant()) {
case "install":
// put any options required for install/uninstall/getinstalledpackages
request.YieldDynamicOption("Destination", Constants.OptionType.Folder, false);
request.YieldDynamicOption("ExcludeVersion", Constants.OptionType.Switch, false);
request.YieldDynamicOption("Scope", Constants.OptionType.String, false, new[] { Constants.CurrentUser, Constants.AllUsers });
//request.YieldDynamicOption("SkipDependencies", Constants.OptionType.Switch, false);
//request.YieldDynamicOption("ContinueOnFailure", Constants.OptionType.Switch, false);
break;
case "source":
// put any options for package sources
request.YieldDynamicOption("ConfigFile", Constants.OptionType.String, false);
request.YieldDynamicOption("SkipValidate", Constants.OptionType.Switch, false);
break;
case "package":
// put any options used when searching for packages
request.YieldDynamicOption("Headers", Constants.OptionType.StringArray, false);
request.YieldDynamicOption("FilterOnTag", Constants.OptionType.StringArray, false);
request.YieldDynamicOption("Contains", Constants.OptionType.String, false);
request.YieldDynamicOption("AllowPrereleaseVersions", Constants.OptionType.Switch, false);
break;
default:
request.Warning(Resources.Messages.UnkownCategory, PackageProviderName, "GetDynamicOptions", category);
break;
}
}
/// <summary>
/// Find-Package bases on the given package Id
/// </summary>
/// <param name="packageId">Package Id</param>
/// <param name="request"></param>
/// <returns></returns>
public IEnumerable<IPackage> FindPackagesById(string packageId, Request request){
request.Debug(Messages.DebugInfoCallMethod3, "HttpClientPackageRepository", "FindPackagesById", packageId);
var query = packageId.MakeFindPackageByIdQuery(_nugetFindPackageIdQueryFormat);
//request.Verbose(query.ToString());
return NuGetClient.FindPackage(query, request);
}