private IEnumerable <IPackage> SearchPackagesWithBackup(string baseUrl, HttpQueryBuilder qb, RequestWrapper request, NuGetSearchContext searchContext, NuGetSearchTerm searchTerm) { // First execute the actual search HashSet <string> foundPackageIds = new HashSet <string>(); return(NuGetWebUtility.GetResults <dynamic, PackageBase>(request, (dynamic root) => { long res = -1; if (root.HasProperty("totalhits")) { res = root.totalhits; request.Debug(Resources.Messages.TotalPackagesDiscovered, res); } else { request.Warning(Resources.Messages.JsonSchemaMismatch, "totalhits"); request.Debug(Resources.Messages.JsonObjectDump, DynamicJsonParser.Serialize(root)); } return res; }, (dynamic root) => GetPackageCollectionsForSearchResult(root, searchContext, searchTerm, foundPackageIds, request), (long packagesToSkip) => { if (packagesToSkip > 0) { HttpQueryBuilder currentQb = qb.CloneAdd(Constants.SkipQueryParam, packagesToSkip.ToString()); return currentQb.AddQueryString(baseUrl); } return qb.AddQueryString(baseUrl); }, (string content) => { return DynamicJsonParser.Parse(content); }, Constants.SearchPageCountInt)); }
public IEnumerable <string> Autocomplete(NuGetSearchTerm autocompleteSearchTerm, RequestWrapper request, WildcardPattern acceptedPattern, bool allowPrerelease) { try { request.Debug(Messages.DebugInfoCallMethod3, "NuGetAutoCompleteFeed3", "Autocomplete", autocompleteSearchTerm.ToString()); return(base.Execute <IEnumerable <string> >((baseUrl) => { HttpQueryBuilder qb = new HttpQueryBuilder(); if (autocompleteSearchTerm.Term == NuGetSearchTerm.NuGetSearchTermType.Id) { qb.Add(Constants.PackageIdQueryParam, autocompleteSearchTerm.Text); } else if (autocompleteSearchTerm.Term == NuGetSearchTerm.NuGetSearchTermType.AutoComplete) { qb.Add(Constants.QueryQueryParam, autocompleteSearchTerm.Text); } if (allowPrerelease) { qb.Add(Constants.PrereleaseQueryParam, "true"); } qb.Add(Constants.TakeQueryParam, Constants.SearchPageCount) .Add(Constants.SemVerLevelQueryParam, Constants.SemVerLevel2); return NuGetWebUtility.GetResults <dynamic, string>(request, (dynamic root) => { long res = -1; if (root.HasProperty("totalhits")) { res = root.totalhits; request.Debug(Resources.Messages.TotalPackagesDiscovered, res); } else { request.Warning(Resources.Messages.JsonSchemaMismatch, "totalhits"); request.Debug(Resources.Messages.JsonObjectDump, DynamicJsonParser.Serialize(root)); } return res; }, (dynamic root) => GetAutoCompleteResults(root, autocompleteSearchTerm, acceptedPattern), (long resultsToSkip) => { if (resultsToSkip > 0) { HttpQueryBuilder currentQb = qb.CloneAdd(Constants.SkipQueryParam, resultsToSkip.ToString()); return currentQb.AddQueryString(baseUrl); } return qb.AddQueryString(baseUrl); }, (string content) => { return DynamicJsonParser.Parse(content); }, Constants.SearchPageCountInt); })); } finally { request.Debug(Messages.DebugInfoReturnCall, "NuGetAutoCompleteFeed3", "Autocomplete"); } }
private static INuGetResourceCollection GetResourcesImpl(string baseUrl, RequestWrapper request) { INuGetResourceCollection res = null; HttpClient client = request.GetClientWithHeaders(); HttpResponseMessage response = PathUtility.GetHttpResponse(client, baseUrl, (() => request.IsCanceled()), ((msg, num) => request.Verbose(Resources.Messages.RetryingDownload, msg, num)), (msg) => request.Verbose(msg), (msg) => request.Debug(msg)); if (!response.IsSuccessStatusCode) { throw new Exception(Resources.Messages.NuGetEndpointDiscoveryFailed); } string content = new StreamReader(NuGetClient.DownloadDataToStream(baseUrl, request)).ReadToEnd(); // If the response starts with the magic XML header, it's v2 if (content.StartsWith(Constants.XmlStartContent)) { res = NuGetResourceCollection2.Make(baseUrl); } else { try { dynamic root = DynamicJsonParser.Parse(content); string version = root.version; if (version != null && version.StartsWith("3.")) { // v3 feed res = NuGetResourceCollection3.Make(root, baseUrl, request); } } catch (Exception ex) { Exception discoveryException = new Exception(Resources.Messages.NuGetEndpointDiscoveryFailed, ex); throw discoveryException; } } if (res == null) { // Couldn't figure out what this is throw new Exception(Resources.Messages.NuGetEndpointDiscoveryFailed); } return(res); }
public PackageEntryInfo GetVersionInfo(PackageEntryInfo packageInfo, RequestWrapper request) { try { request.Debug(Messages.DebugInfoCallMethod, "NuGetFilesFeed3", "GetVersionInfo"); if (packageInfo == null) { throw new ArgumentNullException("packageInfo"); } if (request == null) { throw new ArgumentNullException("request"); } string query = String.Format(CultureInfo.InvariantCulture, Constants.VersionIndexTemplate, this.baseUrl, this.baseUrl.EndsWith("/") ? String.Empty : "/", packageInfo.Id.ToLowerInvariant()); Stream queryResponse = NuGetClient.DownloadDataToStream(query, request, ignoreNullResponse: true); if (queryResponse != null) { dynamic root = DynamicJsonParser.Parse(new StreamReader(queryResponse).ReadToEnd()); if (root.HasProperty("versions")) { foreach (string v in root.versions) { packageInfo.AddVersion(new SemanticVersion(v)); } } else { request.Debug(Messages.VersionIndexDownloadFailed, packageInfo.Id); } } else { request.Debug(Messages.VersionIndexDownloadFailed, packageInfo.Id); } } finally { request.Debug(Messages.DebugInfoReturnCall, "NuGetFilesFeed3", "GetVersionInfo"); } return(packageInfo); }
private PackageBase GetPackageFromCatalogUrl(string catalogUrl, RequestWrapper request, HashSet <SemanticVersion> packageSemanticVersions, NuGetSearchContext context) { Stream catalogResponseStream = NuGetClient.DownloadDataToStream(catalogUrl, request); if (catalogResponseStream != null) { string content = new StreamReader(catalogResponseStream).ReadToEnd(); dynamic catalogContent = DynamicJsonParser.Parse(content); if ((packageSemanticVersions == null || packageSemanticVersions.Contains(new SemanticVersion(catalogContent.version)))) { PackageBase pb = this.ResourcesCollection.PackageConverter.Make(DynamicJsonParser.Parse(content), context.PackageInfo); if (pb != null) { return(pb); } } } else { request.Warning(Messages.CouldNotGetResponseFromQuery, catalogUrl); } return(null); }
/// <summary> /// Find packages when the registration URL is already known. /// </summary> /// <param name="registrationUrl"></param> /// <param name="request"></param> /// <returns></returns> internal IEnumerable <PackageBase> Find(string registrationUrl, NuGetSearchContext context, RequestWrapper request, bool finalAttempt) { request.Debug(Messages.DebugInfoCallMethod3, "NuGetPackageFeed3", "Find", registrationUrl); List <PackageBase> packages = null; PackageBase cachedPackage; if (!registrationUrl.Contains("index.json") && ConcurrentInMemoryCache.Instance.TryGet <PackageBase>(registrationUrl, out cachedPackage)) { if (cachedPackage == null) { return(packages); } else { packages = new List <PackageBase>() { cachedPackage }; return(packages); } } Stream s = NuGetClient.DownloadDataToStream(registrationUrl, request, ignoreNullResponse: true, tries: 1); if (s != null) { packages = new List <PackageBase>(); // Now we can get the catalog entry dynamic root = DynamicJsonParser.Parse(new StreamReader(s).ReadToEnd()); if (root.Metadata.HasProperty("type")) { // First check if this is a Package or PackageRegistration type // Package is from packageId + version // PackageRegistration is from packageId bool isRegistrationType = false; foreach (string t in root.Metadata.type) { if (t.Equals("PackageRegistration", StringComparison.OrdinalIgnoreCase)) { isRegistrationType = true; break; } } if (context.PackageInfo.AllVersions.Count == 0) { // Only when the version list is restricted is this method usually faster this.ResourcesCollection.FilesFeed.GetVersionInfo(context.PackageInfo, request); } HashSet <SemanticVersion> packageSemanticVersions = null; if (context.PackageInfo.AllVersions.Count > 0) { packageSemanticVersions = FilterVersionsByRequirements(context, context.PackageInfo); } if (isRegistrationType) { // This is a registration index, like: "https://api.nuget.org/v3/registration3/json/index.json" // Get all versions from files service // In addition, when DeepMetadataBypass is enabled, we MUST use the registration index to get package info // If a call to -Name -RequiredVersion is done, DeepMetadataBypass will never be enabled (for now) // If we wanted, we could enable this by checking if !isRegistrationType && context.EnableDeepMetadataBypass, then call into Find with the registration index URL if (!context.AllVersions && packageSemanticVersions != null && !context.EnableDeepMetadataBypass) { foreach (SemanticVersion packageVersion in packageSemanticVersions) { NuGetSearchResult result = this.ResourcesCollection.PackagesFeed.Find(new NuGetSearchContext() { PackageInfo = context.PackageInfo, RequiredVersion = packageVersion, EnableDeepMetadataBypass = context.EnableDeepMetadataBypass }, request); PackageBase package = result.Result == null ? null : result.Result.FirstOrDefault() as PackageBase; if (package != null) { packages.Add(package); } } } else { // Going to collect versions from the registration index in here // Map of package version -> either PackageBase (if context.EnableDeepMetadataBypass) or catalog URL Dictionary <SemanticVersion, object> catalogObjects = new Dictionary <SemanticVersion, object>(); // If the version list hasn't been built yet, we can build it from the registration page instead of using FilesFeed bool buildPackageInfoVersions = context.PackageInfo.AllVersions.Count == 0; // Fallback to catalog crawling in these cases: // - Bypass deep metadata // - Getting version list failed // - All versions required foreach (dynamic catalogPage in root.items) { dynamic actualCatalogPage = catalogPage; if (!actualCatalogPage.HasProperty("items")) { // Sometimes the catalog page on the PackageRegistration entry doesn't have the actual page contents // In this case, the ID metadata tag points to the full catalog entry Stream fullCatalogPageResponseStream = NuGetClient.DownloadDataToStream(actualCatalogPage.Metadata.id, request); if (fullCatalogPageResponseStream != null) { actualCatalogPage = DynamicJsonParser.Parse(new StreamReader(fullCatalogPageResponseStream).ReadToEnd()); } } foreach (dynamic packageEntry in actualCatalogPage.items) { SemanticVersion version = new SemanticVersion(packageEntry.catalogentry.version); if (buildPackageInfoVersions) { context.PackageInfo.AddVersion(version); } bool hasCatalogEntry = packageEntry.HasProperty("catalogentry"); if (context.EnableDeepMetadataBypass || !hasCatalogEntry) { // Bypass retrieving "deep" (but required) metadata like packageHash // Also do this if there's no catalog entry PackageBase pb = null; if (packageEntry.HasProperty("catalogentry")) { pb = this.ResourcesCollection.PackageConverter.Make(packageEntry.catalogentry, context.PackageInfo); } else { // In some implementations (lookin' at you MyGet) there's no catalogEntry object pb = this.ResourcesCollection.PackageConverter.Make(packageEntry, context.PackageInfo); } if (pb != null) { catalogObjects[version] = pb; } } else { catalogObjects[version] = this.ResourcesCollection.CatalogUrlConverter.Make(packageEntry); } } } packageSemanticVersions = FilterVersionsByRequirements(context, context.PackageInfo); foreach (SemanticVersion version in packageSemanticVersions) { if (!catalogObjects.ContainsKey(version)) { continue; } if (context.EnableDeepMetadataBypass) { packages.Add((PackageBase)catalogObjects[version]); } else { PackageBase pb = GetPackageFromCatalogUrl((string)catalogObjects[version], request, packageSemanticVersions, context); if (pb != null) { packages.Add(pb); } } } } } else { // In some implementations (lookin' at you MyGet) there's no catalogEntry object PackageBase pb = ConcurrentInMemoryCache.Instance.GetOrAdd <PackageBase>(registrationUrl, () => { if (!root.HasProperty("catalogentry")) { if ((packageSemanticVersions == null || packageSemanticVersions.Contains(new SemanticVersion(root.version)))) { return(this.ResourcesCollection.PackageConverter.Make(root)); } else { return(null); } } else { return(GetPackageFromCatalogUrl(this.ResourcesCollection.CatalogUrlConverter.Make(root), request, packageSemanticVersions, context)); } }); if (pb != null) { packages.Add(pb); } } } else { request.Warning(Messages.JsonSchemaMismatch, "type"); request.Debug(Messages.JsonObjectDump, DynamicJsonParser.Serialize(root)); } if (context.RequiredVersion == null && context.MinimumVersion == null && context.MaximumVersion == null && packages != null) { PackageBase absoluteLatestPackage = packages.Where(p => p.IsPrerelease).OrderByDescending(pb => ((IPackage)pb).Version).FirstOrDefault(); if (absoluteLatestPackage != null) { absoluteLatestPackage.IsAbsoluteLatestVersion = true; } PackageBase latestPackage = packages.Where(p => !p.IsPrerelease).OrderByDescending(pb => ((IPackage)pb).Version).FirstOrDefault(); if (latestPackage != null) { latestPackage.IsLatestVersion = true; } } } else if (finalAttempt) { // This is the last retry of this URL. It's definitely not a good one. ConcurrentInMemoryCache.Instance.GetOrAdd <PackageBase>(registrationUrl, () => null); } return(packages); }