public NuGetSearchResult Search(NuGetSearchContext searchContext, NuGetRequest nugetRequest) { nugetRequest.Debug(Messages.DebugInfoCallMethod, "NuGetSearchFeed2", "Search"); if (nugetRequest == null) { return(searchContext.MakeResult()); } return(Search(searchContext, new RequestWrapper(nugetRequest))); }
private IEnumerable <IPackage> FindImpl(NuGetSearchContext findContext, RequestWrapper request) { request.Debug(Messages.DebugInfoCallMethod3, "NuGetPackageFeed3", "FindImpl", findContext.PackageInfo.Id); try { return(base.Execute <IEnumerable <IPackage> >((baseUrl) => GetPackagesForBaseUrl(baseUrl, findContext, request))); } finally { request.Debug(Messages.DebugInfoReturnCall, "NuGetPackageFeed3", "FindImpl"); } }
public NuGetSearchResult Find(NuGetSearchContext findContext, RequestWrapper request) { request.Debug(Messages.DebugInfoCallMethod3, "NuGetPackageFeed3", "Find", findContext.PackageInfo.Id); if (System.Management.Automation.WildcardPattern.ContainsWildcardCharacters(findContext.PackageInfo.Id)) { // Short circuit when there's wildcards - this will never work return(findContext.MakeResult(new List <IPackage>())); } NuGetSearchResult result = findContext.MakeResult(FindImpl(findContext, request), versionPostFilterRequired: false); request.Debug(Messages.DebugInfoReturnCall, "NuGetPackageFeed3", "Find"); return(result); }
public IPackage FindPackage(NuGetSearchContext findContext, NuGetRequest request) { request.Debug(Messages.DebugInfoCallMethod3, "NuGetPackageRepository", "FindPackage", findContext.PackageInfo.Id); NuGetSearchResult result = this.ResourceProvider.PackagesFeed.Find(findContext, request); if (result.Result != null) { return(result.Result.FirstOrDefault()); } else { return(null); } }
private HashSet <SemanticVersion> FilterVersionsByRequirements(NuGetSearchContext findContext, PackageEntryInfo packageInfo) { HashSet <SemanticVersion> set = new HashSet <SemanticVersion>(); if (findContext.MinimumVersion == null && findContext.MaximumVersion == null && findContext.RequiredVersion == null && !findContext.AllVersions) { if (findContext.AllowPrerelease) { set.Add(packageInfo.AbsoluteLatestVersion); } else { set.Add(packageInfo.LatestVersion); } } else { SemanticVersion latestVersion = null; foreach (SemanticVersion v in packageInfo.AllVersions) { if ((findContext.MinimumVersion != null && findContext.MinimumVersion > v) || (findContext.MaximumVersion != null && findContext.MaximumVersion < v) || (findContext.RequiredVersion != null && findContext.RequiredVersion != v)) { continue; } bool isAllowed = findContext.AllowPrerelease || String.IsNullOrWhiteSpace(v.SpecialVersion); if (findContext.AllVersions && isAllowed) { set.Add(v); } else if (isAllowed) { if (latestVersion == null || latestVersion < v) { latestVersion = v; } } } if (latestVersion != null && !findContext.AllVersions) { set.Add(latestVersion); } } return(set); }
public NuGetSearchResult Search(NuGetSearchContext searchContext, RequestWrapper request) { request.Debug(Messages.DebugInfoCallMethod, "NuGetSearchFeed2", "Search"); if (request == null) { return(searchContext.MakeResult()); } string searchString = this.ResourcesCollection.GetSearchQueryDelegate(searchContext.SearchTerms); request.Debug(Messages.DebugInfoCallMethod3, "NuGetSearchFeed2", "Search", searchString); var searchQuery = searchString.MakeSearchQuery(this.baseUrl, searchContext.AllowPrerelease, searchContext.AllVersions); return(searchContext.MakeResult(NuGetWebUtility.SendRequest(searchQuery, request))); }
/// <summary> /// Search the entire repository for the case when a user does not provider package name or uses wildcards in the name. /// </summary> /// <param name="searchTerm">The Searchterm</param> /// <param name="nugetRequest"></param> /// <returns></returns> public NuGetSearchResult Search(NuGetSearchContext searchContext, NuGetRequest nugetRequest) { var packages = SearchImpl(this.ResourceProvider.GetSearchQueryDelegate(searchContext.SearchTerms), nugetRequest); if (packages == null) { return(searchContext.MakeResult(Enumerable.Empty <IPackage>())); } if (nugetRequest != null && nugetRequest.AllVersions.Value) { //return whatever we can find return(searchContext.MakeResult(packages)); } //return the latest version return(searchContext.MakeResult(packages.GroupBy(p => p.Id).Select(each => each.OrderByDescending(pp => pp.Version).FirstOrDefault()))); }
public NuGetSearchResult Search(NuGetSearchContext searchContext, RequestWrapper request) { // This is a search scenario, so it should be safe to skip some metadata for the sake of performance searchContext.EnableDeepMetadataBypass = true; return(base.Execute <NuGetSearchResult>((baseUrl) => { // For now we'll just get all versions and return the latest HttpQueryBuilder qb = new HttpQueryBuilder(); // Once searchTermQb encodes the searchTerm, don't encode the ":" part of the resulting string qb.Add(Constants.QueryQueryParam, this.ResourcesCollection.GetSearchQueryDelegate(searchContext.SearchTerms), separator: "=", encode: false).Add(Constants.TakeQueryParam, Constants.SearchPageCount) .Add(Constants.SemVerLevelQueryParam, Constants.SemVerLevel2); if (searchContext.AllowPrerelease) { qb.Add(Constants.PrereleaseQueryParam, "true"); } NuGetSearchTerm searchTerm = searchContext.SearchTerms.Where(st => st.Term == NuGetSearchTerm.NuGetSearchTermType.SearchTerm).FirstOrDefault(); IEnumerable <IPackage> packages = SearchPackagesWithBackup(baseUrl, qb, request, searchContext, searchTerm); return searchContext.MakeResult(packages, versionPostFilterRequired: true, namePostFilterRequired: false, containsPostFilterRequired: false); })); }
private IEnumerable <IPackage> GetPackagesForBaseUrl(string baseUrl, NuGetSearchContext findContext, RequestWrapper request) { bool urlHit = false; int attempts = 3; while (!urlHit && attempts-- > 0) { foreach (string candidateUrl in GetCandidateUrls(findContext.PackageInfo.Id, findContext.RequiredVersion, baseUrl)) { IEnumerable <IPackage> packages = Find(candidateUrl, findContext, request, attempts == 0); if (packages != null) { urlHit = true; foreach (IPackage package in packages) { yield return(package); } break; } } } }
public NuGetSearchResult Find(NuGetSearchContext findContext, RequestWrapper request) { if (string.IsNullOrWhiteSpace(findContext.PackageInfo.Id)) { return(null); } request.Debug(Messages.DebugInfoCallMethod3, "NuGetPackageFeed2", "FindPackage", findContext.PackageInfo.Id); var query = findContext.PackageInfo.Id.MakeFindPackageByIdQuery(this.nugetFindPackageIdQueryFormat); var packages = NuGetClient.FindPackage(query, request).Where(package => findContext.PackageInfo.Id.Equals(package.Id, StringComparison.OrdinalIgnoreCase)); if (findContext.RequiredVersion != null) { //Usually versions has a limited number, ToArray should be ok. var versions = findContext.RequiredVersion.GetComparableVersionStrings().ToArray(); packages = packages.Where(package => versions.Contains(package.Version, StringComparer.OrdinalIgnoreCase)); } //Will only enumerate packages once return(findContext.MakeResult(packages)); }
public static bool IsValidByName(PackageEntryInfo packageEntry, NuGetSearchContext searchContext) { NuGetSearchTerm originalPsTerm = searchContext.SearchTerms == null ? null : searchContext.SearchTerms.Where(st => st.Term == NuGetSearchTerm.NuGetSearchTermType.OriginalPSPattern).FirstOrDefault(); bool valid = true; if (originalPsTerm != null) { if (!String.IsNullOrWhiteSpace(originalPsTerm.Text) && SMA.WildcardPattern.ContainsWildcardCharacters(originalPsTerm.Text)) { // Applying the wildcard pattern matching const SMA.WildcardOptions wildcardOptions = SMA.WildcardOptions.CultureInvariant | SMA.WildcardOptions.IgnoreCase; var wildcardPattern = new SMA.WildcardPattern(originalPsTerm.Text, wildcardOptions); valid = wildcardPattern.IsMatch(packageEntry.Id); } else if (!String.IsNullOrWhiteSpace(originalPsTerm.Text)) { valid = packageEntry.Id.IndexOf(originalPsTerm.Text, StringComparison.OrdinalIgnoreCase) > -1; } } return(valid); }
private IEnumerable <IEnumerable <PackageBase> > GetPackageCollectionsForSearchResult(dynamic searchResult, NuGetSearchContext searchContext, NuGetSearchTerm searchTerm, HashSet <string> foundPackageIds, RequestWrapper request) { if (searchResult.HasProperty("data")) { foreach (dynamic packageEntry in searchResult.data) { foundPackageIds.Add(packageEntry.id); yield return(GetPackagesForPackageEntry(packageEntry, searchContext, request)); } } }
public NuGetSearchResult Find(NuGetSearchContext findContext, NuGetRequest request) => throw new NotImplementedException();
public NuGetSearchResult Search(NuGetSearchContext searchContext, NuGetRequest request) { request.Debug(Messages.DebugInfoCallMethod, "NuGetPackageRepository", "Search"); return(this.ResourceProvider.QueryFeed.Search(searchContext, request)); }
private IEnumerable <PackageBase> GetPackagesForPackageEntry(dynamic packageEntry, NuGetSearchContext searchContext, RequestWrapper request) { PackageEntryInfo packageEntryInfo = new PackageEntryInfo(packageEntry.id); if (!PackageFilterUtility.IsValidByName(packageEntryInfo, searchContext)) { yield break; } // This will help us take packageEntryInfo.LatestVersion and packageEntryInfo.AbsoluteLatestVersion and get the matching package easily // We're not setting isLatestVersion here so we don't have to deal with "is it latest or absolute latest" Dictionary <SemanticVersion, PackageBase> versionToPackageTable = new Dictionary <SemanticVersion, PackageBase>(); NuGetSearchContext individualPackageSearchContext = new NuGetSearchContext() { PackageInfo = packageEntryInfo, AllVersions = searchContext.AllVersions, AllowPrerelease = searchContext.AllowPrerelease, RequiredVersion = searchContext.RequiredVersion, MinimumVersion = searchContext.MinimumVersion, MaximumVersion = searchContext.MaximumVersion, EnableDeepMetadataBypass = searchContext.EnableDeepMetadataBypass }; bool latestVersionRequired = !searchContext.AllVersions && searchContext.RequiredVersion == null && searchContext.MinimumVersion == null && searchContext.MaximumVersion == null; if (searchContext.EnableDeepMetadataBypass) { if (latestVersionRequired) { // Use the search result page to get the metadata for the latest version SemanticVersion individualPackageVersion = new SemanticVersion(packageEntry.version); packageEntryInfo.AddVersion(individualPackageVersion); PackageBase pb = this.ResourcesCollection.PackageConverter.Make(packageEntry); if (pb != null) { yield return(pb); } } else { // Go to the registration index of this package first. This allows us to bypass "deep" (but required) metadata in certain cases. NuGetPackageFeed3 packageFeed3 = (NuGetPackageFeed3)this.ResourcesCollection.PackagesFeed; NuGetSearchResult result = packageFeed3.Find(individualPackageSearchContext, request); foreach (PackageBase pb in result.Result.Cast <PackageBase>()) { yield return(pb); } } } else { // Either we want a specific version or we want all metadata for any packages foreach (dynamic packageVersionEntry in packageEntry.versions) { if (packageEntry.version.Equals(packageVersionEntry.version) || searchContext.AllVersions) { if (packageVersionEntry.Metadata.HasProperty("id")) { // Collect all versions from the search results so we can manually set isLatestVersion and isAbsoluteLatestVersion later SemanticVersion individualPackageVersion = new SemanticVersion(packageVersionEntry.version); packageEntryInfo.AddVersion(individualPackageVersion); // Skip prerelease versions if AllowPrereleaseVersions is not specified if (!String.IsNullOrEmpty(individualPackageVersion.SpecialVersion) && !searchContext.AllowPrerelease) { continue; } long?versionDownloadCount = null; if (packageVersionEntry.HasProperty("downloads")) { versionDownloadCount = packageVersionEntry.downloads; } string registrationUrl = packageVersionEntry.Metadata.id; // This should be PackageFeed3 // There should be a better way to reuse this function NuGetPackageFeed3 packageFeed3 = (NuGetPackageFeed3)this.ResourcesCollection.PackagesFeed; PackageBase packageVersionPackage = packageFeed3.Find(registrationUrl, individualPackageSearchContext, request, true).FirstOrDefault(); if (packageVersionPackage != null) { if (versionDownloadCount.HasValue) { packageVersionPackage.VersionDownloadCount = versionDownloadCount.Value; } // Reset these so we haven't collected all versions yet, so this is wrong packageVersionPackage.IsLatestVersion = false; packageVersionPackage.IsAbsoluteLatestVersion = false; versionToPackageTable[individualPackageVersion] = packageVersionPackage; } } } } // Now manually set the latest versions if (packageEntryInfo.LatestVersion != null && versionToPackageTable.ContainsKey(packageEntryInfo.LatestVersion)) { versionToPackageTable[packageEntryInfo.LatestVersion].IsLatestVersion = true; } if (packageEntryInfo.AbsoluteLatestVersion != null && versionToPackageTable.ContainsKey(packageEntryInfo.AbsoluteLatestVersion)) { versionToPackageTable[packageEntryInfo.AbsoluteLatestVersion].IsAbsoluteLatestVersion = true; } // I think this is the best we can do for enumeration (reads all versions of a package before yielding anything) foreach (PackageBase package in versionToPackageTable.Values) { yield return(package); } } }
public NuGetSearchResult Search(NuGetSearchContext searchContext, NuGetRequest nugetRequest) { return(Search(searchContext, new RequestWrapper(nugetRequest))); }
public NuGetSearchResult FindPackagesById(NuGetSearchContext findContext, NuGetRequest request) { request.Debug(Messages.DebugInfoCallMethod3, "NuGetPackageRepository", "FindPackagesById", findContext.PackageInfo.Id); return(this.ResourceProvider.PackagesFeed.Find(findContext, request)); }
private IEnumerable <IPackage> SearchPackagesWithBackup(string baseUrl, HttpQueryBuilder qb, RequestWrapper request, NuGetSearchContext searchContext, NuGetSearchTerm searchTerm) { // First execute the actual search HashSet <string> foundPackageIds = new HashSet <string>(); return(NuGetWebUtility.GetResults <dynamic, PackageBase>(request, (dynamic root) => { long res = -1; if (root.HasProperty("totalhits")) { res = root.totalhits; request.Debug(Resources.Messages.TotalPackagesDiscovered, res); } else { request.Warning(Resources.Messages.JsonSchemaMismatch, "totalhits"); request.Debug(Resources.Messages.JsonObjectDump, DynamicJsonParser.Serialize(root)); } return res; }, (dynamic root) => GetPackageCollectionsForSearchResult(root, searchContext, searchTerm, foundPackageIds, request), (long packagesToSkip) => { if (packagesToSkip > 0) { HttpQueryBuilder currentQb = qb.CloneAdd(Constants.SkipQueryParam, packagesToSkip.ToString()); return currentQb.AddQueryString(baseUrl); } return qb.AddQueryString(baseUrl); }, (string content) => { return DynamicJsonParser.Parse(content); }, Constants.SearchPageCountInt)); }
/// <summary> /// Find packages when the registration URL is already known. /// </summary> /// <param name="registrationUrl"></param> /// <param name="request"></param> /// <returns></returns> internal IEnumerable <PackageBase> Find(string registrationUrl, NuGetSearchContext context, RequestWrapper request, bool finalAttempt) { request.Debug(Messages.DebugInfoCallMethod3, "NuGetPackageFeed3", "Find", registrationUrl); List <PackageBase> packages = null; PackageBase cachedPackage; if (!registrationUrl.Contains("index.json") && ConcurrentInMemoryCache.Instance.TryGet <PackageBase>(registrationUrl, out cachedPackage)) { if (cachedPackage == null) { return(packages); } else { packages = new List <PackageBase>() { cachedPackage }; return(packages); } } Stream s = NuGetClient.DownloadDataToStream(registrationUrl, request, ignoreNullResponse: true, tries: 1); if (s != null) { packages = new List <PackageBase>(); // Now we can get the catalog entry dynamic root = DynamicJsonParser.Parse(new StreamReader(s).ReadToEnd()); if (root.Metadata.HasProperty("type")) { // First check if this is a Package or PackageRegistration type // Package is from packageId + version // PackageRegistration is from packageId bool isRegistrationType = false; foreach (string t in root.Metadata.type) { if (t.Equals("PackageRegistration", StringComparison.OrdinalIgnoreCase)) { isRegistrationType = true; break; } } if (context.PackageInfo.AllVersions.Count == 0) { // Only when the version list is restricted is this method usually faster this.ResourcesCollection.FilesFeed.GetVersionInfo(context.PackageInfo, request); } HashSet <SemanticVersion> packageSemanticVersions = null; if (context.PackageInfo.AllVersions.Count > 0) { packageSemanticVersions = FilterVersionsByRequirements(context, context.PackageInfo); } if (isRegistrationType) { // This is a registration index, like: "https://api.nuget.org/v3/registration3/json/index.json" // Get all versions from files service // In addition, when DeepMetadataBypass is enabled, we MUST use the registration index to get package info // If a call to -Name -RequiredVersion is done, DeepMetadataBypass will never be enabled (for now) // If we wanted, we could enable this by checking if !isRegistrationType && context.EnableDeepMetadataBypass, then call into Find with the registration index URL if (!context.AllVersions && packageSemanticVersions != null && !context.EnableDeepMetadataBypass) { foreach (SemanticVersion packageVersion in packageSemanticVersions) { NuGetSearchResult result = this.ResourcesCollection.PackagesFeed.Find(new NuGetSearchContext() { PackageInfo = context.PackageInfo, RequiredVersion = packageVersion, EnableDeepMetadataBypass = context.EnableDeepMetadataBypass }, request); PackageBase package = result.Result == null ? null : result.Result.FirstOrDefault() as PackageBase; if (package != null) { packages.Add(package); } } } else { // Going to collect versions from the registration index in here // Map of package version -> either PackageBase (if context.EnableDeepMetadataBypass) or catalog URL Dictionary <SemanticVersion, object> catalogObjects = new Dictionary <SemanticVersion, object>(); // If the version list hasn't been built yet, we can build it from the registration page instead of using FilesFeed bool buildPackageInfoVersions = context.PackageInfo.AllVersions.Count == 0; // Fallback to catalog crawling in these cases: // - Bypass deep metadata // - Getting version list failed // - All versions required foreach (dynamic catalogPage in root.items) { dynamic actualCatalogPage = catalogPage; if (!actualCatalogPage.HasProperty("items")) { // Sometimes the catalog page on the PackageRegistration entry doesn't have the actual page contents // In this case, the ID metadata tag points to the full catalog entry Stream fullCatalogPageResponseStream = NuGetClient.DownloadDataToStream(actualCatalogPage.Metadata.id, request); if (fullCatalogPageResponseStream != null) { actualCatalogPage = DynamicJsonParser.Parse(new StreamReader(fullCatalogPageResponseStream).ReadToEnd()); } } foreach (dynamic packageEntry in actualCatalogPage.items) { SemanticVersion version = new SemanticVersion(packageEntry.catalogentry.version); if (buildPackageInfoVersions) { context.PackageInfo.AddVersion(version); } bool hasCatalogEntry = packageEntry.HasProperty("catalogentry"); if (context.EnableDeepMetadataBypass || !hasCatalogEntry) { // Bypass retrieving "deep" (but required) metadata like packageHash // Also do this if there's no catalog entry PackageBase pb = null; if (packageEntry.HasProperty("catalogentry")) { pb = this.ResourcesCollection.PackageConverter.Make(packageEntry.catalogentry, context.PackageInfo); } else { // In some implementations (lookin' at you MyGet) there's no catalogEntry object pb = this.ResourcesCollection.PackageConverter.Make(packageEntry, context.PackageInfo); } if (pb != null) { catalogObjects[version] = pb; } } else { catalogObjects[version] = this.ResourcesCollection.CatalogUrlConverter.Make(packageEntry); } } } packageSemanticVersions = FilterVersionsByRequirements(context, context.PackageInfo); foreach (SemanticVersion version in packageSemanticVersions) { if (!catalogObjects.ContainsKey(version)) { continue; } if (context.EnableDeepMetadataBypass) { packages.Add((PackageBase)catalogObjects[version]); } else { PackageBase pb = GetPackageFromCatalogUrl((string)catalogObjects[version], request, packageSemanticVersions, context); if (pb != null) { packages.Add(pb); } } } } } else { // In some implementations (lookin' at you MyGet) there's no catalogEntry object PackageBase pb = ConcurrentInMemoryCache.Instance.GetOrAdd <PackageBase>(registrationUrl, () => { if (!root.HasProperty("catalogentry")) { if ((packageSemanticVersions == null || packageSemanticVersions.Contains(new SemanticVersion(root.version)))) { return(this.ResourcesCollection.PackageConverter.Make(root)); } else { return(null); } } else { return(GetPackageFromCatalogUrl(this.ResourcesCollection.CatalogUrlConverter.Make(root), request, packageSemanticVersions, context)); } }); if (pb != null) { packages.Add(pb); } } } else { request.Warning(Messages.JsonSchemaMismatch, "type"); request.Debug(Messages.JsonObjectDump, DynamicJsonParser.Serialize(root)); } if (context.RequiredVersion == null && context.MinimumVersion == null && context.MaximumVersion == null && packages != null) { PackageBase absoluteLatestPackage = packages.Where(p => p.IsPrerelease).OrderByDescending(pb => ((IPackage)pb).Version).FirstOrDefault(); if (absoluteLatestPackage != null) { absoluteLatestPackage.IsAbsoluteLatestVersion = true; } PackageBase latestPackage = packages.Where(p => !p.IsPrerelease).OrderByDescending(pb => ((IPackage)pb).Version).FirstOrDefault(); if (latestPackage != null) { latestPackage.IsLatestVersion = true; } } } else if (finalAttempt) { // This is the last retry of this URL. It's definitely not a good one. ConcurrentInMemoryCache.Instance.GetOrAdd <PackageBase>(registrationUrl, () => null); } return(packages); }
/// <summary> /// Find-Package /// </summary> /// <param name="packageId">Package id</param> /// <param name="version">Package Name</param> /// <param name="request"></param> /// <returns></returns> public virtual IPackage FindPackage(NuGetSearchContext findContext, NuGetRequest request) { return(FindPackage(OpenPackage, findContext.PackageInfo.Id, findContext.RequiredVersion, request)); }
/// <summary> /// Find-Package based the given Id /// </summary> /// <param name="packageId">Package Id</param> /// <param name="request"></param> /// <returns></returns> public NuGetSearchResult FindPackagesById(NuGetSearchContext findContext, NuGetRequest request) { IEnumerable <IPackage> packages = FindPackagesById(OpenPackage, findContext.PackageInfo.Id, request); return(findContext.MakeResult(packages)); }
public NuGetSearchResult Find(NuGetSearchContext findContext, NuGetRequest request) { return(Find(findContext, new RequestWrapper(request))); }
public NuGetSearchResult Find(NuGetSearchContext findContext, RequestWrapper request) { throw new NotImplementedException(); }
private PackageBase GetPackageFromCatalogUrl(string catalogUrl, RequestWrapper request, HashSet <SemanticVersion> packageSemanticVersions, NuGetSearchContext context) { Stream catalogResponseStream = NuGetClient.DownloadDataToStream(catalogUrl, request); if (catalogResponseStream != null) { string content = new StreamReader(catalogResponseStream).ReadToEnd(); dynamic catalogContent = DynamicJsonParser.Parse(content); if ((packageSemanticVersions == null || packageSemanticVersions.Contains(new SemanticVersion(catalogContent.version)))) { PackageBase pb = this.ResourcesCollection.PackageConverter.Make(DynamicJsonParser.Parse(content), context.PackageInfo); if (pb != null) { return(pb); } } } else { request.Warning(Messages.CouldNotGetResponseFromQuery, catalogUrl); } return(null); }