public override bool IsAvailable(RequestWrapper request) { foreach (NuGetServiceInfo endpoint in this.Endpoints) { Stream s = NuGetClient.DownloadDataToStream(endpoint.Url, request); if (s != null) { return(true); } } return(false); }
private static INuGetResourceCollection GetResourcesImpl(string baseUrl, RequestWrapper request) { INuGetResourceCollection res = null; HttpClient client = request.GetClientWithHeaders(); HttpResponseMessage response = PathUtility.GetHttpResponse(client, baseUrl, (() => request.IsCanceled()), ((msg, num) => request.Verbose(Resources.Messages.RetryingDownload, msg, num)), (msg) => request.Verbose(msg), (msg) => request.Debug(msg)); if (!response.IsSuccessStatusCode) { throw new Exception(Resources.Messages.NuGetEndpointDiscoveryFailed); } string content = new StreamReader(NuGetClient.DownloadDataToStream(baseUrl, request)).ReadToEnd(); // If the response starts with the magic XML header, it's v2 if (content.StartsWith(Constants.XmlStartContent)) { res = NuGetResourceCollection2.Make(baseUrl); } else { try { dynamic root = DynamicJsonParser.Parse(content); string version = root.version; if (version != null && version.StartsWith("3.")) { // v3 feed res = NuGetResourceCollection3.Make(root, baseUrl, request); } } catch (Exception ex) { Exception discoveryException = new Exception(Resources.Messages.NuGetEndpointDiscoveryFailed, ex); throw discoveryException; } } if (res == null) { // Couldn't figure out what this is throw new Exception(Resources.Messages.NuGetEndpointDiscoveryFailed); } return(res); }
public PackageEntryInfo GetVersionInfo(PackageEntryInfo packageInfo, RequestWrapper request) { try { request.Debug(Messages.DebugInfoCallMethod, "NuGetFilesFeed3", "GetVersionInfo"); if (packageInfo == null) { throw new ArgumentNullException("packageInfo"); } if (request == null) { throw new ArgumentNullException("request"); } string query = String.Format(CultureInfo.InvariantCulture, Constants.VersionIndexTemplate, this.baseUrl, this.baseUrl.EndsWith("/") ? String.Empty : "/", packageInfo.Id.ToLowerInvariant()); Stream queryResponse = NuGetClient.DownloadDataToStream(query, request, ignoreNullResponse: true); if (queryResponse != null) { dynamic root = DynamicJsonParser.Parse(new StreamReader(queryResponse).ReadToEnd()); if (root.HasProperty("versions")) { foreach (string v in root.versions) { packageInfo.AddVersion(new SemanticVersion(v)); } } else { request.Debug(Messages.VersionIndexDownloadFailed, packageInfo.Id); } } else { request.Debug(Messages.VersionIndexDownloadFailed, packageInfo.Id); } } finally { request.Debug(Messages.DebugInfoReturnCall, "NuGetFilesFeed3", "GetVersionInfo"); } return(packageInfo); }
private PackageBase GetPackageFromCatalogUrl(string catalogUrl, RequestWrapper request, HashSet <SemanticVersion> packageSemanticVersions, NuGetSearchContext context) { Stream catalogResponseStream = NuGetClient.DownloadDataToStream(catalogUrl, request); if (catalogResponseStream != null) { string content = new StreamReader(catalogResponseStream).ReadToEnd(); dynamic catalogContent = DynamicJsonParser.Parse(content); if ((packageSemanticVersions == null || packageSemanticVersions.Contains(new SemanticVersion(catalogContent.version)))) { PackageBase pb = this.ResourcesCollection.PackageConverter.Make(DynamicJsonParser.Parse(content), context.PackageInfo); if (pb != null) { return(pb); } } } else { request.Warning(Messages.CouldNotGetResponseFromQuery, catalogUrl); } return(null); }
/// <summary> /// Find packages when the registration URL is already known. /// </summary> /// <param name="registrationUrl"></param> /// <param name="request"></param> /// <returns></returns> internal IEnumerable <PackageBase> Find(string registrationUrl, NuGetSearchContext context, RequestWrapper request, bool finalAttempt) { request.Debug(Messages.DebugInfoCallMethod3, "NuGetPackageFeed3", "Find", registrationUrl); List <PackageBase> packages = null; PackageBase cachedPackage; if (!registrationUrl.Contains("index.json") && ConcurrentInMemoryCache.Instance.TryGet <PackageBase>(registrationUrl, out cachedPackage)) { if (cachedPackage == null) { return(packages); } else { packages = new List <PackageBase>() { cachedPackage }; return(packages); } } Stream s = NuGetClient.DownloadDataToStream(registrationUrl, request, ignoreNullResponse: true, tries: 1); if (s != null) { packages = new List <PackageBase>(); // Now we can get the catalog entry dynamic root = DynamicJsonParser.Parse(new StreamReader(s).ReadToEnd()); if (root.Metadata.HasProperty("type")) { // First check if this is a Package or PackageRegistration type // Package is from packageId + version // PackageRegistration is from packageId bool isRegistrationType = false; foreach (string t in root.Metadata.type) { if (t.Equals("PackageRegistration", StringComparison.OrdinalIgnoreCase)) { isRegistrationType = true; break; } } if (context.PackageInfo.AllVersions.Count == 0) { // Only when the version list is restricted is this method usually faster this.ResourcesCollection.FilesFeed.GetVersionInfo(context.PackageInfo, request); } HashSet <SemanticVersion> packageSemanticVersions = null; if (context.PackageInfo.AllVersions.Count > 0) { packageSemanticVersions = FilterVersionsByRequirements(context, context.PackageInfo); } if (isRegistrationType) { // This is a registration index, like: "https://api.nuget.org/v3/registration3/json/index.json" // Get all versions from files service // In addition, when DeepMetadataBypass is enabled, we MUST use the registration index to get package info // If a call to -Name -RequiredVersion is done, DeepMetadataBypass will never be enabled (for now) // If we wanted, we could enable this by checking if !isRegistrationType && context.EnableDeepMetadataBypass, then call into Find with the registration index URL if (!context.AllVersions && packageSemanticVersions != null && !context.EnableDeepMetadataBypass) { foreach (SemanticVersion packageVersion in packageSemanticVersions) { NuGetSearchResult result = this.ResourcesCollection.PackagesFeed.Find(new NuGetSearchContext() { PackageInfo = context.PackageInfo, RequiredVersion = packageVersion, EnableDeepMetadataBypass = context.EnableDeepMetadataBypass }, request); PackageBase package = result.Result == null ? null : result.Result.FirstOrDefault() as PackageBase; if (package != null) { packages.Add(package); } } } else { // Going to collect versions from the registration index in here // Map of package version -> either PackageBase (if context.EnableDeepMetadataBypass) or catalog URL Dictionary <SemanticVersion, object> catalogObjects = new Dictionary <SemanticVersion, object>(); // If the version list hasn't been built yet, we can build it from the registration page instead of using FilesFeed bool buildPackageInfoVersions = context.PackageInfo.AllVersions.Count == 0; // Fallback to catalog crawling in these cases: // - Bypass deep metadata // - Getting version list failed // - All versions required foreach (dynamic catalogPage in root.items) { dynamic actualCatalogPage = catalogPage; if (!actualCatalogPage.HasProperty("items")) { // Sometimes the catalog page on the PackageRegistration entry doesn't have the actual page contents // In this case, the ID metadata tag points to the full catalog entry Stream fullCatalogPageResponseStream = NuGetClient.DownloadDataToStream(actualCatalogPage.Metadata.id, request); if (fullCatalogPageResponseStream != null) { actualCatalogPage = DynamicJsonParser.Parse(new StreamReader(fullCatalogPageResponseStream).ReadToEnd()); } } foreach (dynamic packageEntry in actualCatalogPage.items) { SemanticVersion version = new SemanticVersion(packageEntry.catalogentry.version); if (buildPackageInfoVersions) { context.PackageInfo.AddVersion(version); } bool hasCatalogEntry = packageEntry.HasProperty("catalogentry"); if (context.EnableDeepMetadataBypass || !hasCatalogEntry) { // Bypass retrieving "deep" (but required) metadata like packageHash // Also do this if there's no catalog entry PackageBase pb = null; if (packageEntry.HasProperty("catalogentry")) { pb = this.ResourcesCollection.PackageConverter.Make(packageEntry.catalogentry, context.PackageInfo); } else { // In some implementations (lookin' at you MyGet) there's no catalogEntry object pb = this.ResourcesCollection.PackageConverter.Make(packageEntry, context.PackageInfo); } if (pb != null) { catalogObjects[version] = pb; } } else { catalogObjects[version] = this.ResourcesCollection.CatalogUrlConverter.Make(packageEntry); } } } packageSemanticVersions = FilterVersionsByRequirements(context, context.PackageInfo); foreach (SemanticVersion version in packageSemanticVersions) { if (!catalogObjects.ContainsKey(version)) { continue; } if (context.EnableDeepMetadataBypass) { packages.Add((PackageBase)catalogObjects[version]); } else { PackageBase pb = GetPackageFromCatalogUrl((string)catalogObjects[version], request, packageSemanticVersions, context); if (pb != null) { packages.Add(pb); } } } } } else { // In some implementations (lookin' at you MyGet) there's no catalogEntry object PackageBase pb = ConcurrentInMemoryCache.Instance.GetOrAdd <PackageBase>(registrationUrl, () => { if (!root.HasProperty("catalogentry")) { if ((packageSemanticVersions == null || packageSemanticVersions.Contains(new SemanticVersion(root.version)))) { return(this.ResourcesCollection.PackageConverter.Make(root)); } else { return(null); } } else { return(GetPackageFromCatalogUrl(this.ResourcesCollection.CatalogUrlConverter.Make(root), request, packageSemanticVersions, context)); } }); if (pb != null) { packages.Add(pb); } } } else { request.Warning(Messages.JsonSchemaMismatch, "type"); request.Debug(Messages.JsonObjectDump, DynamicJsonParser.Serialize(root)); } if (context.RequiredVersion == null && context.MinimumVersion == null && context.MaximumVersion == null && packages != null) { PackageBase absoluteLatestPackage = packages.Where(p => p.IsPrerelease).OrderByDescending(pb => ((IPackage)pb).Version).FirstOrDefault(); if (absoluteLatestPackage != null) { absoluteLatestPackage.IsAbsoluteLatestVersion = true; } PackageBase latestPackage = packages.Where(p => !p.IsPrerelease).OrderByDescending(pb => ((IPackage)pb).Version).FirstOrDefault(); if (latestPackage != null) { latestPackage.IsLatestVersion = true; } } } else if (finalAttempt) { // This is the last retry of this URL. It's definitely not a good one. ConcurrentInMemoryCache.Instance.GetOrAdd <PackageBase>(registrationUrl, () => null); } return(packages); }
/// <summary> /// Send the request to the server with buffer size to account for the case where there are more data /// that we need to fetch /// </summary> /// <param name="query"></param> /// <param name="request"></param> /// <returns></returns> public static IEnumerable <PackageBase> SendRequest(string query, NuGetRequest request) { const int bufferSize = 40; // number of threads sending the requests const int numberOfSenders = 4; var startPoint = 0; var tasks = new List <Task <Stream> >(); bool stopSending = false; object stopLock = new Object(); // Send one request first // this initial query is of the form http://www.nuget.org/api/v2/FindPackagesById()?id='jquery'&$skip={0}&$top={1} UriBuilder initialQuery = new UriBuilder(query.InsertSkipAndTop()); PackageBase firstPackage = null; // Send out an initial request // we send out 1 initial request first to check for redirection and check whether repository supports odata using (Stream stream = NuGetClient.InitialDownloadDataToStream(initialQuery, startPoint, bufferSize, request)) { if (stream == null) { yield break; } XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true); var entries = document.Root.ElementsNoNamespace("entry").ToList(); // If the initial request has different number of entries than the buffer size, return it because this means the server // does not understand odata request or there is no more data. in the former case, we have to stop to prevent infinite loop if (entries.Count != bufferSize) { request.Debug(Messages.PackagesReceived, entries.Count); stopSending = true; } foreach (XElement entry in entries) { var package = new PackageBase(); // set the first package of the request. this is used later to verify that the case when the number of packages in the repository // is the same as the buffer size and the repository does not support odata query. in that case, we want to check whether the first package // exists anywhere in the second call. if it is, then we cancel the request (this is to prevent infinite loop) if (firstPackage == null) { firstPackage = package; } PackageUtility.ReadEntryElement(ref package, entry); yield return(package); } } if (stopSending || request.IsCanceled) { yield break; } // To avoid more redirection (for example, if the initial query is nuget.org, it will be changed to www.nuget.org query = initialQuery.Uri.ToString(); // Sending the initial requests for (var i = 0; i < numberOfSenders; i++) { // Update the start point to fetch the packages startPoint += bufferSize; // Get the query var newQuery = string.Format(query, startPoint, bufferSize); // Send it tasks.Add(Task.Factory.StartNew(() => { Stream items = NuGetClient.DownloadDataToStream(newQuery, request); return(items); })); } //Wait for the responses, parse the data, and send to the user while (tasks.Count > 0) { //Cast because the compiler warning: Co-variant array conversion from Task[] to Task[] can cause run-time exception on write operation. var index = Task.WaitAny(tasks.Cast <Task>().ToArray()); using (Stream stream = tasks[index].Result) { if (stream == null) { yield break; } XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true); var entries = document.Root.ElementsNoNamespace("entry").ToList(); if (entries.Count < bufferSize) { request.Debug(Messages.PackagesReceived, entries.Count); lock (stopLock) { stopSending = true; } } foreach (XElement entry in entries) { var package = new PackageBase(); PackageUtility.ReadEntryElement(ref package, entry); if (firstPackage != null) { // check whether first package in the first request exists anywhere in the second request if (string.Equals(firstPackage.GetFullName(), package.GetFullName(), StringComparison.OrdinalIgnoreCase) && string.Equals(firstPackage.Version, package.Version, StringComparison.OrdinalIgnoreCase)) { lock (stopLock) { stopSending = true; } break; } } yield return(package); } // we only needs to check for the existence of the first package in the second request. don't need to do for subsequent request if (firstPackage != null) { firstPackage = null; } } // checks whether we should stop sending requests if (!stopSending && !request.IsCanceled) { // Make sure nobody else is updating the startPoint lock (stopLock) { // update the startPoint startPoint += bufferSize; } // Make a new request with the new startPoint var newQuery = string.Format(query, startPoint, bufferSize); //Keep sending a request tasks[index] = (Task.Factory.StartNew(searchQuery => { var items = NuGetClient.DownloadDataToStream(searchQuery.ToStringSafe(), request); return(items); }, newQuery)); } else { if (request.IsCanceled) { request.Warning(Messages.RequestCanceled, "HttpClientPackageRepository", "SendRequest"); //stop sending request to the remote server stopSending = true; } tasks.RemoveAt(index); } } }
/// <summary> /// Get results from paged query for NuGet v3 /// </summary> /// <typeparam name="B">Response body type</typeparam> /// <typeparam name="R">Result type</typeparam> /// <param name="request">Current request</param> /// <param name="getTotalResultsCountFromResponse">Delegate to get total result count from first response. Return 0 or a negative number to indicate failure.</param> /// <param name="getResultsFromResponse">Delegate to get results from a response body.</param> /// <param name="getPackageQuery">Delegate to get the next query, skipping a given number of results.</param> /// <param name="parseResponseBody">Delegate to parse the response string body to a response object body.</param> /// <returns>All results.</returns> public static IEnumerable <R> GetResults <B, R>(RequestWrapper request, Func <B, long> getTotalResultsCountFromResponse, Func <B, IEnumerable <IEnumerable <R> > > getResultsFromResponse, Func <long, string> getNextResponseQuery, Func <string, B> parseResponseBody, int pageCount) { ProgressTracker progressTracker = new ProgressTracker(ProgressTracker.GetRandomId(), 0, 100); int childProgressTrackerId = request.StartProgress(progressTracker.ProgressID, Resources.Messages.NuGetServerReadStarted); long total = -1; long resultCount = 0; bool successful = true; bool trackIndividualProgress = false; TaskGroup <IEnumerable <IEnumerable <R> > > taskGroup = new TaskGroup <IEnumerable <IEnumerable <R> > >(); System.Threading.CancellationTokenSource cancelToken = new System.Threading.CancellationTokenSource(); string query = getNextResponseQuery(resultCount); string content = new StreamReader(NuGetClient.DownloadDataToStream(query, request)).ReadToEnd(); B response = parseResponseBody(content); total = getTotalResultsCountFromResponse(response); if (total < 0) { total = -2; request.Warning(Messages.FailedToParseTotalHitsCount, query); successful = false; } else { request.Progress(childProgressTrackerId, 0, String.Format(CultureInfo.CurrentCulture, Resources.Messages.NuGetServerReadProgress, resultCount, total)); // When the result count is low enough, track individual results. Otherwise, track by page. trackIndividualProgress = total <= (pageCount * 2); taskGroup.Add(Task.Factory.StartNew <IEnumerable <IEnumerable <R> > >(() => { return(getResultsFromResponse(response)); }, cancelToken.Token)); } if (total >= 0) { long numberOfPages = total / pageCount + (total % pageCount == 0 ? 0 : 1); long pageSkipCount = 0; for (long pageNum = 1; pageNum < numberOfPages; pageNum++) { pageSkipCount += pageCount; string pageQuery = getNextResponseQuery(pageSkipCount); taskGroup.Add(Task.Factory.StartNew <IEnumerable <IEnumerable <R> > >((q) => { string pageContent = new StreamReader(NuGetClient.DownloadDataToStream((string)q, request)).ReadToEnd(); B pageResponse = parseResponseBody(pageContent); return(getResultsFromResponse(pageResponse)); }, pageQuery, cancelToken.Token)); } } while (taskGroup.HasAny) { if (request.IsCanceled()) { cancelToken.Cancel(); successful = false; break; } IEnumerable <IEnumerable <R> > resultCollections = taskGroup.WaitAny(); foreach (IEnumerable <R> resultCollection in resultCollections) { resultCount++; // If trackIndividualProgress == false, this progress will be reported at the end if (trackIndividualProgress) { // Report an individual package is done processing request.Progress(childProgressTrackerId, progressTracker.ConvertPercentToProgress(((double)resultCount) / total), String.Format(CultureInfo.CurrentCulture, Resources.Messages.NuGetServerReadProgress, resultCount, total)); } foreach (R result in resultCollection) { yield return(result); } } if (!trackIndividualProgress) { // Report that this page is done processing request.Progress(childProgressTrackerId, progressTracker.ConvertPercentToProgress(((double)resultCount) / total), String.Format(CultureInfo.CurrentCulture, Resources.Messages.NuGetServerReadProgress, resultCount, total)); } } request.CompleteProgress(childProgressTrackerId, successful); request.CompleteProgress(progressTracker.ProgressID, successful); }