/// <summary> /// Retrieve entries for the given index page entries. /// </summary> public async Task <List <CatalogEntry> > GetEntriesAsync(IEnumerable <CatalogPageEntry> pages, CancellationToken token) { var maxThreads = Math.Max(1, MaxThreads); var cache = new ReferenceCache(); var entries = new List <CatalogEntry>(); var tasks = new List <Task <JObject> >(maxThreads); foreach (var page in pages) { token.ThrowIfCancellationRequested(); while (tasks.Count > maxThreads) { entries.AddRange(await CompleteTaskAsync(tasks, cache)); } tasks.Add(_httpSource.GetJObjectAsync(page.Uri, _cacheContext, _log, token)); } while (tasks.Count > 0) { entries.AddRange(await CompleteTaskAsync(tasks, cache)); } return(entries); }
public async static Task <IEnumerable <JObject> > LoadRanges( HttpSource httpClient, Uri registrationUri, VersionRange range, ILogger log, CancellationToken token) { var index = await httpClient.GetJObjectAsync(registrationUri, ignoreNotFounds : true, log : log, token : token); if (index == null) { // The server returned a 404, the package does not exist return(Enumerable.Empty <JObject>()); } IList <Task <JObject> > rangeTasks = new List <Task <JObject> >(); foreach (JObject item in index["items"]) { var lower = NuGetVersion.Parse(item["lower"].ToString()); var upper = NuGetVersion.Parse(item["upper"].ToString()); if (IsItemRangeRequired(range, lower, upper)) { JToken items; if (!item.TryGetValue("items", out items)) { var rangeUri = item["@id"].ToObject <Uri>(); rangeTasks.Add(httpClient.GetJObjectAsync( rangeUri, ignoreNotFounds: true, log: log, token: token)); } else { rangeTasks.Add(Task.FromResult(item)); } } } await Task.WhenAll(rangeTasks.ToArray()); return(rangeTasks.Select((t) => t.Result)); }
private async Task <JObject> GetPackageFromLeafAsync(PackageIdentity package, ILogger log, CancellationToken token) { /// If the registration leaf is missing, <see cref="HttpSourceRequest.IgnoreNotFounds"/> will cause this to return null. return(await _client.GetJObjectAsync( new HttpSourceRequest( _registration.GetUri(package), log) { IgnoreNotFounds = true }, log, token)); }
public virtual async Task <JObject> SearchPage(string searchTerm, SearchFilter filters, int skip, int take, Common.ILogger log, CancellationToken cancellationToken) { for (var i = 0; i < _searchEndpoints.Length; i++) { var endpoint = _searchEndpoints[i]; // The search term comes in already encoded from VS var queryUrl = new UriBuilder(endpoint.AbsoluteUri); var queryString = "q=" + searchTerm + "&skip=" + skip.ToString() + "&take=" + take.ToString() + "&prerelease=" + filters.IncludePrerelease.ToString().ToLowerInvariant(); if (filters.IncludeDelisted) { queryString += "&includeDelisted=true"; } if (filters.SupportedFrameworks != null && filters.SupportedFrameworks.Any()) { var frameworks = string.Join("&", filters.SupportedFrameworks.Select( fx => "supportedFramework=" + fx.ToString())); queryString += "&" + frameworks; } if (filters.PackageTypes != null && filters.PackageTypes.Any()) { var types = string.Join("&", filters.PackageTypes.Select( s => "packageTypeFilter=" + s)); queryString += "&" + types; } queryUrl.Query = queryString; if (!cancellationToken.IsCancellationRequested) { JObject searchJson = null; try { searchJson = await _client.GetJObjectAsync( uri : queryUrl.Uri, ignoreNotFounds : false, log : log, token : cancellationToken); } catch when(i < _searchEndpoints.Length - 1) { // Ignore all failures until the last endpoint }
/// <summary> /// Retrieve entries for the given index page entries. /// </summary> public async Task <List <CatalogEntry> > GetEntriesAsync(IEnumerable <CatalogPageEntry> pages, CancellationToken token) { var tasks = pages.Select(page => new Func <Task <JObject> >(() => _httpSource.GetJObjectAsync(page.Uri, _cacheContext, _log, token))); var maxThreads = Math.Max(1, MaxThreads); var cache = new ReferenceCache(); var entries = new ConcurrentBag <CatalogEntry>(); var process = new Func <Task <JObject>, Task <bool> >(e => CompleteTaskAsync(e, cache, entries)); await TaskUtils.RunAsync(tasks, useTaskRun : false, maxThreads : maxThreads, process : process, token : token); return(entries.ToList()); }
// Get a temp API key from nuget.org for pushing to https://nuget.smbsrc.net/ private async Task <string> GetSecureApiKey( PackageIdentity packageIdentity, string apiKey, bool noServiceEndpoint, TimeSpan requestTimeout, ILogger logger, CancellationToken token) { if (string.IsNullOrEmpty(apiKey)) { return(apiKey); } var serviceEndpointUrl = GetServiceEndpointUrl(NuGetConstants.DefaultGalleryServerUrl, string.Format(TempApiKeyServiceEndpoint, packageIdentity.Id, packageIdentity.Version), noServiceEndpoint); try { var result = await _httpSource.GetJObjectAsync( new HttpSourceRequest( () => { var request = HttpRequestMessageFactory.Create( HttpMethod.Post, serviceEndpointUrl, new HttpRequestMessageConfiguration( logger: logger, promptOn403: false)); request.Headers.Add(ApiKeyHeader, apiKey); return(request); }) { RequestTimeout = requestTimeout, MaxTries = 1 }, logger, token); return(result.Value <string>("Key") ?? InvalidApiKey); } catch (HttpRequestException ex) { #if NETCOREAPP if (ex.Message.Contains("Response status code does not indicate success: 403 (Forbidden).", StringComparison.OrdinalIgnoreCase)) #else if (ex.Message.Contains("Response status code does not indicate success: 403 (Forbidden).")) #endif { return(InvalidApiKey); } throw; } }
/// <summary> /// Ensure index.json has been loaded. /// </summary> protected async Task EnsureServiceIndexAsync(Uri uri, CancellationToken token) { if (_serviceIndex == null) { await EnsureHttpSourceAsync(); var index = await _httpSource.GetJObjectAsync(_indexUri, _cacheContext, _log, token); var resources = (index["resources"] as JArray); if (resources == null) { throw new InvalidOperationException($"{uri.AbsoluteUri} does not contain a 'resources' property. Use the root service index.json for the nuget v3 feed."); } _serviceIndex = new ServiceIndexResourceV3(index, DateTime.UtcNow); } }
public async Task HttpSource_TimesOutDownload() { // Arrange using (var td = TestDirectory.Create()) { var expectedMilliseconds = 250; var server = new TcpListenerServer { Mode = TestServerMode.SlowResponseBody, SleepDuration = TimeSpan.FromSeconds(10) }; var packageSource = new PackageSource(FakeSource); var handler = new HttpClientHandler(); var handlerResource = new HttpHandlerResourceV3(handler, handler); var httpSource = new HttpSource( packageSource, () => Task.FromResult((HttpHandlerResource)handlerResource), NullThrottle.Instance) { HttpCacheDirectory = td }; var logger = new TestLogger(); // Act & Assert var actual = await Assert.ThrowsAsync <IOException>(() => server.ExecuteAsync(uri => httpSource.GetJObjectAsync( new HttpSourceRequest(uri, logger) { DownloadTimeout = TimeSpan.FromMilliseconds(expectedMilliseconds) }, logger, CancellationToken.None))); Assert.IsType <TimeoutException>(actual.InnerException); Assert.EndsWith( $"timed out because no data was received for {expectedMilliseconds}ms.", actual.Message); } }