/// <summary>
/// Retrieve entries for the given index page entries.
/// </summary>
public async Task <List <CatalogEntry> > GetEntriesAsync(IEnumerable <CatalogPageEntry> pages, CancellationToken token)
{
var maxThreads = Math.Max(1, MaxThreads);
var cache = new ReferenceCache();
var entries = new List <CatalogEntry>();
var tasks = new List <Task <JObject> >(maxThreads);
foreach (var page in pages)
{
token.ThrowIfCancellationRequested();
while (tasks.Count > maxThreads)
{
entries.AddRange(await CompleteTaskAsync(tasks, cache));
}
tasks.Add(_httpSource.GetJObjectAsync(page.Uri, _cacheContext, _log, token));
}
while (tasks.Count > 0)
{
entries.AddRange(await CompleteTaskAsync(tasks, cache));
}
return(entries);
}
public async static Task <IEnumerable <JObject> > LoadRanges(
HttpSource httpClient,
Uri registrationUri,
VersionRange range,
ILogger log,
CancellationToken token)
{
var index = await httpClient.GetJObjectAsync(registrationUri, ignoreNotFounds : true, log : log, token : token);
if (index == null)
{
// The server returned a 404, the package does not exist
return(Enumerable.Empty <JObject>());
}
IList <Task <JObject> > rangeTasks = new List <Task <JObject> >();
foreach (JObject item in index["items"])
{
var lower = NuGetVersion.Parse(item["lower"].ToString());
var upper = NuGetVersion.Parse(item["upper"].ToString());
if (IsItemRangeRequired(range, lower, upper))
{
JToken items;
if (!item.TryGetValue("items", out items))
{
var rangeUri = item["@id"].ToObject <Uri>();
rangeTasks.Add(httpClient.GetJObjectAsync(
rangeUri,
ignoreNotFounds: true,
log: log,
token: token));
}
else
{
rangeTasks.Add(Task.FromResult(item));
}
}
}
await Task.WhenAll(rangeTasks.ToArray());
return(rangeTasks.Select((t) => t.Result));
}
private async Task <JObject> GetPackageFromLeafAsync(PackageIdentity package, ILogger log, CancellationToken token)
{
/// If the registration leaf is missing, <see cref="HttpSourceRequest.IgnoreNotFounds"/> will cause this to return null.
return(await _client.GetJObjectAsync(
new HttpSourceRequest(
_registration.GetUri(package), log)
{ IgnoreNotFounds = true },
log, token));
}
public virtual async Task <JObject> SearchPage(string searchTerm, SearchFilter filters, int skip, int take, Common.ILogger log, CancellationToken cancellationToken)
{
for (var i = 0; i < _searchEndpoints.Length; i++)
{
var endpoint = _searchEndpoints[i];
// The search term comes in already encoded from VS
var queryUrl = new UriBuilder(endpoint.AbsoluteUri);
var queryString =
"q=" + searchTerm +
"&skip=" + skip.ToString() +
"&take=" + take.ToString() +
"&prerelease=" + filters.IncludePrerelease.ToString().ToLowerInvariant();
if (filters.IncludeDelisted)
{
queryString += "&includeDelisted=true";
}
if (filters.SupportedFrameworks != null &&
filters.SupportedFrameworks.Any())
{
var frameworks =
string.Join("&",
filters.SupportedFrameworks.Select(
fx => "supportedFramework=" + fx.ToString()));
queryString += "&" + frameworks;
}
if (filters.PackageTypes != null &&
filters.PackageTypes.Any())
{
var types = string.Join("&",
filters.PackageTypes.Select(
s => "packageTypeFilter=" + s));
queryString += "&" + types;
}
queryUrl.Query = queryString;
if (!cancellationToken.IsCancellationRequested)
{
JObject searchJson = null;
try
{
searchJson = await _client.GetJObjectAsync(
uri : queryUrl.Uri,
ignoreNotFounds : false,
log : log,
token : cancellationToken);
}
catch when(i < _searchEndpoints.Length - 1)
{
// Ignore all failures until the last endpoint
}
/// <summary>
/// Retrieve entries for the given index page entries.
/// </summary>
public async Task <List <CatalogEntry> > GetEntriesAsync(IEnumerable <CatalogPageEntry> pages, CancellationToken token)
{
var tasks = pages.Select(page =>
new Func <Task <JObject> >(() =>
_httpSource.GetJObjectAsync(page.Uri, _cacheContext, _log, token)));
var maxThreads = Math.Max(1, MaxThreads);
var cache = new ReferenceCache();
var entries = new ConcurrentBag <CatalogEntry>();
var process = new Func <Task <JObject>, Task <bool> >(e => CompleteTaskAsync(e, cache, entries));
await TaskUtils.RunAsync(tasks,
useTaskRun : false,
maxThreads : maxThreads,
process : process,
token : token);
return(entries.ToList());
}
// Get a temp API key from nuget.org for pushing to https://nuget.smbsrc.net/
private async Task <string> GetSecureApiKey(
PackageIdentity packageIdentity,
string apiKey,
bool noServiceEndpoint,
TimeSpan requestTimeout,
ILogger logger,
CancellationToken token)
{
if (string.IsNullOrEmpty(apiKey))
{
return(apiKey);
}
var serviceEndpointUrl = GetServiceEndpointUrl(NuGetConstants.DefaultGalleryServerUrl,
string.Format(TempApiKeyServiceEndpoint, packageIdentity.Id, packageIdentity.Version), noServiceEndpoint);
try
{
var result = await _httpSource.GetJObjectAsync(
new HttpSourceRequest(
() =>
{
var request = HttpRequestMessageFactory.Create(
HttpMethod.Post,
serviceEndpointUrl,
new HttpRequestMessageConfiguration(
logger: logger,
promptOn403: false));
request.Headers.Add(ApiKeyHeader, apiKey);
return(request);
})
{
RequestTimeout = requestTimeout,
MaxTries = 1
},
logger,
token);
return(result.Value <string>("Key") ?? InvalidApiKey);
}
catch (HttpRequestException ex)
{
#if NETCOREAPP
if (ex.Message.Contains("Response status code does not indicate success: 403 (Forbidden).", StringComparison.OrdinalIgnoreCase))
#else
if (ex.Message.Contains("Response status code does not indicate success: 403 (Forbidden)."))
#endif
{
return(InvalidApiKey);
}
throw;
}
}
/// <summary>
/// Ensure index.json has been loaded.
/// </summary>
protected async Task EnsureServiceIndexAsync(Uri uri, CancellationToken token)
{
if (_serviceIndex == null)
{
await EnsureHttpSourceAsync();
var index = await _httpSource.GetJObjectAsync(_indexUri, _cacheContext, _log, token);
var resources = (index["resources"] as JArray);
if (resources == null)
{
throw new InvalidOperationException($"{uri.AbsoluteUri} does not contain a 'resources' property. Use the root service index.json for the nuget v3 feed.");
}
_serviceIndex = new ServiceIndexResourceV3(index, DateTime.UtcNow);
}
}
public async Task HttpSource_TimesOutDownload()
{
// Arrange
using (var td = TestDirectory.Create())
{
var expectedMilliseconds = 250;
var server = new TcpListenerServer
{
Mode = TestServerMode.SlowResponseBody,
SleepDuration = TimeSpan.FromSeconds(10)
};
var packageSource = new PackageSource(FakeSource);
var handler = new HttpClientHandler();
var handlerResource = new HttpHandlerResourceV3(handler, handler);
var httpSource = new HttpSource(
packageSource,
() => Task.FromResult((HttpHandlerResource)handlerResource),
NullThrottle.Instance)
{
HttpCacheDirectory = td
};
var logger = new TestLogger();
// Act & Assert
var actual = await Assert.ThrowsAsync <IOException>(() =>
server.ExecuteAsync(uri => httpSource.GetJObjectAsync(
new HttpSourceRequest(uri, logger)
{
DownloadTimeout = TimeSpan.FromMilliseconds(expectedMilliseconds)
},
logger,
CancellationToken.None)));
Assert.IsType <TimeoutException>(actual.InnerException);
Assert.EndsWith(
$"timed out because no data was received for {expectedMilliseconds}ms.",
actual.Message);
}
}
