private async Task <bool> CompleteTaskAsync(Task <JObject> task, ReferenceCache cache, ConcurrentBag <CatalogEntry> entries)
{
var json = await task;
foreach (var item in json["items"])
{
// Store the url in pieces so it can be cached.
// Split on /
var urlParts = item["@id"]
.ToObject <string>().Split('/')
.Select(s => cache.GetString(s))
.ToArray();
var entry = new CatalogEntry(
urlParts,
cache.GetString(item["@type"].ToObject <string>()),
cache.GetString(item["commitId"].ToObject <string>()),
cache.GetDate(item["commitTimeStamp"].ToObject <string>()),
cache.GetString(item["nuget:id"].ToObject <string>()),
cache.GetVersion(item["nuget:version"].ToObject <string>()),
_serviceIndex,
GetJson,
GetNuspec,
GetNupkg);
entries.Add(entry);
}
return(true);
}
private List <CatalogEntry> GetEntriesFromJson(JObject json, ReferenceCache cache)
{
var entries = new List <CatalogEntry>();
foreach (var item in json["items"])
{
// Store the url in pieces so it can be cached.
// Split on /
var urlParts = item["@id"]
.ToObject <string>().Split('/')
.Select(s => cache.GetString(s))
.ToArray();
var entry = new CatalogEntry(
urlParts,
cache.GetString(item["@type"].ToObject <string>()),
cache.GetString(item["commitId"].ToObject <string>()),
cache.GetDate(item["commitTimeStamp"].ToObject <string>()),
cache.GetString(item["nuget:id"].ToObject <string>()),
cache.GetVersion(item["nuget:version"].ToObject <string>()),
_serviceIndex,
GetJson,
GetNuspec,
GetNupkg);
entries.Add(entry);
}
return(entries);
}
/// <summary>
/// Retrieve entries for the given index page entries.
/// </summary>
public async Task <List <CatalogEntry> > GetEntriesAsync(IEnumerable <CatalogPageEntry> pages, CancellationToken token)
{
var maxThreads = Math.Max(1, MaxThreads);
var cache = new ReferenceCache();
var entries = new List <CatalogEntry>();
var tasks = new List <Task <JObject> >(maxThreads);
foreach (var page in pages)
{
token.ThrowIfCancellationRequested();
while (tasks.Count > maxThreads)
{
entries.AddRange(await CompleteTaskAsync(tasks, cache));
}
tasks.Add(_httpSource.GetJObjectAsync(page.Uri, _cacheContext, _log, token));
}
while (tasks.Count > 0)
{
entries.AddRange(await CompleteTaskAsync(tasks, cache));
}
return(entries);
}
/// <summary>
/// Retrieve entries for the given index page entries.
/// </summary>
public async Task <List <CatalogEntry> > GetEntriesAsync(IEnumerable <CatalogPageEntry> pages, CancellationToken token)
{
var tasks = pages.Select(page =>
new Func <Task <JObject> >(() =>
_httpSource.GetJObjectAsync(page.Uri, _cacheContext, _log, token)));
var maxThreads = Math.Max(1, MaxThreads);
var cache = new ReferenceCache();
var entries = new ConcurrentBag <CatalogEntry>();
var process = new Func <Task <JObject>, Task <bool> >(e => CompleteTaskAsync(e, cache, entries));
await TaskUtils.RunAsync(tasks,
useTaskRun : false,
maxThreads : maxThreads,
process : process,
token : token);
return(entries.ToList());
}
private async Task <List <CatalogEntry> > CompleteTaskAsync(List <Task <JObject> > tasks, ReferenceCache cache)
{
var entries = new List <CatalogEntry>();
if (tasks.Count > 0)
{
var task = await Task.WhenAny(tasks);
var json = await task;
tasks.Remove(task);
entries.AddRange(GetEntriesFromJson(json, cache));
}
return(entries);
}
