private async Task <bool> CompleteTaskAsync(Task <JObject> task, ReferenceCache cache, ConcurrentBag <CatalogEntry> entries) { var json = await task; foreach (var item in json["items"]) { // Store the url in pieces so it can be cached. // Split on / var urlParts = item["@id"] .ToObject <string>().Split('/') .Select(s => cache.GetString(s)) .ToArray(); var entry = new CatalogEntry( urlParts, cache.GetString(item["@type"].ToObject <string>()), cache.GetString(item["commitId"].ToObject <string>()), cache.GetDate(item["commitTimeStamp"].ToObject <string>()), cache.GetString(item["nuget:id"].ToObject <string>()), cache.GetVersion(item["nuget:version"].ToObject <string>()), _serviceIndex, GetJson, GetNuspec, GetNupkg); entries.Add(entry); } return(true); }
private List <CatalogEntry> GetEntriesFromJson(JObject json, ReferenceCache cache) { var entries = new List <CatalogEntry>(); foreach (var item in json["items"]) { // Store the url in pieces so it can be cached. // Split on / var urlParts = item["@id"] .ToObject <string>().Split('/') .Select(s => cache.GetString(s)) .ToArray(); var entry = new CatalogEntry( urlParts, cache.GetString(item["@type"].ToObject <string>()), cache.GetString(item["commitId"].ToObject <string>()), cache.GetDate(item["commitTimeStamp"].ToObject <string>()), cache.GetString(item["nuget:id"].ToObject <string>()), cache.GetVersion(item["nuget:version"].ToObject <string>()), _serviceIndex, GetJson, GetNuspec, GetNupkg); entries.Add(entry); } return(entries); }
/// <summary> /// Retrieve entries for the given index page entries. /// </summary> public async Task <List <CatalogEntry> > GetEntriesAsync(IEnumerable <CatalogPageEntry> pages, CancellationToken token) { var maxThreads = Math.Max(1, MaxThreads); var cache = new ReferenceCache(); var entries = new List <CatalogEntry>(); var tasks = new List <Task <JObject> >(maxThreads); foreach (var page in pages) { token.ThrowIfCancellationRequested(); while (tasks.Count > maxThreads) { entries.AddRange(await CompleteTaskAsync(tasks, cache)); } tasks.Add(_httpSource.GetJObjectAsync(page.Uri, _cacheContext, _log, token)); } while (tasks.Count > 0) { entries.AddRange(await CompleteTaskAsync(tasks, cache)); } return(entries); }
/// <summary> /// Retrieve entries for the given index page entries. /// </summary> public async Task <List <CatalogEntry> > GetEntriesAsync(IEnumerable <CatalogPageEntry> pages, CancellationToken token) { var tasks = pages.Select(page => new Func <Task <JObject> >(() => _httpSource.GetJObjectAsync(page.Uri, _cacheContext, _log, token))); var maxThreads = Math.Max(1, MaxThreads); var cache = new ReferenceCache(); var entries = new ConcurrentBag <CatalogEntry>(); var process = new Func <Task <JObject>, Task <bool> >(e => CompleteTaskAsync(e, cache, entries)); await TaskUtils.RunAsync(tasks, useTaskRun : false, maxThreads : maxThreads, process : process, token : token); return(entries.ToList()); }
private async Task <List <CatalogEntry> > CompleteTaskAsync(List <Task <JObject> > tasks, ReferenceCache cache) { var entries = new List <CatalogEntry>(); if (tasks.Count > 0) { var task = await Task.WhenAny(tasks); var json = await task; tasks.Remove(task); entries.AddRange(GetEntriesFromJson(json, cache)); } return(entries); }