private async Task <ConcurrentBag <CatalogIndexEntry> > GetEntriesAsync(IEnumerable <Uri> pageUris) { var pageUriBag = new ConcurrentBag <Uri>(pageUris); var entries = new ConcurrentBag <CatalogIndexEntry>(); var interner = new StringInterner(); var tasks = Enumerable .Range(0, ServicePointManager.DefaultConnectionLimit) .Select(i => ProcessPageUris(pageUriBag, entries, interner)) .ToList(); await Task.WhenAll(tasks); return(entries); }
private async Task ProcessPageUris(ConcurrentBag <Uri> pageUriBag, ConcurrentBag <CatalogIndexEntry> entries, StringInterner interner) { await Task.Yield(); Uri pageUri; while (pageUriBag.TryTake(out pageUri)) { var json = await _httpClient.GetJObjectAsync(pageUri); foreach (var item in json["items"]) { // This string is unique. var id = item["@id"].ToString(); // These strings should be shared. var type = interner.Intern(item["@type"].ToString()); var commitId = interner.Intern(item["commitId"].ToString()); var nugetId = interner.Intern(item["nuget:id"].ToString()); var nugetVersion = interner.Intern(item["nuget:version"].ToString()); // No string is directly operated on here. var commitTimeStamp = item["commitTimeStamp"].ToObject <DateTime>(); var entry = new CatalogIndexEntry( new Uri(id), type, commitId, commitTimeStamp, nugetId, NuGetVersion.Parse(nugetVersion)); entries.Add(entry); } } }