private async Task <ConcurrentBag <CatalogIndexEntry> > GetEntriesAsync(IEnumerable <Uri> pageUris)
{
var pageUriBag = new ConcurrentBag <Uri>(pageUris);
var entries = new ConcurrentBag <CatalogIndexEntry>();
var interner = new StringInterner();
var tasks = Enumerable
.Range(0, ServicePointManager.DefaultConnectionLimit)
.Select(i => ProcessPageUris(pageUriBag, entries, interner))
.ToList();
await Task.WhenAll(tasks);
return(entries);
}
private async Task ProcessPageUris(ConcurrentBag <Uri> pageUriBag, ConcurrentBag <CatalogIndexEntry> entries, StringInterner interner)
{
await Task.Yield();
Uri pageUri;
while (pageUriBag.TryTake(out pageUri))
{
var json = await _httpClient.GetJObjectAsync(pageUri);
foreach (var item in json["items"])
{
// This string is unique.
var id = item["@id"].ToString();
// These strings should be shared.
var type = interner.Intern(item["@type"].ToString());
var commitId = interner.Intern(item["commitId"].ToString());
var nugetId = interner.Intern(item["nuget:id"].ToString());
var nugetVersion = interner.Intern(item["nuget:version"].ToString());
// No string is directly operated on here.
var commitTimeStamp = item["commitTimeStamp"].ToObject <DateTime>();
var entry = new CatalogIndexEntry(
new Uri(id),
type,
commitId,
commitTimeStamp,
nugetId,
NuGetVersion.Parse(nugetVersion));
entries.Add(entry);
}
}
}
