protected override async Task Fetch(CollectorHttpClient client, Uri index, DateTime last)
{
IList<JObject> items = new List<JObject>();
JObject root = await client.GetJObjectAsync(index);
JToken context = null;
root.TryGetValue("@context", out context);
IEnumerable<JToken> rootItems = root["item"].OrderBy(item => item["timeStamp"]["@value"].ToObject<DateTime>());
foreach (JObject rootItem in rootItems)
{
DateTime pageTimeStamp = rootItem["timeStamp"]["@value"].ToObject<DateTime>();
if (pageTimeStamp > last)
{
Uri pageUri = rootItem["url"].ToObject<Uri>();
JObject page = await client.GetJObjectAsync(pageUri);
IEnumerable<JToken> pageItems = page["item"].OrderBy(item => item["timeStamp"]["@value"].ToObject<DateTime>());
foreach (JObject pageItem in pageItems)
{
DateTime itemTimeStamp = pageItem["timeStamp"]["@value"].ToObject<DateTime>();
if (itemTimeStamp > last)
{
Uri itemUri = pageItem["url"].ToObject<Uri>();
items.Add(pageItem);
if (items.Count == _batchSize)
{
await ProcessBatch(client, items, (JObject)context);
BatchCount++;
items.Clear();
}
}
}
}
}
if (items.Count > 0)
{
await ProcessBatch(client, items, (JObject)context);
BatchCount++;
}
}
protected override async Task ProcessBatch(CollectorHttpClient client, IList<JObject> items, JObject context)
{
List<Task<JObject>> tasks = new List<Task<JObject>>();
foreach (JObject item in items)
{
Uri itemUri = new Uri(item["url"].ToString());
string type = item["@type"].ToString();
if (type == "TestItem")
{
tasks.Add(client.GetJObjectAsync(itemUri));
}
}
await Task.WhenAll(tasks.ToArray());
foreach (Task<JObject> task in tasks)
{
Console.WriteLine(task.Result["name"]);
}
}
