protected override async Task<bool> Fetch(CollectorHttpClient client, ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken)
{
JObject root = await client.GetJObjectAsync(Index, cancellationToken);
IEnumerable<CatalogItem> rootItems = root["items"]
.Select(item => new CatalogItem(item))
.Where(item => item.CommitTimeStamp > front.Value)
.OrderBy(item => item.CommitTimeStamp);
bool acceptNextBatch = false;
foreach (CatalogItem rootItem in rootItems)
{
JObject page = await client.GetJObjectAsync(rootItem.Uri, cancellationToken);
JToken context = null;
page.TryGetValue("@context", out context);
var batches = await CreateBatches(page["items"]
.Select(item => new CatalogItem(item))
.Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value));
var orderedBatches = batches
.OrderBy(batch => batch.CommitTimeStamp)
.ToList();
var lastBatch = orderedBatches.LastOrDefault();
foreach (var batch in orderedBatches)
{
acceptNextBatch = await OnProcessBatch(
client,
batch.Items.Select(item => item.Value),
context,
batch.CommitTimeStamp,
batch.CommitTimeStamp == lastBatch.CommitTimeStamp,
cancellationToken);
front.Value = batch.CommitTimeStamp;
await front.Save(cancellationToken);
Trace.TraceInformation("CommitCatalog.Fetch front.Save has value: {0}", front);
if (!acceptNextBatch)
{
break;
}
}
if (!acceptNextBatch)
{
break;
}
}
return acceptNextBatch;
}
async Task <bool> ProcessBatch(CollectorHttpClient client, IList <JObject> items, JToken context, ReadWriteCursor front, DateTime resumeDateTime)
{
bool acceptNextBatch = await OnProcessBatch(client, items, (JObject)context, resumeDateTime);
BatchCount++;
items.Clear();
front.Value = resumeDateTime;
await front.Save();
return(acceptNextBatch);
}
protected override async Task <bool> Fetch(CollectorHttpClient client, ReadWriteCursor front, ReadCursor back)
{
IList <JObject> items = new List <JObject>();
JObject root = await client.GetJObjectAsync(Index);
IEnumerable <CatalogItem> rootItems = root["items"]
.Select(item => new CatalogItem(item))
.Where(item => item.CommitTimeStamp > front.Value)
.OrderBy(item => item.CommitTimeStamp);
bool acceptNextBatch = false;
foreach (CatalogItem rootItem in rootItems)
{
JObject page = await client.GetJObjectAsync(rootItem.Uri);
JToken context = null;
page.TryGetValue("@context", out context);
var batches = page["items"]
.Select(item => new CatalogItem(item))
.Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value)
.GroupBy(item => item.CommitTimeStamp)
.OrderBy(group => group.Key);
foreach (var batch in batches)
{
acceptNextBatch = await OnProcessBatch(client, batch.Select(item => item.Value), context, batch.Key);
front.Value = batch.Key;
await front.Save();
Trace.TraceInformation("CommitCatalog.Fetch front.Save has value: {0}", front);
if (!acceptNextBatch)
{
break;
}
}
if (!acceptNextBatch)
{
break;
}
}
return(acceptNextBatch);
}
async Task<bool> ProcessBatch(CollectorHttpClient client, IList<JObject> items, JToken context, ReadWriteCursor front, DateTime resumeDateTime, CancellationToken cancellationToken)
{
bool acceptNextBatch = await OnProcessBatch(client, items, (JObject)context, resumeDateTime, cancellationToken);
BatchCount++;
items.Clear();
front.Value = resumeDateTime;
await front.Save(cancellationToken);
return acceptNextBatch;
}