private async Task ProcessPageAsync(DateTimeOffset minCommitTimestamp, CatalogPageItem pageItem, CancellationToken token)
{
var page = await _client.GetPageAsync(pageItem.Url, token);
var leafItems = page.GetLeavesInBounds(
minCommitTimestamp,
_settings.MaxCommitTimestamp,
_settings.ExcludeRedundantLeaves);
_logger.LogInformation(
"On page {page}, {leaves} out of {totalLeaves} were in the time bounds.",
pageItem.Url,
leafItems.Count,
page.Items.Count);
DateTimeOffset?newCursor = null;
var tasks = new List <Task <CatalogLeaf> >();
foreach (var batch in leafItems
.Select((v, i) => new { Index = i, Value = v })
.GroupBy(v => v.Index / 25)
.Select(v => v.Select(p => p.Value)))
{
foreach (var leafItem in batch)
{
newCursor = leafItem.CommitTimestamp;
tasks.Add(ProcessLeafAsync(leafItem, token));
}
await Task.WhenAll(tasks);
foreach (var task in tasks)
{
if (task.Result is PackageDeleteCatalogLeaf del)
{
await _leafProcessor.ProcessPackageDeleteAsync(del, token);
}
else if (task.Result is PackageDetailsCatalogLeaf detail)
{
await _leafProcessor.ProcessPackageDetailsAsync(detail, token);
}
else
{
_logger.LogError("Unsupported leaf type: {type}.", task.Result?.GetType());
}
}
tasks.Clear();
}
if (newCursor.HasValue)
{
await _cursor.SetAsync(newCursor.Value, token);
}
}
private async Task <bool> ProcessPageAsync(DateTimeOffset minCommitTimestamp, CatalogPageItem pageItem)
{
var page = await _client.GetPageAsync(pageItem.Url);
var leafItems = page.GetLeavesInBounds(
minCommitTimestamp,
_settings.MaxCommitTimestamp,
_settings.ExcludeRedundantLeaves);
_logger.LogInformation(
"On page {page}, {leaves} out of {totalLeaves} were in the time bounds.",
pageItem.Url,
leafItems.Count,
page.Items.Count);
DateTimeOffset?newCursor = null;
var success = true;
for (var i = 0; i < leafItems.Count; i++)
{
var leafItem = leafItems[i];
if (newCursor.HasValue && newCursor.Value != leafItem.CommitTimestamp)
{
await _cursor.SetAsync(newCursor.Value);
}
newCursor = leafItem.CommitTimestamp;
success = await ProcessLeafAsync(leafItem);
if (!success)
{
_logger.LogWarning(
"{unprocessedLeaves} out of {leaves} leaves were left incomplete due to a processing failure.",
leafItems.Count - i,
leafItems.Count);
break;
}
}
if (newCursor.HasValue && success)
{
await _cursor.SetAsync(newCursor.Value);
}
return(success);
}
