private async Task ProcessPageAsync(DateTimeOffset minCommitTimestamp, CatalogPageItem pageItem, CancellationToken token) { var page = await _client.GetPageAsync(pageItem.Url, token); var leafItems = page.GetLeavesInBounds( minCommitTimestamp, _settings.MaxCommitTimestamp, _settings.ExcludeRedundantLeaves); _logger.LogInformation( "On page {page}, {leaves} out of {totalLeaves} were in the time bounds.", pageItem.Url, leafItems.Count, page.Items.Count); DateTimeOffset?newCursor = null; var tasks = new List <Task <CatalogLeaf> >(); foreach (var batch in leafItems .Select((v, i) => new { Index = i, Value = v }) .GroupBy(v => v.Index / 25) .Select(v => v.Select(p => p.Value))) { foreach (var leafItem in batch) { newCursor = leafItem.CommitTimestamp; tasks.Add(ProcessLeafAsync(leafItem, token)); } await Task.WhenAll(tasks); foreach (var task in tasks) { if (task.Result is PackageDeleteCatalogLeaf del) { await _leafProcessor.ProcessPackageDeleteAsync(del, token); } else if (task.Result is PackageDetailsCatalogLeaf detail) { await _leafProcessor.ProcessPackageDetailsAsync(detail, token); } else { _logger.LogError("Unsupported leaf type: {type}.", task.Result?.GetType()); } } tasks.Clear(); } if (newCursor.HasValue) { await _cursor.SetAsync(newCursor.Value, token); } }
private async Task <bool> ProcessPageAsync(DateTimeOffset minCommitTimestamp, CatalogPageItem pageItem) { var page = await _client.GetPageAsync(pageItem.Url); var leafItems = page.GetLeavesInBounds( minCommitTimestamp, _settings.MaxCommitTimestamp, _settings.ExcludeRedundantLeaves); _logger.LogInformation( "On page {page}, {leaves} out of {totalLeaves} were in the time bounds.", pageItem.Url, leafItems.Count, page.Items.Count); DateTimeOffset?newCursor = null; var success = true; for (var i = 0; i < leafItems.Count; i++) { var leafItem = leafItems[i]; if (newCursor.HasValue && newCursor.Value != leafItem.CommitTimestamp) { await _cursor.SetAsync(newCursor.Value); } newCursor = leafItem.CommitTimestamp; success = await ProcessLeafAsync(leafItem); if (!success) { _logger.LogWarning( "{unprocessedLeaves} out of {leaves} leaves were left incomplete due to a processing failure.", leafItems.Count - i, leafItems.Count); break; } } if (newCursor.HasValue && success) { await _cursor.SetAsync(newCursor.Value); } return(success); }