public async Task ProcessAsync(IEnumerable <CatalogLeafItem> catalogLeafItems, CancellationToken cancellationToken = default)
{
await ParallelAsync.RunAsync(
new ConcurrentBag <CatalogLeafItem>(catalogLeafItems),
_leafProcessor.ProcessAsync,
cancellationToken);
await _leafProcessor.CompleteAsync(cancellationToken);
}
private async Task ProduceIndexActionsAsync(
ChannelWriter <IndexAction <KeyedDocument> > channel,
ConcurrentBag <string> packageIds,
CancellationToken cancellationToken)
{
await ParallelAsync.RunAsync(
packageIds,
ProduceIndexActionsAsync,
cancellationToken);
_logger.LogInformation("Finished producing index actions");
channel.Complete();
return;
async Task ProduceIndexActionsAsync(string packageId, CancellationToken cancellationToken1)
{
_logger.LogInformation("Adding package {PackageId}...", packageId);
var packages = await _packages.FindAsync(packageId, includeUnlisted : false);
if (packages.Count == 0)
{
_logger.LogWarning(
"Could not find any listed packages for package ID {PackageId}, skipping...",
packageId);
return;
}
var registration = new PackageRegistration(
packageId,
packages);
foreach (var action in _actionBuilder.AddPackage(registration))
{
if (!channel.TryWrite(action))
{
await channel.WriteAsync(action, cancellationToken);
}
}
}
}
public async Task ProcessAsync(
IEnumerable <CatalogLeafItem> catalogLeafItems,
CancellationToken cancellationToken = default)
{
var messages = catalogLeafItems.Select(ToMessage).ToList();
// Split large message batches and process them in parallel.
if (messages.Count > MaxBatchElements)
{
var messageBatches = SplitMessages(messages);
await ParallelAsync.RunAsync(
messageBatches,
_queue.SendAsync,
cancellationToken);
}
else
{
await _queue.SendAsync(messages, cancellationToken);
}
}
public async static Task <(CatalogIndex, IEnumerable <CatalogLeafItem>)> LoadCatalogAsync(
this ICatalogClient catalogClient,
DateTimeOffset minCursor,
DateTimeOffset maxCursor,
ILogger logger,
CancellationToken cancellationToken)
{
var catalogIndex = await catalogClient.GetIndexAsync(cancellationToken);
var catalogLeafItems = new ConcurrentBag <CatalogLeafItem>();
var catalogPageUrls = new ConcurrentBag <CatalogPageItem>(
catalogIndex.GetPagesInBounds(minCursor, maxCursor));
await ParallelAsync.RunAsync(
catalogPageUrls,
ProcessCatalogPageAsync,
cancellationToken);
return(catalogIndex, catalogLeafItems);
async Task ProcessCatalogPageAsync(CatalogPageItem pageItem, CancellationToken token)
{
logger.LogInformation("Processing catalog page {CatalogPageUrl}...", pageItem.CatalogPageUrl);
var page = await catalogClient.GetPageAsync(pageItem.CatalogPageUrl, token);
var leafs = page.GetLeavesInBounds(minCursor, maxCursor, excludeRedundantLeaves: true);
foreach (var catalogLeafItem in leafs)
{
catalogLeafItems.Add(catalogLeafItem);
}
logger.LogInformation("Processed catalog page {CatalogPageUrl}", pageItem.CatalogPageUrl);
}
}
