public async Task ProcessAsync(IEnumerable <CatalogLeafItem> catalogLeafItems, CancellationToken cancellationToken = default) { await ParallelAsync.RunAsync( new ConcurrentBag <CatalogLeafItem>(catalogLeafItems), _leafProcessor.ProcessAsync, cancellationToken); await _leafProcessor.CompleteAsync(cancellationToken); }
private async Task ProduceIndexActionsAsync( ChannelWriter <IndexAction <KeyedDocument> > channel, ConcurrentBag <string> packageIds, CancellationToken cancellationToken) { await ParallelAsync.RunAsync( packageIds, ProduceIndexActionsAsync, cancellationToken); _logger.LogInformation("Finished producing index actions"); channel.Complete(); return; async Task ProduceIndexActionsAsync(string packageId, CancellationToken cancellationToken1) { _logger.LogInformation("Adding package {PackageId}...", packageId); var packages = await _packages.FindAsync(packageId, includeUnlisted : false); if (packages.Count == 0) { _logger.LogWarning( "Could not find any listed packages for package ID {PackageId}, skipping...", packageId); return; } var registration = new PackageRegistration( packageId, packages); foreach (var action in _actionBuilder.AddPackage(registration)) { if (!channel.TryWrite(action)) { await channel.WriteAsync(action, cancellationToken); } } } }
public async Task ProcessAsync( IEnumerable <CatalogLeafItem> catalogLeafItems, CancellationToken cancellationToken = default) { var messages = catalogLeafItems.Select(ToMessage).ToList(); // Split large message batches and process them in parallel. if (messages.Count > MaxBatchElements) { var messageBatches = SplitMessages(messages); await ParallelAsync.RunAsync( messageBatches, _queue.SendAsync, cancellationToken); } else { await _queue.SendAsync(messages, cancellationToken); } }
public async static Task <(CatalogIndex, IEnumerable <CatalogLeafItem>)> LoadCatalogAsync( this ICatalogClient catalogClient, DateTimeOffset minCursor, DateTimeOffset maxCursor, ILogger logger, CancellationToken cancellationToken) { var catalogIndex = await catalogClient.GetIndexAsync(cancellationToken); var catalogLeafItems = new ConcurrentBag <CatalogLeafItem>(); var catalogPageUrls = new ConcurrentBag <CatalogPageItem>( catalogIndex.GetPagesInBounds(minCursor, maxCursor)); await ParallelAsync.RunAsync( catalogPageUrls, ProcessCatalogPageAsync, cancellationToken); return(catalogIndex, catalogLeafItems); async Task ProcessCatalogPageAsync(CatalogPageItem pageItem, CancellationToken token) { logger.LogInformation("Processing catalog page {CatalogPageUrl}...", pageItem.CatalogPageUrl); var page = await catalogClient.GetPageAsync(pageItem.CatalogPageUrl, token); var leafs = page.GetLeavesInBounds(minCursor, maxCursor, excludeRedundantLeaves: true); foreach (var catalogLeafItem in leafs) { catalogLeafItems.Add(catalogLeafItem); } logger.LogInformation("Processed catalog page {CatalogPageUrl}", pageItem.CatalogPageUrl); } }