Beispiel #1
0
        public async Task ProcessAsync(IEnumerable <CatalogLeafItem> catalogLeafItems, CancellationToken cancellationToken = default)
        {
            await ParallelAsync.RunAsync(
                new ConcurrentBag <CatalogLeafItem>(catalogLeafItems),
                _leafProcessor.ProcessAsync,
                cancellationToken);

            await _leafProcessor.CompleteAsync(cancellationToken);
        }
        private async Task ProduceIndexActionsAsync(
            ChannelWriter <IndexAction <KeyedDocument> > channel,
            ConcurrentBag <string> packageIds,
            CancellationToken cancellationToken)
        {
            await ParallelAsync.RunAsync(
                packageIds,
                ProduceIndexActionsAsync,
                cancellationToken);

            _logger.LogInformation("Finished producing index actions");
            channel.Complete();
            return;

            async Task ProduceIndexActionsAsync(string packageId, CancellationToken cancellationToken1)
            {
                _logger.LogInformation("Adding package {PackageId}...", packageId);

                var packages = await _packages.FindAsync(packageId, includeUnlisted : false);

                if (packages.Count == 0)
                {
                    _logger.LogWarning(
                        "Could not find any listed packages for package ID {PackageId}, skipping...",
                        packageId);
                    return;
                }

                var registration = new PackageRegistration(
                    packageId,
                    packages);

                foreach (var action in _actionBuilder.AddPackage(registration))
                {
                    if (!channel.TryWrite(action))
                    {
                        await channel.WriteAsync(action, cancellationToken);
                    }
                }
            }
        }
        public async Task ProcessAsync(
            IEnumerable <CatalogLeafItem> catalogLeafItems,
            CancellationToken cancellationToken = default)
        {
            var messages = catalogLeafItems.Select(ToMessage).ToList();

            // Split large message batches and process them in parallel.
            if (messages.Count > MaxBatchElements)
            {
                var messageBatches = SplitMessages(messages);

                await ParallelAsync.RunAsync(
                    messageBatches,
                    _queue.SendAsync,
                    cancellationToken);
            }
            else
            {
                await _queue.SendAsync(messages, cancellationToken);
            }
        }
        public async static Task <(CatalogIndex, IEnumerable <CatalogLeafItem>)> LoadCatalogAsync(
            this ICatalogClient catalogClient,
            DateTimeOffset minCursor,
            DateTimeOffset maxCursor,
            ILogger logger,
            CancellationToken cancellationToken)
        {
            var catalogIndex = await catalogClient.GetIndexAsync(cancellationToken);

            var catalogLeafItems = new ConcurrentBag <CatalogLeafItem>();
            var catalogPageUrls  = new ConcurrentBag <CatalogPageItem>(
                catalogIndex.GetPagesInBounds(minCursor, maxCursor));

            await ParallelAsync.RunAsync(
                catalogPageUrls,
                ProcessCatalogPageAsync,
                cancellationToken);

            return(catalogIndex, catalogLeafItems);

            async Task ProcessCatalogPageAsync(CatalogPageItem pageItem, CancellationToken token)
            {
                logger.LogInformation("Processing catalog page {CatalogPageUrl}...", pageItem.CatalogPageUrl);

                var page = await catalogClient.GetPageAsync(pageItem.CatalogPageUrl, token);

                var leafs = page.GetLeavesInBounds(minCursor, maxCursor, excludeRedundantLeaves: true);

                foreach (var catalogLeafItem in leafs)
                {
                    catalogLeafItems.Add(catalogLeafItem);
                }

                logger.LogInformation("Processed catalog page {CatalogPageUrl}", pageItem.CatalogPageUrl);
            }
        }