public async Task <ArticleBatchTaskResult> Process(BanlistType banlistType) { var response = new ArticleBatchTaskResult(); const string baseBanlistUrl = "http://yugioh.wikia.com/wiki/July_1999_Lists"; var banListArticleIds = _banlistUrlDataSource.GetBanlists(banlistType, baseBanlistUrl); var articleItemProcessor = _articleHandler.Handler(ArticleCategory.ForbiddenAndLimited); foreach (var banListArticleId in banListArticleIds) { _logger.Info($"{banlistType.ToString().ToUpper()} banlists for the year: {banListArticleId.Key}"); foreach (var articleId in banListArticleId.Value) { _logger.Info($"{banlistType.ToString().ToUpper()} banlist articleId: {articleId}"); var articleResult = await articleItemProcessor.ProcessItem(new UnexpandedArticle { Id = articleId }); if (articleResult.IsSuccessfullyProcessed) { response.Processed += 1; } } } return(response); }
public async Task <ArticleBatchTaskResult> Process(string category, UnexpandedArticle[] articles) { if (string.IsNullOrWhiteSpace(category)) { throw new ArgumentException(nameof(category)); } if (articles == null) { throw new ArgumentException(nameof(articles)); } var response = new ArticleBatchTaskResult(); foreach (var article in articles) { try { var result = await _articleProcessor.Process(category, article); if (result.IsSuccessfullyProcessed) { response.Processed += 1; } } catch (Exception ex) { response.Failed.Add(new ArticleException { Article = article, Exception = ex }); } } return(response); }
public async Task <ArticleBatchTaskResult> Process(BanlistType banlistType) { var response = new ArticleBatchTaskResult(); const string baseBanlistUrl = "http://yugioh.fandom.com/wiki/July_1999_Lists"; var banListArticleIds = _banlistUrlDataSource.GetBanlists(banlistType, baseBanlistUrl); foreach (var(year, banlistIds) in banListArticleIds) { _logger.LogInformation("{@BanlistType} banlists for the year: {@Year}", banlistType.ToString().ToUpper(), year); foreach (var articleId in banlistIds) { _logger.LogInformation("{@BanlistType} banlist articleId: {@ArticleId}", banlistType.ToString().ToUpper(), articleId); var articleResult = await _articleProcessor.Process(ArticleCategory.ForbiddenAndLimited, new UnexpandedArticle { Id = articleId }); if (articleResult.IsSuccessfullyProcessed) { response.Processed += 1; } } } return(response); }
public async Task <ArticleBatchTaskResult> Process(string category, int pageSize) { var response = new ArticleBatchTaskResult { Category = category }; await foreach (var unexpandedArticleBatch in _articleCategoryDataSource.Producer(category, pageSize)) { var articleBatchTaskResult = await _articleBatchProcessor.Process(category, unexpandedArticleBatch); response.Processed += articleBatchTaskResult.Processed; response.Failed.AddRange(articleBatchTaskResult.Failed); } return(response); }
public Task <ArticleBatchTaskResult> Process(string category, int pageSize) { var response = new ArticleBatchTaskResult { Category = category }; var processorCount = Environment.ProcessorCount; // Pipeline members var articleBatchBufferBlock = new BufferBlock <UnexpandedArticle[]>(); var articleTransformBlock = new TransformBlock <UnexpandedArticle[], ArticleBatchTaskResult>(articles => _articleBatchProcessor.Process(category, articles)); var articleActionBlock = new ActionBlock <ArticleBatchTaskResult>(delegate(ArticleBatchTaskResult result) { response.Processed += result.Processed; response.Failed.AddRange(result.Failed); }, // Specify a maximum degree of parallelism. new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = processorCount }); // Form the pipeline articleBatchBufferBlock.LinkTo(articleTransformBlock); articleTransformBlock.LinkTo(articleActionBlock); // Create the completion tasks: articleBatchBufferBlock.Completion .ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)articleTransformBlock).Fault(t.Exception); } else { articleTransformBlock.Complete(); } }); articleTransformBlock.Completion .ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)articleActionBlock).Fault(t.Exception); } else { articleActionBlock.Complete(); } }); // Process "Category" and generate article batch data _articleCategoryDataSource.Producer(category, pageSize, articleBatchBufferBlock); // Mark the head of the pipeline as complete. The continuation tasks // propagate completion through the pipeline as each part of the // pipeline finishes. articleActionBlock.Completion.Wait(); return(Task.FromResult(response)); }