public async Task <ArticleBatchTaskResult> Process(BanlistType banlistType)
{
var response = new ArticleBatchTaskResult();
const string baseBanlistUrl = "http://yugioh.wikia.com/wiki/July_1999_Lists";
var banListArticleIds = _banlistUrlDataSource.GetBanlists(banlistType, baseBanlistUrl);
var articleItemProcessor = _articleHandler.Handler(ArticleCategory.ForbiddenAndLimited);
foreach (var banListArticleId in banListArticleIds)
{
_logger.Info($"{banlistType.ToString().ToUpper()} banlists for the year: {banListArticleId.Key}");
foreach (var articleId in banListArticleId.Value)
{
_logger.Info($"{banlistType.ToString().ToUpper()} banlist articleId: {articleId}");
var articleResult = await articleItemProcessor.ProcessItem(new UnexpandedArticle { Id = articleId });
if (articleResult.IsSuccessfullyProcessed)
{
response.Processed += 1;
}
}
}
return(response);
}
public async Task <ArticleBatchTaskResult> Process(string category, UnexpandedArticle[] articles)
{
if (string.IsNullOrWhiteSpace(category))
{
throw new ArgumentException(nameof(category));
}
if (articles == null)
{
throw new ArgumentException(nameof(articles));
}
var response = new ArticleBatchTaskResult();
foreach (var article in articles)
{
try
{
var result = await _articleProcessor.Process(category, article);
if (result.IsSuccessfullyProcessed)
{
response.Processed += 1;
}
}
catch (Exception ex)
{
response.Failed.Add(new ArticleException {
Article = article, Exception = ex
});
}
}
return(response);
}
public async Task <ArticleBatchTaskResult> Process(BanlistType banlistType)
{
var response = new ArticleBatchTaskResult();
const string baseBanlistUrl = "http://yugioh.fandom.com/wiki/July_1999_Lists";
var banListArticleIds = _banlistUrlDataSource.GetBanlists(banlistType, baseBanlistUrl);
foreach (var(year, banlistIds) in banListArticleIds)
{
_logger.LogInformation("{@BanlistType} banlists for the year: {@Year}", banlistType.ToString().ToUpper(), year);
foreach (var articleId in banlistIds)
{
_logger.LogInformation("{@BanlistType} banlist articleId: {@ArticleId}", banlistType.ToString().ToUpper(), articleId);
var articleResult = await _articleProcessor.Process(ArticleCategory.ForbiddenAndLimited, new UnexpandedArticle { Id = articleId });
if (articleResult.IsSuccessfullyProcessed)
{
response.Processed += 1;
}
}
}
return(response);
}
public async Task <ArticleBatchTaskResult> Process(string category, int pageSize)
{
var response = new ArticleBatchTaskResult {
Category = category
};
await foreach (var unexpandedArticleBatch in _articleCategoryDataSource.Producer(category, pageSize))
{
var articleBatchTaskResult = await _articleBatchProcessor.Process(category, unexpandedArticleBatch);
response.Processed += articleBatchTaskResult.Processed;
response.Failed.AddRange(articleBatchTaskResult.Failed);
}
return(response);
}
public Task <ArticleBatchTaskResult> Process(string category, int pageSize)
{
var response = new ArticleBatchTaskResult {
Category = category
};
var processorCount = Environment.ProcessorCount;
// Pipeline members
var articleBatchBufferBlock = new BufferBlock <UnexpandedArticle[]>();
var articleTransformBlock = new TransformBlock <UnexpandedArticle[], ArticleBatchTaskResult>(articles => _articleBatchProcessor.Process(category, articles));
var articleActionBlock = new ActionBlock <ArticleBatchTaskResult>(delegate(ArticleBatchTaskResult result)
{
response.Processed += result.Processed;
response.Failed.AddRange(result.Failed);
},
// Specify a maximum degree of parallelism.
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = processorCount
});
// Form the pipeline
articleBatchBufferBlock.LinkTo(articleTransformBlock);
articleTransformBlock.LinkTo(articleActionBlock);
// Create the completion tasks:
articleBatchBufferBlock.Completion
.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)articleTransformBlock).Fault(t.Exception);
}
else
{
articleTransformBlock.Complete();
}
});
articleTransformBlock.Completion
.ContinueWith(t =>
{
if (t.IsFaulted)
{
((IDataflowBlock)articleActionBlock).Fault(t.Exception);
}
else
{
articleActionBlock.Complete();
}
});
// Process "Category" and generate article batch data
_articleCategoryDataSource.Producer(category, pageSize, articleBatchBufferBlock);
// Mark the head of the pipeline as complete. The continuation tasks
// propagate completion through the pipeline as each part of the
// pipeline finishes.
articleActionBlock.Completion.Wait();
return(Task.FromResult(response));
}