public IDictionary <int, List <int> > GetBanlists(BanlistType banlistType, string banlistUrl)
{
var articleIdsList = new Dictionary <int, List <int> >();
var banlistUrlsByYear = _banlistWebPage.GetBanlistUrlList(banlistType, banlistUrl);
foreach (var banlist in banlistUrlsByYear)
{
var banlistYear = int.Parse(banlist.Key);
var articleIds = new List <int>();
foreach (var url in banlist.Value)
{
var banlistPageHtml = _htmlWebPage.Load(url).DocumentNode.InnerHtml;
Regex regex = new Regex("wgArticleId=([^,]*),");
Match match = regex.Match(banlistPageHtml);
var wgArticleId = int.Parse(match.Groups[1].Value);
articleIds.Add(wgArticleId);
}
articleIdsList.Add(banlistYear, articleIds);
}
return(articleIdsList);
}
public async Task <ArticleBatchTaskResult> Process(BanlistType banlistType)
{
var response = new ArticleBatchTaskResult();
const string baseBanlistUrl = "http://yugioh.fandom.com/wiki/July_1999_Lists";
var banListArticleIds = _banlistUrlDataSource.GetBanlists(banlistType, baseBanlistUrl);
foreach (var(year, banlistIds) in banListArticleIds)
{
_logger.LogInformation("{@BanlistType} banlists for the year: {@Year}", banlistType.ToString().ToUpper(), year);
foreach (var articleId in banlistIds)
{
_logger.LogInformation("{@BanlistType} banlist articleId: {@ArticleId}", banlistType.ToString().ToUpper(), articleId);
var articleResult = await _articleProcessor.Process(ArticleCategory.ForbiddenAndLimited, new UnexpandedArticle { Id = articleId });
if (articleResult.IsSuccessfullyProcessed)
{
response.Processed += 1;
}
}
}
return(response);
}
public async Task <ArticleBatchTaskResult> Process(BanlistType banlistType)
{
var response = new ArticleBatchTaskResult();
const string baseBanlistUrl = "http://yugioh.wikia.com/wiki/July_1999_Lists";
var banListArticleIds = _banlistUrlDataSource.GetBanlists(banlistType, baseBanlistUrl);
var articleItemProcessor = _articleHandler.Handler(ArticleCategory.ForbiddenAndLimited);
foreach (var banListArticleId in banListArticleIds)
{
_logger.Info($"{banlistType.ToString().ToUpper()} banlists for the year: {banListArticleId.Key}");
foreach (var articleId in banListArticleId.Value)
{
_logger.Info($"{banlistType.ToString().ToUpper()} banlist articleId: {articleId}");
var articleResult = await articleItemProcessor.ProcessItem(new UnexpandedArticle { Id = articleId });
if (articleResult.IsSuccessfullyProcessed)
{
response.Processed += 1;
}
}
}
return(response);
}
public HtmlNode GetBanlistHtmlNode(BanlistType banlistType, HtmlDocument document)
{
return(document
.DocumentNode
.SelectSingleNode($"//*[contains(@class,'nowraplinks navbox-subgroup')]/tr/th/i[contains(text(), '{banlistType.ToString().ToUpper()}')]")
.ParentNode
.ParentNode
.SelectSingleNode("./td/table/tr[1]/td[1]/div/ul"));
}
public void Given_A_Banlist_Web_Url_And_BanlistType_Should_Contain_Key(BanlistType banlistType, string banlistUrl)
{
// Arrange
// Act
var banlistUrls = _sut.GetBanlistUrlList(banlistType, banlistUrl);
// Assert
banlistUrls.Should().ContainKey("2005");
}
public Dictionary <string, List <Uri> > GetBanlistUrlList(BanlistType banlistType, string banlistUrl)
{
var banlistUrlListNode = _banlistHtmlDocument.GetBanlistHtmlNode(banlistType, banlistUrl);
return(GetBanlistUrlList(banlistUrlListNode));
}
public HtmlNode GetBanlistHtmlNode(BanlistType banlistType, string banlistUrl)
{
return(GetBanlistHtmlNode(banlistType, _htmlWebPage.Load(banlistUrl)));
}
public void Given_A_Banlist_TitleText_Should_Extract_Banlist_Type(string titleText, BanlistType expected)
{
// Arrange
// Act
var result = BanlistHelpers.ExtractBanlistArticleDetails(Arg.Any <int>(), titleText);
// Assert
result.BanlistType.Should().Be(expected);
}
public void Given_A_Banlist_Web_Url_And_BanlistType_Should_Have_A_Count_Greater_Than_1(BanlistType banlistType, string banlistUrl)
{
// Arrange
// Act
var banlistUrls = _sut.GetBanlistUrlList(banlistType, banlistUrl);
// Assert
banlistUrls.Should().HaveCountGreaterThan(1);
}
public void Given_A_Banlist_Web_Url_And_BanlistType_Should_Return_HtmlNode_List_Of_Banlists_Ids(BanlistType banlistType, string banlistUrl)
{
// Arrange
// Act
var banlistsIds = _sut.GetBanlists(banlistType, banlistUrl);
// Assert
banlistsIds.Should().ContainKey(2005);
}
public void Given_A_Banlist_Web_Url_And_BanlistType_Should_Return_HtmlNode_List_Of_Available_Banlists(BanlistType banlistType, string banlistUrl)
{
// Arrange
// Act
var htmlNodeList = _sut.GetBanlistHtmlNode(banlistType, banlistUrl);
// Assert
htmlNodeList.Name.Should().BeEquivalentTo("ul");
}
