private void SearchArticles(string baseUrl, List <Article> allArticles, int headerSize = 1)
{
var htmlDoc = _spider.LoadPage(baseUrl);
var headers = _spider.GetHeadersOfSize(htmlDoc, headerSize);
foreach (var header in headers)
{
try
{
header.InnerText.Trim();
if (!_validator.ConsideredArticleHeader(header.InnerText))
{
continue;
}
var(articleUrl, articleHtmlDocument) = _spider.DownloadArticleByHeader(baseUrl, header);
if (articleUrl == null && articleHtmlDocument == null)
{
continue;
}
var articleBodyText = _extractor.ExtractBodyTextFromArticleDocument(articleHtmlDocument);
var article = CreateArticle(articleHtmlDocument, articleBodyText, allArticles, baseUrl, articleUrl, header.InnerText);
var foundKeywords = ScanArticleForKeyWords(article);
if (foundKeywords.Count() > 0)
{
article.Keywords = foundKeywords;
_pipeline.SendForAnalysis(article);
}
}
catch (Exception e)
{
_logger.LogError($"Failed to download and process article {header.InnerText} with the following exception {e.Message} stack trace: {e.StackTrace}");
continue;
}
}
headerSize++;
if (headerSize <= 4)
{
SearchArticles(baseUrl, allArticles, headerSize);
}
}