public bool Exists(string interestTag)
{
return(_interestTagRepository.Exists(acc => acc.TagName == interestTag));
}
/// <summary>
/// Crawl through aggregated links and check if its a valid article for the site.
/// </summary>
/// <param name="links"> List of gathered links</param>
/// <returns> List of valid Articles </returns>
public async Task <List <Article> > ArticleCrawler(List <string[]> links)
{
// Will hold the valid Articles.
var list = new List <Article>();
// Initiate new httpclient and htmlDocument to request and traverse html.
var httpClient = new HttpClient();
var htmlDoc = new HtmlDocument();
// For each article, check if valid
foreach (var art in links)
{
try
{
// Will hold the proper attributes for the site.
List <string> siteAttribute = null;
var html = await httpClient.GetStringAsync(art[0]);
htmlDoc.LoadHtml(html);
// Associate the proper attributes for the Site.
foreach (var site in Sites)
{
if (site.Key == art[1])
{
siteAttribute = site.Value;
break;
}
}
// Boolean to check if article matches a keyword/tag
bool tagMatch = false;
string[] contentTags = GatherTags(htmlDoc, siteAttribute);
tagMatch = MatchTags(contentTags);
// If tagMatched, gather the rest of the necessary Article information.
if (tagMatch)
{
// Will hold the article info.
string title = GatherTitle(htmlDoc, siteAttribute);
string description = GatherDescription(htmlDoc, siteAttribute);
// Gather Tag info
if (!interestTagRepository.Exists(a => a.TagName == siteAttribute[18]))
{
InterestTag newTag = new InterestTag()
{
TagName = siteAttribute[18],
AccountUsername = null,
ArticleTags = null
};
interestTagRepository.Insert(newTag);
Console.WriteLine($"{newTag.TagName} added to DB");
}
InterestTag tag = interestTagRepository.GetSingle(d => d.TagName.Equals(siteAttribute[18]));
// Create new Article Object and assign the gathered variables.
var goodArticle = new Article
{
// Assigns articleType by attribute[18]. ie. Technology, Medical, etc.
TagName = tag.TagName,
ArticleTitle = title,
// Assigns the articleLink as the url from the article that is being crawled.
ArticleLink = art[0],
ArticleDescription = description
};
// Ran into some articles that did not have a Title, decided to make the description the title.
if (goodArticle.ArticleTitle == "")
{
goodArticle.ArticleTitle = goodArticle.ArticleDescription;
}
//Ran into some articles that did not have a description, decided to input my own placeholder description.
if (goodArticle.ArticleDescription == "")
{
goodArticle.ArticleDescription = "Click to read article!";
}
// Add the valid article to the list of valid articles.
list.Add(goodArticle);
}
else
{
Console.WriteLine($"{art[0]} not added to {siteAttribute[18]}");
}
}
// Catch HttpRequestException if it fails to get a response from requested article.
catch (HttpRequestException e)
{
Console.WriteLine(DateTime.Now + ": Site " + art[0] + " response: " + e.Message);
}
catch (NullReferenceException e)
{
Console.WriteLine(DateTime.Now + ": Received an '" + e.Message + "' Error from " + art[0]);
}
}
return(list);
}
