private static IEnumerable <KeyValuePair <string, int> > GenerateFrequencyList(IEnumerable <string> episodeLinks,
IDictionary <string, string> caseDictionary)
{
var dictionary = new Dictionary <string, int>();
var i = 0;
foreach (var episodeLink in episodeLinks)
{
var xDoc = XDocument.Load(episodeLink);
if (xDoc.Root == null)
{
continue;
}
var firstElement = xDoc.Root.Elements().First();
if (firstElement.Name.LocalName.Equals("error"))
{
Console.WriteLine($"Episode {episodeLink} doesn't exist");
continue;
}
// Get the HTML of the wiki page
var allHtml = firstElement.Elements().First().Value;
// Get the paragraphs only
var paragraphs = new HtmlParser().Parse(allHtml).GetElementsByTagName("p");
// Concatenate the paragraphs
var text = paragraphs.Aggregate("", (current, paragraph) => current + (paragraph.TextContent += "\n"));
// Get a list of words from the text
var allWords = GetWords(text);
foreach (var word in allWords)
{
// If it's a number or has a length of 1, continue
if (int.TryParse(word, out var _) || word.Length == 1)
{
continue;
}
if (dictionary.ContainsKey(word.ToLower()))
{
dictionary[word.ToLower()]++;
}
else
{
dictionary.Add(word.ToLower(), 1);
caseDictionary.Add(word.ToLower(), word);
}
}
Console.WriteLine("Finished episode " + ++i);
}
return(dictionary.OrderByDescending(w => w.Value));
}