private static IEnumerable <KeyValuePair <string, int> > GenerateFrequencyList(IEnumerable <string> episodeLinks, IDictionary <string, string> caseDictionary) { var dictionary = new Dictionary <string, int>(); var i = 0; foreach (var episodeLink in episodeLinks) { var xDoc = XDocument.Load(episodeLink); if (xDoc.Root == null) { continue; } var firstElement = xDoc.Root.Elements().First(); if (firstElement.Name.LocalName.Equals("error")) { Console.WriteLine($"Episode {episodeLink} doesn't exist"); continue; } // Get the HTML of the wiki page var allHtml = firstElement.Elements().First().Value; // Get the paragraphs only var paragraphs = new HtmlParser().Parse(allHtml).GetElementsByTagName("p"); // Concatenate the paragraphs var text = paragraphs.Aggregate("", (current, paragraph) => current + (paragraph.TextContent += "\n")); // Get a list of words from the text var allWords = GetWords(text); foreach (var word in allWords) { // If it's a number or has a length of 1, continue if (int.TryParse(word, out var _) || word.Length == 1) { continue; } if (dictionary.ContainsKey(word.ToLower())) { dictionary[word.ToLower()]++; } else { dictionary.Add(word.ToLower(), 1); caseDictionary.Add(word.ToLower(), word); } } Console.WriteLine("Finished episode " + ++i); } return(dictionary.OrderByDescending(w => w.Value)); }