public ScrapedPageViewModel BuildScrapedPageViewModel(Uri targetUri) { var viewModel = new ScrapedPageViewModel { TargetUri = targetUri }; var scraper = new WebLoader(); HtmlDocument htmlDoc; try { htmlDoc = scraper.LoadHtmlDocument(targetUri.ToString()); } catch (Exception ex) { //very rare scenario where we actually just want to swallow this - no logging or further action necessary in this scenario //target site won't be scraped. Send user to scrape-specific error page suggesting to try another site. return(null); } IEnumerable <string> filteredNodes = scraper.GetDocumentText(htmlDoc).Select(x => x.InnerText); string normalizedPageString = string.Join(" ", filteredNodes) .Replace("\t", "") // ensure we don't include formatting as an word .NormalizeLineEndings(""); List <string> wordList = StringUtility.GetWords(normalizedPageString.ToLowerInvariant()).ToList(); var wordFrequencyStats = StringUtility.AggregateWordFrequencyStats(wordList.Where(w => w.Length > 2)).ToList(); viewModel.TotalWordCount = wordList.Count; viewModel.UniqueWordCount = StringUtility.GetUniqueWords(wordList).Count(); viewModel.WordFrequencyStats = AssembleTagCloudListForViewModel(wordFrequencyStats); viewModel.ImageList = AssembleImageListForViewModel(targetUri, scraper.PluckImageNodes(htmlDoc)); return(viewModel); }