public ScrapedResults GetDataFromUrl(string url) { var results = new ScrapedResults(); try { using (WebClient webClient = new WebClient()) { string source = webClient.DownloadString(url); HtmlDocument document = new HtmlDocument(); document.LoadHtml(source); results.Uri = new Uri(url); results.Images = getImages(document, webClient, results.Uri); var wordStats = getWordStats(document); var wordCount = wordStats.Sum(stat => stat.Count); results.PageUrl = url; results.WordStats = wordStats.OrderByDescending(w => w.Count).Take(Constants.GeneralSettings.TopWordResultQty).ToList(); results.Source = source; results.WordCount = wordCount; } } catch (Exception ex) { var message = ex.Message; results.ErrorMessage = Constants.GeneralSettings.ScraperErrorMessage; //TODO:IncorporateLogging } return(results); }
public void Test_DoesTakeValidUrl() { string url = "https://www.fallsdigital.com"; var builder = new PageBuilder(); ScrapedResults results = builder.GetDataFromUrl(url); Assert.IsTrue(results.WordCount > 0); }
public ActionResult GetScrapedResults(ScrapedResults request) { if (ModelState.IsValid) { var model = _pageBuilder.GetDataFromUrl(request.PageUrl); return(PartialView("_Request", model)); } else { return(PartialView("_Request", request)); } }