public ScrapedResults GetDataFromUrl(string url)
{
var results = new ScrapedResults();
try
{
using (WebClient webClient = new WebClient())
{
string source = webClient.DownloadString(url);
HtmlDocument document = new HtmlDocument();
document.LoadHtml(source);
results.Uri = new Uri(url);
results.Images = getImages(document, webClient, results.Uri);
var wordStats = getWordStats(document);
var wordCount = wordStats.Sum(stat => stat.Count);
results.PageUrl = url;
results.WordStats = wordStats.OrderByDescending(w => w.Count).Take(Constants.GeneralSettings.TopWordResultQty).ToList();
results.Source = source;
results.WordCount = wordCount;
}
}
catch (Exception ex)
{
var message = ex.Message;
results.ErrorMessage = Constants.GeneralSettings.ScraperErrorMessage;
//TODO:IncorporateLogging
}
return(results);
}
public void Test_DoesTakeValidUrl()
{
string url = "https://www.fallsdigital.com";
var builder = new PageBuilder();
ScrapedResults results = builder.GetDataFromUrl(url);
Assert.IsTrue(results.WordCount > 0);
}
public ActionResult GetScrapedResults(ScrapedResults request)
{
if (ModelState.IsValid)
{
var model = _pageBuilder.GetDataFromUrl(request.PageUrl);
return(PartialView("_Request", model));
}
else
{
return(PartialView("_Request", request));
}
}