public TrailResponse Get()
{
var document = _pageRetriever.GetPage("https://gorctrails.com/trails");
var trails = document.DocumentNode.SelectNodes("//li")
.Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("views-row"))
.Select(li => new Trail(_statusParser.ExtractName(li), _statusParser.ExtractArea(li), _statusParser.ExtractDirectUrl(li), _statusParser.EvaluateStatus(li)));
return(new TrailResponse {
Trails = trails.ToList()
});
}
public void Process(string baseUrl)
{
var doc = pageRetriever.GetPage(baseUrl);
Logger.Log("crawling data");
var body = this.GetBody(doc);
var footer = this.GetFooter(body);
var containerFooter = footer.ChildNodes.Single(node => node.Name == "div");
var imgs = this.GetImagesUrls(containerFooter);
foreach (var url in imgs)
{
var imageBytes = pageRetriever.GetImage($"{baseUrl}{url}");
fileManager.SaveImage(imageBytes, $"../../images/{url.Split('/').Last()}");
}
Logger.Log("done!");
}