private List <CarInfo> ScrapMultiple(PagingInfo pagingInfo, DealerInfo dealer, ISelector selector) { var result = new List <CarInfo>(); foreach (var pagedUrl in pagingInfo.PagedUrls) { #if DEBUG var s = DateTime.Now; #endif //HtmlAgilityPack.HtmlDocument doc = LoadWebSiteAsync(dealer.Url + pagedUrl); //var node = LoadWebSiteScrapySharp(dealer.Url + pagedUrl); HtmlDocument doc = LoadWebsite(pagedUrl); HtmlNodeCollection rows = null; foreach (var rowSelector in selector.GetRowSelectors()) { rows = doc?.DocumentNode.SelectNodes(rowSelector); if (rows != null) { break; } } if (rows != null) { rows.ToList().ForEach(row => { var carInfo = selector.ParseHtmlIntoCarInfo(row, dealer); carInfo.WebSite = dealer.Url; carInfo.DealerName = dealer.Name; var map = selector.GetCleanupMap(); if (map != null) { map.ForEach(e => { carInfo.GetType().GetProperty(e.Item1).SetValue(carInfo, carInfo.GetType().GetProperty(e.Item1).GetValue(carInfo)?.ToString().Replace(e.Item2, "").Trim()); }); } var regexMap = selector.GetRegexMap(); if (regexMap != null) { regexMap.ForEach(a => { if (carInfo.GetType().GetProperty(a.Item1).GetValue(carInfo) != null) { carInfo.GetType().GetProperty(a.Item1).SetValue(carInfo, a.Item2.Replace(carInfo.GetType().GetProperty(a.Item1).GetValue(carInfo)?.ToString(), " ").Trim()); } }); } result.Add(carInfo); }); } #if DEBUG NLogger.Instance.Info(string.Format("Finished scrape for URL {0}, {1} cars. ({2} ms)", pagedUrl, result.GroupBy(a => a.VIN).Select(a => a.First()).Count(), (DateTime.Now - s).TotalMilliseconds)); #endif } return(result); }