private List <WebScrapeOutput> GetContent(HtmlDocument htmlDoc, WebScrapeInput input, int pageNumber) { List <WebScrapeOutput> result = new List <WebScrapeOutput>(); try { var bodyContainer = htmlDoc.DocumentNode.SelectSingleNode("//html"); var parentContainers = htmlDoc.DocumentNode.SelectNodes(input.ParentContainer.XPath); if (parentContainers == null) { Logger.Write($"ERROR: Parent Container not found for WebsiteName: {input.Website.Name}; URL: {input.Website.URL} in ProcessRequest -- RestClientScraper -> DataGrabber."); } else { // loop through all parents foreach (HtmlNode parentContainer in parentContainers) { var productContainers = parentContainer.SelectNodes(input.Container.XPath); if (productContainers == null) { Logger.Write($"ERROR: Product Container not found for WebsiteName: {input.Website.Name} in ProcessRequest -- RestClientScraper -> DataGrabber."); } else { // loop through all products foreach (HtmlNode container in productContainers) { // clear input product details input.ClearProductDetails(); // get all product details -- which are not input attributes foreach (ElementMapping mapping in input.ProductDetails.Where(v => !v.IsInputAttribute)) { mapping.Value = GetHTMLContent(container, bodyContainer, mapping, input.Website.URL); } // set result if value is not empty if (!string.IsNullOrEmpty(input.ProductDetails.Where(v => !v.IsInputAttribute).FirstOrDefault().Value)) { result.Add(input.Format(pageNumber, input.Website.UniqueID, input.Website.MappingID)); // format to output } } } } } } catch (Exception ex) { Logger.Write("Exception in GetContent -- RestClientScraper -> DataGrabber. Message: " + ex.Message); } finally { } return(result); }