private static ProductDTO ExtractProductDetails(IEnumerable <HtmlNode> nodes)
{
var product = new ProductDTO {
Source = "Walmart"
};
var title = HtmlNodeHelper.GetFirstByNameAndAttribute(nodes, "a", "data-type", "itemTitles");
if (string.IsNullOrWhiteSpace(title?.InnerText))
{
return(null);
}
product.Title = title.InnerText;
var price = HtmlNodeHelper.GetFirstByNameAndClass(nodes, "span", "price-characteristic");
if (string.IsNullOrWhiteSpace(price?.InnerText))
{
return(null);
}
product.Price = price.InnerText;
product.Image = HtmlNodeHelper.GetFirstValueByNameAndAttribute(nodes, "img", "src");
product.Link = _BaseUrl + HtmlNodeHelper.GetFirstValueByNameAndAttribute(nodes, "a", "href");
return(product);
}
private static IEnumerable <ProductDTO> MapProducts(HtmlDocument doc)
{
var products = new List <ProductDTO>();
for (int i = 0; i < _IndexesCount; i++)
{
var li = HtmlNodeHelper.GetFirstByNameAndAttribute(
doc.DocumentNode.Descendants(), "li", "data-tl-id", $"ProductTileGridView-{i}");
if (li == null)
{
continue;
}
var product = ExtractProductDetails(li.Descendants());
if (product != null)
{
products.Add(product);
}
}
return(products);
}
private static ProductDTO ExtractProductFromHtml(HtmlDocument doc, int index)
{
var product = new ProductDTO {
Source = "Amazon"
};
var node = HtmlNodeHelper.GetFirstByNameAndAttribute(doc.DocumentNode.Descendants(), "div", "data-index", index.ToString());
if (node == null || !node.InnerHtml.Contains("price") && !node.OuterHtml.Contains("price"))
{
return(null);
}
var descendants = node.Descendants();
var span = HtmlNodeHelper.GetFirstByNameAndClass(descendants, "span", "a-size-medium a-color-base a-text-normal");
if (string.IsNullOrWhiteSpace(span?.InnerText) || span.InnerHtml.Length <= 1)
{
span = HtmlNodeHelper.GetFirstByNameAndClass(descendants, "span", "a-size-base-plus a-color-base a-text-normal");
}
if (string.IsNullOrWhiteSpace(span?.InnerText))
{
return(null);
}
product.Title = span.InnerText;
product.Price = HtmlNodeHelper.GetFirstByNameAndClass(descendants, "span", "a-price-whole")?.InnerText;
if (product.Price == null)
{
return(null);
}
product.Image = HtmlNodeHelper.GetFirstValueByNameAndAttribute(descendants, "img", "src");
product.Link = HttpUtility.HtmlDecode(_BaseUrl + HtmlNodeHelper.GetFirstValueByNameAndAttribute(descendants, "a", "href"));
return(product);
}