private static ProductDTO ExtractProductDetails(IEnumerable <HtmlNode> nodes)
{
var product = new ProductDTO {
Source = "BestBuy"
};
var title = HtmlNodeHelper.GetFirstByNameAndClass(nodes, "h4", "sku-header");;
if (string.IsNullOrWhiteSpace(title?.InnerText))
{
return(null);
}
product.Title = title.InnerText;
var price = HtmlNodeHelper.GetFirstByNameAndClass(nodes, "div", "priceView-hero-price priceView-customer-price");
if (string.IsNullOrWhiteSpace(price?.InnerText))
{
return(null);
}
product.Price = price.InnerText;
product.Image = HtmlNodeHelper.GetFirstValueByNameAndAttribute(nodes, "img", "src");
product.Link = _BaseUrl + HtmlNodeHelper.GetFirstValueByNameAndAttribute(nodes, "a", "href");
return(product);
}
private static ProductDTO ExtractProductDetails(IEnumerable <HtmlNode> nodes)
{
var product = new ProductDTO {
Source = "Costco"
};
var title = HtmlNodeHelper.GetFirstByNameAndClass(nodes, "span", "description");;
if (string.IsNullOrWhiteSpace(title?.InnerText))
{
return(null);
}
product.Title = title.InnerText;
var price = HtmlNodeHelper.GetFirstByNameAndClass(nodes, "div", "price");
if (string.IsNullOrWhiteSpace(price?.InnerText))
{
return(null);
}
product.Price = price.InnerText;
product.Image = HtmlNodeHelper.GetFirstValueByNameAndAttribute(nodes, "img", "src");
product.Link = HtmlNodeHelper.GetFirstValueByNameAndAttribute(nodes, "a", "href");
return(product);
}
private static ProductDTO ExtractProductDetails(IEnumerable <HtmlNode> nodes)
{
var product = new ProductDTO {
Source = "Walmart"
};
var title = HtmlNodeHelper.GetFirstByNameAndAttribute(nodes, "a", "data-type", "itemTitles");
if (string.IsNullOrWhiteSpace(title?.InnerText))
{
return(null);
}
product.Title = title.InnerText;
var price = HtmlNodeHelper.GetFirstByNameAndClass(nodes, "span", "price-characteristic");
if (string.IsNullOrWhiteSpace(price?.InnerText))
{
return(null);
}
product.Price = price.InnerText;
product.Image = HtmlNodeHelper.GetFirstValueByNameAndAttribute(nodes, "img", "src");
product.Link = _BaseUrl + HtmlNodeHelper.GetFirstValueByNameAndAttribute(nodes, "a", "href");
return(product);
}
private static ProductDTO ExtractProductDetails(IEnumerable <HtmlNode> nodes)
{
var product = new ProductDTO {
Source = "EBay"
};
var title = nodes.FirstOrDefault(n => n.Name == "h3");
if (string.IsNullOrWhiteSpace(title?.InnerText))
{
return(null);
}
product.Title = title.InnerText;
var price = HtmlNodeHelper.GetFirstByNameAndClass(nodes, "span", "s-item__price");
if (string.IsNullOrWhiteSpace(price?.InnerText))
{
return(null);
}
product.Price = price.InnerText;
product.Image = HtmlNodeHelper.GetFirstValueByNameAndAttribute(nodes, "img", "src");
product.Link = HtmlNodeHelper.GetFirstValueByNameAndAttribute(nodes, "a", "href");
return(product);
}
private static IEnumerable <ProductDTO> ExtractProductFromHtml(HtmlDocument doc)
{
var node = HtmlNodeHelper.GetFirstByNameAndClass(doc.DocumentNode.Descendants(), "ul", "srp-results srp-list clearfix");
var descendants = node?.Descendants();
if (descendants?.Any() != true)
{
return(Enumerable.Empty <ProductDTO>());
}
var lis = HtmlNodeHelper.GetAllByNameAndAttribute(descendants, "li", "class", "s-item");
var products = new List <ProductDTO>();
for (int i = 0; i < _IndexesCount && i < lis.Count(); i++)
{
var product = ExtractProductDetails(lis.ElementAt(i).Descendants());
if (product != null)
{
products.Add(product);
}
}
return(products);
}
public PostRepository(IConfiguration configuration, APIContext context)
{
_config = configuration;
_db = context;
var url = _config["PostsUrl"];
var web = new HtmlWeb();
Doc = web.Load(url);
_htmlNodeHelper = new HtmlNodeHelper(_config);
}
private static IEnumerable <ProductDTO> ExtractProductFromHtml(HtmlDocument doc)
{
var node = HtmlNodeHelper.GetFirstByNameAndClass(doc.DocumentNode.Descendants(), "div", "product-list grid");
var descendants = node.Descendants();
var divs = HtmlNodeHelper.GetAllByNameAndAttribute(descendants, "div", "class", "col-xs-6 col-lg-4 col-xl-3 product");
var products = new List <ProductDTO>();
for (int i = 0; i < _IndexesCount && i < divs.Count(); i++)
{
var product = ExtractProductDetails(divs.ElementAt(i).Descendants());
if (product != null)
{
products.Add(product);
}
}
return(products);
}
private static IEnumerable <ProductDTO> MapProducts(HtmlDocument doc)
{
var products = new List <ProductDTO>();
for (int i = 0; i < _IndexesCount; i++)
{
var li = HtmlNodeHelper.GetFirstByNameAndAttribute(
doc.DocumentNode.Descendants(), "li", "data-tl-id", $"ProductTileGridView-{i}");
if (li == null)
{
continue;
}
var product = ExtractProductDetails(li.Descendants());
if (product != null)
{
products.Add(product);
}
}
return(products);
}
private static ProductDTO ExtractProductFromHtml(HtmlDocument doc, int index)
{
var product = new ProductDTO {
Source = "Amazon"
};
var node = HtmlNodeHelper.GetFirstByNameAndAttribute(doc.DocumentNode.Descendants(), "div", "data-index", index.ToString());
if (node == null || !node.InnerHtml.Contains("price") && !node.OuterHtml.Contains("price"))
{
return(null);
}
var descendants = node.Descendants();
var span = HtmlNodeHelper.GetFirstByNameAndClass(descendants, "span", "a-size-medium a-color-base a-text-normal");
if (string.IsNullOrWhiteSpace(span?.InnerText) || span.InnerHtml.Length <= 1)
{
span = HtmlNodeHelper.GetFirstByNameAndClass(descendants, "span", "a-size-base-plus a-color-base a-text-normal");
}
if (string.IsNullOrWhiteSpace(span?.InnerText))
{
return(null);
}
product.Title = span.InnerText;
product.Price = HtmlNodeHelper.GetFirstByNameAndClass(descendants, "span", "a-price-whole")?.InnerText;
if (product.Price == null)
{
return(null);
}
product.Image = HtmlNodeHelper.GetFirstValueByNameAndAttribute(descendants, "img", "src");
product.Link = HttpUtility.HtmlDecode(_BaseUrl + HtmlNodeHelper.GetFirstValueByNameAndAttribute(descendants, "a", "href"));
return(product);
}
