public IEnumerable <IProduct> Search(string keyword)
{
var url = "https://www.makro.co.za";
var doc = _webScrapper.Scrap($"{url}/search?text={keyword}");
var elements = doc.DocumentNode.CssSelect(".mak-product-tiles-container__product-tile");
var resultProducts = new List <Product>();
if (elements.Any())
{
var index = 0;
foreach (var element in elements)
{
var productLinkImage = element.CssSelect(".product-tile-inner > .product-tile-inner__img").FirstOrDefault();
if (productLinkImage == null)
{
continue;
}
var productLinkElementAttributes = productLinkImage.Attributes;
var sourceUrl = $"{url}{productLinkElementAttributes.AttributesWithName("href").First().Value}";
var name = productLinkElementAttributes.AttributesWithName("title").First().Value;
var imageElementAttributes = element.CssSelect(".product-tile-inner > .product-tile-inner__img > img").First().Attributes;
var imageUrl = imageElementAttributes.AttributesWithName("data-src").First().Value;
var cultures = new CultureInfo("en-US");
var priceValue = element.CssSelect(".product-tile-inner > .price")
.First()
.InnerHtml
.Trim()
.Trim('\n')
.Trim('R')
.TrimStart()
.Replace("<span class=\"mak-product__cents\">", ".").Replace("</span>", "");
var price = Convert.ToDecimal(priceValue, cultures);
resultProducts.Add(new Product
{
Name = HttpUtility.HtmlDecode(name),
Description = HttpUtility.HtmlDecode(name),
Price = price,
Units = 1,
Brand = null,
Source = StoreSiteName.Makro.ToString(),
SourceUrl = sourceUrl,
ImageUrl = imageUrl,
Order = index
});
++index;
}
}
return(resultProducts);
}
public IEnumerable <IProduct> Search(string keyword)
{
var url = "https://matrixwarehouse.co.za";
var doc = _webScrapper.Scrap($"{url}/?s={keyword}&product_cat=0&post_type=product");
var elements = doc.DocumentNode.CssSelect(".product.type-product");
var resultProducts = new List <Product>();
if (elements.Any())
{
var index = 0;
foreach (var element in elements)
{
var productLinkElement = element.CssSelect(".product-inner > .woocommerce-LoopProduct-link.woocommerce-loop-product__link").FirstOrDefault();
if (productLinkElement != null)
{
var productLinkElementAttributes = productLinkElement.Attributes;
var sourceUrl = productLinkElementAttributes.AttributesWithName("href").First().Value;
var name = element.CssSelect(".product-inner > .woocommerce-LoopProduct-link.woocommerce-loop-product__link > .woocommerce-loop-product__title").First().InnerText;
var imageElementAttributes = element.CssSelect(".product-inner > .woocommerce-LoopProduct-link.woocommerce-loop-product__link > .attachment-woocommerce_thumbnail").First().Attributes;
var imageUrl = imageElementAttributes.AttributesWithName("src").First().Value;
var cultures = new CultureInfo("en-US");
var regularPriceBox = element.CssSelect(".product-inner > .woocommerce-LoopProduct-link.woocommerce-loop-product__link > .price > .woocommerce-Price-amount.amount").FirstOrDefault();
var specialPriceBox = element.CssSelect(".product-inner > .woocommerce-LoopProduct-link.woocommerce-loop-product__link > .price > ins > .woocommerce-Price-amount.amount").FirstOrDefault();
var priceValue = (regularPriceBox != null || specialPriceBox != null) ? (regularPriceBox ?? specialPriceBox).InnerText.Replace("R", "").Trim('\n').Trim('R').Trim() : "0,0";
var price = Convert.ToDecimal(priceValue, cultures);
resultProducts.Add(new Product
{
Name = HttpUtility.HtmlDecode(name),
Description = name,
Price = price,
Units = 1,
Brand = null,
Source = StoreSiteName.MatrixWarehouse.ToString(),
SourceUrl = sourceUrl,
ImageUrl = imageUrl,
Order = index
});
++index;
}
}
}
return(resultProducts);
}
public IEnumerable <IProduct> Search(string keyword)
{
var url = "https://clicks.co.za";
var doc = _webScrapper.Scrap($"{url}/search/?cat=&text={keyword}");
var elements = doc.DocumentNode.CssSelect(".plpsep");
var resultProducts = new List <Product>();
if (elements.Any())
{
var index = 0;
foreach (var element in elements)
{
var productLinkElement = element.CssSelect(".productBlock > a").FirstOrDefault();
if (productLinkElement != null)
{
var productLinkElementAttributes = productLinkElement.Attributes;
var sourceUrl = $"{url}{productLinkElementAttributes.AttributesWithName("href").First().Value}";
var name = productLinkElementAttributes.AttributesWithName("title").First().Value;
var imageElementAttributes = element.CssSelect(".productBlock > a > img").First().Attributes;
var imageUrl = $"{url}{imageElementAttributes.AttributesWithName("src").First().Value}";
var cultures = new CultureInfo("en-US");
var priceValue = element.CssSelect(".productBlock > .detailContent > .price-wrap > .price")
.First().InnerText.Replace("R", "").Trim();
var price = Convert.ToDecimal(priceValue, cultures);
resultProducts.Add(new Product
{
Name = HttpUtility.HtmlDecode(name),
Description = name,
Price = price,
Units = 1,
Brand = null,
Source = StoreSiteName.Clicks.ToString(),
SourceUrl = sourceUrl,
ImageUrl = imageUrl,
Order = index
});
++index;
}
}
}
return(resultProducts);
}
public IEnumerable <IProduct> Search(string keyword)
{
var url = "https://www.pnp.co.za";
var doc = _webScrapper.Scrap($"{url}/pnpstorefront/pnp/en/search/?text={keyword}");
var elements = doc.DocumentNode.CssSelect(".productCarouselItemContainer");
var resultProducts = new List <Product>();
if (elements.Any())
{
var index = 0;
foreach (var element in elements)
{
var productLinkElementAttributes = element.CssSelect(".productCarouselItem > .product-card-grid > a").First().Attributes;
var productLink = productLinkElementAttributes.AttributesWithName("href").FirstOrDefault();
if (productLink != null)
{
var sourceUrl = $"{url}{productLink.Value}";
var imageElementAttributes = element.CssSelect(".productCarouselItem > .product-card-grid > a > .thumb > img").First().Attributes;
var imageUrl = imageElementAttributes.AttributesWithName("src").First().Value;
var name = imageElementAttributes.AttributesWithName("title").First().Value;
var cultures = new CultureInfo("en-US");
var priceValue = element.CssSelect(".productCarouselItem > .product-card-grid > a > .product-price > .item-price > .currentPrice")
.First().InnerHtml.Replace("R", "").Replace("<span>", ".").Replace("</span>", "").Trim();
var price = Convert.ToDecimal(priceValue, cultures);
resultProducts.Add(new Product
{
Name = HttpUtility.HtmlDecode(name),
Description = name,
Price = price,
Units = 1,
Brand = null,
Source = StoreSiteName.PicknPay.ToString(),
SourceUrl = sourceUrl,
ImageUrl = imageUrl,
Order = index
});
++index;
}
}
}
return(resultProducts);
}
public IEnumerable <IProduct> Search(string keyword)
{
var url = "https://www.dischem.co.za";
var doc = _webScrapper.Scrap($"{url}/catalogsearch/result/?q={keyword}");
var elements = doc.DocumentNode.CssSelect(".item.product.product-item");
var resultProducts = new List <Product>();
if (elements.Any())
{
var index = 0;
foreach (var element in elements)
{
var productLinkElementAttributes = element.CssSelect(".product-item-info > .image > .product-item-photo").First().Attributes;
var sourceUrl = productLinkElementAttributes.AttributesWithName("href").First().Value;
var name = element.CssSelect(".product-item-info > .product-item-details > .product-item-name > .product-item-link").First().InnerText;
var imageElementAttributes = element.CssSelect(".product-item-info > .image > .product-item-photo > .product-image-container > .product-image-wrapper > .product-image-photo").First().Attributes;
var imageUrl = imageElementAttributes.AttributesWithName("src").First().Value;
var cultures = new CultureInfo("en-US");
var priceElement = element
.CssSelect(".product-item-info > .product-item-details > .price-box-wrapper-listing > .price-box > .price-box-inner-wrapper > .price-container > .price-wrapper");
var specialElement = element
.CssSelect(".product-item-info > .product-item-details > .price-box-wrapper-listing > .price-box > .price-box-inner-wrapper > .special-price > .price-container > .price-wrapper");
var priceVal = (priceElement.FirstOrDefault() ?? specialElement.FirstOrDefault());
var priceValue = (priceVal != null) ? priceVal.Attributes.AttributesWithName("data-price-amount").First().Value : "0";
var price = Convert.ToDecimal(priceValue, cultures);
resultProducts.Add(new Product
{
Name = $"{HttpUtility.HtmlDecode(name)}{(priceVal != null ? "" : "Out of stock")}",
Description = name,
Price = price,
Units = 1,
Brand = null,
Source = StoreSiteName.Dischem.ToString(),
SourceUrl = sourceUrl,
ImageUrl = imageUrl,
Order = index
});
++index;
}
}
return(resultProducts);
}
public IEnumerable <IProduct> Search(string keyword)
{
var url = "https://www.checkers.co.za";
var doc = _webScrapper.Scrap($"{url}/search?q={keyword}");
var elements = doc.DocumentNode.CssSelect(".product-frame");
var resultProducts = new List <Product>();
if (elements.Any())
{
var index = 0;
foreach (var element in elements)
{
var productLinkElementAttributes = element.CssSelect(".item-product > .item-product__content > .item-product__image > .product-listening-click").First().Attributes;
var sourceUrl = $"{url}{productLinkElementAttributes.AttributesWithName("href").First().Value}";
var name = productLinkElementAttributes.AttributesWithName("title").First().Value;
var imageElementAttributes = element.CssSelect(".item-product > .item-product__content > .item-product__image > .product-listening-click > img").First().Attributes;
var srcAttribute = imageElementAttributes.AttributesWithName("data-original-src").FirstOrDefault();
var imageUrl = srcAttribute != null ? $"{url}{srcAttribute.Value}" : null;
var cultures = new CultureInfo("en-US");
var priceValue = element.CssSelect(".item-product > .item-product__content > .item-product__caption > .item-product__details > .js-item-product-price > .special-price > .special-price__price > .now")
.First().InnerText.Replace("R", "").Replace("Per", "").Replace("Kg", "").Trim();
var price = string.IsNullOrEmpty(priceValue) ? 0 : Convert.ToDecimal(priceValue, cultures);
resultProducts.Add(new Product
{
Name = HttpUtility.HtmlDecode(name),
Description = name,
Price = price,
Units = 1,
Brand = null,
Source = StoreSiteName.Checkers.ToString(),
SourceUrl = sourceUrl,
ImageUrl = imageUrl,
Order = index
});
++index;
}
}
return(resultProducts);
}
public IEnumerable <IProduct> Search(string keyword)
{
var url = "https://www.hificorp.co.za";
var doc = _webScrapper.Scrap($"{url}/catalog/category/searchResult?q={keyword}");
var elements = doc.DocumentNode.CssSelect(".products-grid > li.item");
var resultProducts = new List <Product>();
if (elements.Any())
{
var index = 0;
foreach (var element in elements)
{
var productLinkElementAttributes = element.CssSelect(".product-wrapper > .img-wrapper > .product-image").First().Attributes;
var sourceUrl = productLinkElementAttributes.AttributesWithName("href").First().Value;
var name = productLinkElementAttributes.AttributesWithName("title").First().Value;
var imageElementAttributes = element.CssSelect(".product-wrapper > .img-wrapper > .product-image > .grid-image-wrapper > img").First().Attributes;
var imageUrl = imageElementAttributes.AttributesWithName("data-src").First().Value;
var cultures = new CultureInfo("en-US");
var regularPriceBox = element.CssSelect(".product-wrapper > .price-grid-box > .regular-price-box > .regular-price").FirstOrDefault();
var specialPriceBox = element.CssSelect(".product-wrapper > .price-grid-box > .special-price-box > .regular-price");
var priceValue = (regularPriceBox ?? specialPriceBox.First()).InnerHtml.Trim('\n').Trim('R').Trim();
var price = Convert.ToDecimal(priceValue, cultures);
resultProducts.Add(new Product
{
Name = HttpUtility.HtmlDecode(name),
Description = HttpUtility.HtmlDecode(name),
Price = price,
Units = 1,
Brand = null,
Source = StoreSiteName.HifiCorp.ToString(),
SourceUrl = sourceUrl,
ImageUrl = imageUrl,
Order = index
});
++index;
}
}
return(resultProducts);
}
public IEnumerable <IProduct> Search(string keyword)
{
var url = "https://www.incredible.co.za";
var doc = _webScrapper.Scrap($"{url}/catalogsearch/result/?q={keyword}");
var elements = doc.DocumentNode.CssSelect("li.item.product.product-item");
var resultProducts = new List <Product>();
if (elements.Any())
{
var index = 0;
foreach (var element in elements)
{
var productLinkElementAttributes = element.CssSelect(".product-item-info .product").First().Attributes;
var sourceUrl = productLinkElementAttributes.AttributesWithName("href").First().Value;
var imageElementAttributes = element.CssSelect(".product-image-photo").First().Attributes;
var imageUrl = imageElementAttributes.AttributesWithName("src").First().Value;
var name = imageElementAttributes.AttributesWithName("alt").First().Value;
var cultures = new CultureInfo("en-US");
var priceValue = element.CssSelect(".product-item-details .price").First().InnerText.Replace("\n", "").Replace("R", "");
var price = Convert.ToDecimal(priceValue, cultures);
resultProducts.Add(new Product
{
Name = HttpUtility.HtmlDecode(name),
Description = name,
Price = price,
Units = 1,
Brand = null,
Source = StoreSiteName.IncredibleConnection.ToString(),
SourceUrl = sourceUrl,
ImageUrl = imageUrl,
Order = index
});
++index;
}
}
return(resultProducts);
}
public IEnumerable <IProduct> Search(string keyword)
{
var url = "https://www.game.co.za";
var doc = _webScrapper.Scrap($"{url}/game-za/en/search/?text={keyword}");
var elements = doc.DocumentNode.CssSelect(".product-item.productListerGridDiv");
var resultProducts = new List <Product>();
if (elements.Any())
{
var index = 0;
foreach (var element in elements)
{
var productLinkElementAttributes = element.CssSelect(".thumb.gtmProductLink").First().Attributes;
var sourceUrl = $"{url}{productLinkElementAttributes.AttributesWithName("href").First().Value}";
var name = productLinkElementAttributes.AttributesWithName("title").First().Value;
var imageElementAttributes = element.CssSelect(".productPrimaryImage > img").First().Attributes;
var imageUrl = $"{url}{imageElementAttributes.AttributesWithName("src").First().Value}";
var cultures = new CultureInfo("en-US");
var price = Convert.ToDecimal(element.CssSelect(".details > .price > .finalPrice").First().InnerText.Trim('R'), cultures);
resultProducts.Add(new Product
{
Name = HttpUtility.HtmlDecode(name),
Description = name,
Price = price,
Units = 1,
Brand = null,
Source = StoreSiteName.Game.ToString(),
SourceUrl = sourceUrl,
ImageUrl = imageUrl,
Order = index
});
++index;
}
}
return(resultProducts);
}
public void GivenUrlWhenSearchingScrapThenReturnSiteContent()
{
var result = _scrapper.Scrap("https://www.takealot.com/all?qsearch=laptop");
Assert.Contains("Takealot", result.DocumentNode.SelectSingleNode("//head/title").InnerHtml);
}
