public IEnumerable <IProduct> Search(string keyword) { var url = "https://www.makro.co.za"; var doc = _webScrapper.Scrap($"{url}/search?text={keyword}"); var elements = doc.DocumentNode.CssSelect(".mak-product-tiles-container__product-tile"); var resultProducts = new List <Product>(); if (elements.Any()) { var index = 0; foreach (var element in elements) { var productLinkImage = element.CssSelect(".product-tile-inner > .product-tile-inner__img").FirstOrDefault(); if (productLinkImage == null) { continue; } var productLinkElementAttributes = productLinkImage.Attributes; var sourceUrl = $"{url}{productLinkElementAttributes.AttributesWithName("href").First().Value}"; var name = productLinkElementAttributes.AttributesWithName("title").First().Value; var imageElementAttributes = element.CssSelect(".product-tile-inner > .product-tile-inner__img > img").First().Attributes; var imageUrl = imageElementAttributes.AttributesWithName("data-src").First().Value; var cultures = new CultureInfo("en-US"); var priceValue = element.CssSelect(".product-tile-inner > .price") .First() .InnerHtml .Trim() .Trim('\n') .Trim('R') .TrimStart() .Replace("<span class=\"mak-product__cents\">", ".").Replace("</span>", ""); var price = Convert.ToDecimal(priceValue, cultures); resultProducts.Add(new Product { Name = HttpUtility.HtmlDecode(name), Description = HttpUtility.HtmlDecode(name), Price = price, Units = 1, Brand = null, Source = StoreSiteName.Makro.ToString(), SourceUrl = sourceUrl, ImageUrl = imageUrl, Order = index }); ++index; } } return(resultProducts); }
public IEnumerable <IProduct> Search(string keyword) { var url = "https://matrixwarehouse.co.za"; var doc = _webScrapper.Scrap($"{url}/?s={keyword}&product_cat=0&post_type=product"); var elements = doc.DocumentNode.CssSelect(".product.type-product"); var resultProducts = new List <Product>(); if (elements.Any()) { var index = 0; foreach (var element in elements) { var productLinkElement = element.CssSelect(".product-inner > .woocommerce-LoopProduct-link.woocommerce-loop-product__link").FirstOrDefault(); if (productLinkElement != null) { var productLinkElementAttributes = productLinkElement.Attributes; var sourceUrl = productLinkElementAttributes.AttributesWithName("href").First().Value; var name = element.CssSelect(".product-inner > .woocommerce-LoopProduct-link.woocommerce-loop-product__link > .woocommerce-loop-product__title").First().InnerText; var imageElementAttributes = element.CssSelect(".product-inner > .woocommerce-LoopProduct-link.woocommerce-loop-product__link > .attachment-woocommerce_thumbnail").First().Attributes; var imageUrl = imageElementAttributes.AttributesWithName("src").First().Value; var cultures = new CultureInfo("en-US"); var regularPriceBox = element.CssSelect(".product-inner > .woocommerce-LoopProduct-link.woocommerce-loop-product__link > .price > .woocommerce-Price-amount.amount").FirstOrDefault(); var specialPriceBox = element.CssSelect(".product-inner > .woocommerce-LoopProduct-link.woocommerce-loop-product__link > .price > ins > .woocommerce-Price-amount.amount").FirstOrDefault(); var priceValue = (regularPriceBox != null || specialPriceBox != null) ? (regularPriceBox ?? specialPriceBox).InnerText.Replace("R", "").Trim('\n').Trim('R').Trim() : "0,0"; var price = Convert.ToDecimal(priceValue, cultures); resultProducts.Add(new Product { Name = HttpUtility.HtmlDecode(name), Description = name, Price = price, Units = 1, Brand = null, Source = StoreSiteName.MatrixWarehouse.ToString(), SourceUrl = sourceUrl, ImageUrl = imageUrl, Order = index }); ++index; } } } return(resultProducts); }
public IEnumerable <IProduct> Search(string keyword) { var url = "https://clicks.co.za"; var doc = _webScrapper.Scrap($"{url}/search/?cat=&text={keyword}"); var elements = doc.DocumentNode.CssSelect(".plpsep"); var resultProducts = new List <Product>(); if (elements.Any()) { var index = 0; foreach (var element in elements) { var productLinkElement = element.CssSelect(".productBlock > a").FirstOrDefault(); if (productLinkElement != null) { var productLinkElementAttributes = productLinkElement.Attributes; var sourceUrl = $"{url}{productLinkElementAttributes.AttributesWithName("href").First().Value}"; var name = productLinkElementAttributes.AttributesWithName("title").First().Value; var imageElementAttributes = element.CssSelect(".productBlock > a > img").First().Attributes; var imageUrl = $"{url}{imageElementAttributes.AttributesWithName("src").First().Value}"; var cultures = new CultureInfo("en-US"); var priceValue = element.CssSelect(".productBlock > .detailContent > .price-wrap > .price") .First().InnerText.Replace("R", "").Trim(); var price = Convert.ToDecimal(priceValue, cultures); resultProducts.Add(new Product { Name = HttpUtility.HtmlDecode(name), Description = name, Price = price, Units = 1, Brand = null, Source = StoreSiteName.Clicks.ToString(), SourceUrl = sourceUrl, ImageUrl = imageUrl, Order = index }); ++index; } } } return(resultProducts); }
public IEnumerable <IProduct> Search(string keyword) { var url = "https://www.pnp.co.za"; var doc = _webScrapper.Scrap($"{url}/pnpstorefront/pnp/en/search/?text={keyword}"); var elements = doc.DocumentNode.CssSelect(".productCarouselItemContainer"); var resultProducts = new List <Product>(); if (elements.Any()) { var index = 0; foreach (var element in elements) { var productLinkElementAttributes = element.CssSelect(".productCarouselItem > .product-card-grid > a").First().Attributes; var productLink = productLinkElementAttributes.AttributesWithName("href").FirstOrDefault(); if (productLink != null) { var sourceUrl = $"{url}{productLink.Value}"; var imageElementAttributes = element.CssSelect(".productCarouselItem > .product-card-grid > a > .thumb > img").First().Attributes; var imageUrl = imageElementAttributes.AttributesWithName("src").First().Value; var name = imageElementAttributes.AttributesWithName("title").First().Value; var cultures = new CultureInfo("en-US"); var priceValue = element.CssSelect(".productCarouselItem > .product-card-grid > a > .product-price > .item-price > .currentPrice") .First().InnerHtml.Replace("R", "").Replace("<span>", ".").Replace("</span>", "").Trim(); var price = Convert.ToDecimal(priceValue, cultures); resultProducts.Add(new Product { Name = HttpUtility.HtmlDecode(name), Description = name, Price = price, Units = 1, Brand = null, Source = StoreSiteName.PicknPay.ToString(), SourceUrl = sourceUrl, ImageUrl = imageUrl, Order = index }); ++index; } } } return(resultProducts); }
public IEnumerable <IProduct> Search(string keyword) { var url = "https://www.dischem.co.za"; var doc = _webScrapper.Scrap($"{url}/catalogsearch/result/?q={keyword}"); var elements = doc.DocumentNode.CssSelect(".item.product.product-item"); var resultProducts = new List <Product>(); if (elements.Any()) { var index = 0; foreach (var element in elements) { var productLinkElementAttributes = element.CssSelect(".product-item-info > .image > .product-item-photo").First().Attributes; var sourceUrl = productLinkElementAttributes.AttributesWithName("href").First().Value; var name = element.CssSelect(".product-item-info > .product-item-details > .product-item-name > .product-item-link").First().InnerText; var imageElementAttributes = element.CssSelect(".product-item-info > .image > .product-item-photo > .product-image-container > .product-image-wrapper > .product-image-photo").First().Attributes; var imageUrl = imageElementAttributes.AttributesWithName("src").First().Value; var cultures = new CultureInfo("en-US"); var priceElement = element .CssSelect(".product-item-info > .product-item-details > .price-box-wrapper-listing > .price-box > .price-box-inner-wrapper > .price-container > .price-wrapper"); var specialElement = element .CssSelect(".product-item-info > .product-item-details > .price-box-wrapper-listing > .price-box > .price-box-inner-wrapper > .special-price > .price-container > .price-wrapper"); var priceVal = (priceElement.FirstOrDefault() ?? specialElement.FirstOrDefault()); var priceValue = (priceVal != null) ? priceVal.Attributes.AttributesWithName("data-price-amount").First().Value : "0"; var price = Convert.ToDecimal(priceValue, cultures); resultProducts.Add(new Product { Name = $"{HttpUtility.HtmlDecode(name)}{(priceVal != null ? "" : "Out of stock")}", Description = name, Price = price, Units = 1, Brand = null, Source = StoreSiteName.Dischem.ToString(), SourceUrl = sourceUrl, ImageUrl = imageUrl, Order = index }); ++index; } } return(resultProducts); }
public IEnumerable <IProduct> Search(string keyword) { var url = "https://www.checkers.co.za"; var doc = _webScrapper.Scrap($"{url}/search?q={keyword}"); var elements = doc.DocumentNode.CssSelect(".product-frame"); var resultProducts = new List <Product>(); if (elements.Any()) { var index = 0; foreach (var element in elements) { var productLinkElementAttributes = element.CssSelect(".item-product > .item-product__content > .item-product__image > .product-listening-click").First().Attributes; var sourceUrl = $"{url}{productLinkElementAttributes.AttributesWithName("href").First().Value}"; var name = productLinkElementAttributes.AttributesWithName("title").First().Value; var imageElementAttributes = element.CssSelect(".item-product > .item-product__content > .item-product__image > .product-listening-click > img").First().Attributes; var srcAttribute = imageElementAttributes.AttributesWithName("data-original-src").FirstOrDefault(); var imageUrl = srcAttribute != null ? $"{url}{srcAttribute.Value}" : null; var cultures = new CultureInfo("en-US"); var priceValue = element.CssSelect(".item-product > .item-product__content > .item-product__caption > .item-product__details > .js-item-product-price > .special-price > .special-price__price > .now") .First().InnerText.Replace("R", "").Replace("Per", "").Replace("Kg", "").Trim(); var price = string.IsNullOrEmpty(priceValue) ? 0 : Convert.ToDecimal(priceValue, cultures); resultProducts.Add(new Product { Name = HttpUtility.HtmlDecode(name), Description = name, Price = price, Units = 1, Brand = null, Source = StoreSiteName.Checkers.ToString(), SourceUrl = sourceUrl, ImageUrl = imageUrl, Order = index }); ++index; } } return(resultProducts); }
public IEnumerable <IProduct> Search(string keyword) { var url = "https://www.hificorp.co.za"; var doc = _webScrapper.Scrap($"{url}/catalog/category/searchResult?q={keyword}"); var elements = doc.DocumentNode.CssSelect(".products-grid > li.item"); var resultProducts = new List <Product>(); if (elements.Any()) { var index = 0; foreach (var element in elements) { var productLinkElementAttributes = element.CssSelect(".product-wrapper > .img-wrapper > .product-image").First().Attributes; var sourceUrl = productLinkElementAttributes.AttributesWithName("href").First().Value; var name = productLinkElementAttributes.AttributesWithName("title").First().Value; var imageElementAttributes = element.CssSelect(".product-wrapper > .img-wrapper > .product-image > .grid-image-wrapper > img").First().Attributes; var imageUrl = imageElementAttributes.AttributesWithName("data-src").First().Value; var cultures = new CultureInfo("en-US"); var regularPriceBox = element.CssSelect(".product-wrapper > .price-grid-box > .regular-price-box > .regular-price").FirstOrDefault(); var specialPriceBox = element.CssSelect(".product-wrapper > .price-grid-box > .special-price-box > .regular-price"); var priceValue = (regularPriceBox ?? specialPriceBox.First()).InnerHtml.Trim('\n').Trim('R').Trim(); var price = Convert.ToDecimal(priceValue, cultures); resultProducts.Add(new Product { Name = HttpUtility.HtmlDecode(name), Description = HttpUtility.HtmlDecode(name), Price = price, Units = 1, Brand = null, Source = StoreSiteName.HifiCorp.ToString(), SourceUrl = sourceUrl, ImageUrl = imageUrl, Order = index }); ++index; } } return(resultProducts); }
public IEnumerable <IProduct> Search(string keyword) { var url = "https://www.incredible.co.za"; var doc = _webScrapper.Scrap($"{url}/catalogsearch/result/?q={keyword}"); var elements = doc.DocumentNode.CssSelect("li.item.product.product-item"); var resultProducts = new List <Product>(); if (elements.Any()) { var index = 0; foreach (var element in elements) { var productLinkElementAttributes = element.CssSelect(".product-item-info .product").First().Attributes; var sourceUrl = productLinkElementAttributes.AttributesWithName("href").First().Value; var imageElementAttributes = element.CssSelect(".product-image-photo").First().Attributes; var imageUrl = imageElementAttributes.AttributesWithName("src").First().Value; var name = imageElementAttributes.AttributesWithName("alt").First().Value; var cultures = new CultureInfo("en-US"); var priceValue = element.CssSelect(".product-item-details .price").First().InnerText.Replace("\n", "").Replace("R", ""); var price = Convert.ToDecimal(priceValue, cultures); resultProducts.Add(new Product { Name = HttpUtility.HtmlDecode(name), Description = name, Price = price, Units = 1, Brand = null, Source = StoreSiteName.IncredibleConnection.ToString(), SourceUrl = sourceUrl, ImageUrl = imageUrl, Order = index }); ++index; } } return(resultProducts); }
public IEnumerable <IProduct> Search(string keyword) { var url = "https://www.game.co.za"; var doc = _webScrapper.Scrap($"{url}/game-za/en/search/?text={keyword}"); var elements = doc.DocumentNode.CssSelect(".product-item.productListerGridDiv"); var resultProducts = new List <Product>(); if (elements.Any()) { var index = 0; foreach (var element in elements) { var productLinkElementAttributes = element.CssSelect(".thumb.gtmProductLink").First().Attributes; var sourceUrl = $"{url}{productLinkElementAttributes.AttributesWithName("href").First().Value}"; var name = productLinkElementAttributes.AttributesWithName("title").First().Value; var imageElementAttributes = element.CssSelect(".productPrimaryImage > img").First().Attributes; var imageUrl = $"{url}{imageElementAttributes.AttributesWithName("src").First().Value}"; var cultures = new CultureInfo("en-US"); var price = Convert.ToDecimal(element.CssSelect(".details > .price > .finalPrice").First().InnerText.Trim('R'), cultures); resultProducts.Add(new Product { Name = HttpUtility.HtmlDecode(name), Description = name, Price = price, Units = 1, Brand = null, Source = StoreSiteName.Game.ToString(), SourceUrl = sourceUrl, ImageUrl = imageUrl, Order = index }); ++index; } } return(resultProducts); }
public void GivenUrlWhenSearchingScrapThenReturnSiteContent() { var result = _scrapper.Scrap("https://www.takealot.com/all?qsearch=laptop"); Assert.Contains("Takealot", result.DocumentNode.SelectSingleNode("//head/title").InnerHtml); }