private async Task <bool> Search(string url, string manufacturer, string model, bool strictSearch) { CultureInfo culture = new CultureInfo("fr-FR", false); var response = await m_refScraper.Analyze(new Uri(url)); if (response.StatusCode == System.Net.HttpStatusCode.OK) { var html = await response.Content.ReadAsStringAsync(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(html); var root = doc.DocumentNode; var productNodes = root.CssSelect("div.second-third-block"); foreach (var productNode in productNodes) { string m = string.Empty; string price = string.Empty; string murl = string.Empty; string brand = string.Empty; foreach (HtmlNode name in productNode.CssSelect(".product-name > a")) { murl = name.GetAttributeValue("href"); int pos = culture.CompareInfo.IndexOf(murl, "-" + model, CompareOptions.IgnoreCase); if (pos == -1) { pos = murl.LastIndexOf('-'); } brand = murl.Substring(0, pos); pos = brand.LastIndexOf('-'); brand = UppercaseFirst(brand.Substring(pos + 1)); m = UppercaseFirst(HttpUtility.HtmlDecode(name.InnerHtml)); } foreach (HtmlNode priceNode in productNode.CssSelect("span.product-price")) { price = HttpUtility.HtmlDecode(priceNode.InnerHtml).Trim(); price = price.Replace(",00", ""); price = Regex.Replace(price, @"\s+", ""); price = price.Insert(price.Length - 1, " "); } bool canAdd = culture.CompareInfo.IndexOf(m, model, CompareOptions.IgnoreCase) >= 0; if (strictSearch) { canAdd &= culture.CompareInfo.IndexOf(brand, manufacturer, CompareOptions.IgnoreCase) >= 0; } if (canAdd) { m_refScraper.AddPrice(brand, m, ShopName, murl, price); } } } else { m_refScraper.ThrowException(ShopName, response.ReasonPhrase); } return(true); }
private async Task <bool> Search(string url, string manufacturer, string model, bool strictSearch) { try { CultureInfo culture = new CultureInfo("fr-FR", false); var response = await m_refScraper.Analyze(new Uri(url)); if (response.StatusCode == System.Net.HttpStatusCode.OK) { var html = await response.Content.ReadAsStringAsync(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(html); var root = doc.DocumentNode; var productNodes = root.CssSelect("li.ajax_block_product"); foreach (var node in productNodes) { string m = string.Empty; string price = string.Empty; string murl = string.Empty; string brand = string.Empty; foreach (HtmlNode modelNode in node.CssSelect("a.product-name")) { m = modelNode.GetAttributeValue("title"); WebScraper.ExtractBrandFromModel(manufacturer, ref m, ref brand); } foreach (HtmlNode hrefNode in node.CssSelect("a.quick-view")) { murl = hrefNode.GetAttributeValue("href"); } foreach (HtmlNode priceNode in node.CssSelect("span.product-price")) { price = priceNode.InnerHtml; price = price.Trim().Replace("\u202F", "").Replace(",", "."); price = price.Replace(".00", ""); } bool canAdd = culture.CompareInfo.IndexOf(m, model, CompareOptions.IgnoreCase) >= 0; if (strictSearch) { canAdd &= culture.CompareInfo.IndexOf(brand, manufacturer, CompareOptions.IgnoreCase) >= 0; } if (canAdd) { m_refScraper.AddPrice(brand, m, ShopName, murl, price); } } } } catch (Exception ex) { m_refScraper.ThrowException(ShopName, ex.Message); } return(true); }
private async Task <bool> Search(string url, string manufacturer, string model, bool strictSearch) { CultureInfo culture = new CultureInfo("fr-FR", false); var response = await m_refScraper.Analyze(new Uri(url)); if (response.StatusCode == System.Net.HttpStatusCode.OK) { var html = await response.Content.ReadAsStringAsync(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(html); var root = doc.DocumentNode; var productNodes = root.CssSelect("li.product-list__item"); foreach (var productNode in productNodes) { string m = string.Empty; string price = string.Empty; string murl = string.Empty; string brand = string.Empty; foreach (HtmlNode infoNode in productNode.CssSelect("div.test-a > a")) { string info = HttpUtility.HtmlDecode(infoNode.GetAttributeValue("onclick")); var data = info.Split(','); m = data[1].Trim('\''); brand = data[4].Trim('\''); } foreach (HtmlNode urlNode in productNode.CssSelect("h2.delta > a")) { murl = "https://kytary.fr" + urlNode.GetAttributeValue("href"); } foreach (HtmlNode priceNode in productNode.CssSelect("p.price")) { price = HttpUtility.HtmlDecode(priceNode.InnerHtml).Trim(); price = price.Replace(",00", ""); price = Regex.Replace(price, @"\s+", ""); price = price.Insert(price.Length - 1, " "); price = price.Replace(",", "."); } bool canAdd = culture.CompareInfo.IndexOf(m, model, CompareOptions.IgnoreCase) >= 0; if (strictSearch) { canAdd &= culture.CompareInfo.IndexOf(brand, manufacturer, CompareOptions.IgnoreCase) >= 0; } if (canAdd) { m_refScraper.AddPrice(brand, m, ShopName, murl, price); } } } else { m_refScraper.ThrowException(ShopName, response.ReasonPhrase); } return(true); }
private async Task <bool> Search(string url, string manufacturer, string model, bool strictSearch) { CultureInfo culture = new CultureInfo("fr-FR", false); var response = await m_refScraper.Analyze(new Uri(url)); if (response.StatusCode == System.Net.HttpStatusCode.OK) { var html = await response.Content.ReadAsStringAsync(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(html); var root = doc.DocumentNode; var productNodes = root.CssSelect("div.fx-product-list-entry"); foreach (var node in productNodes) { string m = string.Empty; string price = string.Empty; string murl = string.Empty; string brand = string.Empty; foreach (HtmlNode brandNode in node.CssSelect("span.title__manufacturer")) { brand = WebScraper.UppercaseFirst(brandNode.InnerHtml); } foreach (HtmlNode modelNode in node.CssSelect("span.title__name")) { m = modelNode.InnerHtml; } foreach (HtmlNode priceNode in node.CssSelect("span.fx-typography-price-primary")) { price = priceNode.InnerHtml; price = price.Trim().Replace(".", ""); } foreach (HtmlNode urlNode in node.CssSelect("a.product__content")) { murl = urlNode.GetAttributeValue("href"); } bool canAdd = culture.CompareInfo.IndexOf(m, model, CompareOptions.IgnoreCase) >= 0; if (strictSearch) { canAdd &= culture.CompareInfo.IndexOf(brand, manufacturer, CompareOptions.IgnoreCase) >= 0; } if (canAdd) { m_refScraper.AddPrice(brand, m, ShopName, murl, price); } } } else { m_refScraper.ThrowException(ShopName, response.ReasonPhrase); } return(true); }
private async Task <bool> Search(string url, string manufacturer, string model, bool strictSearch) { CultureInfo culture = new CultureInfo("fr-FR", false); var response = await m_refScraper.Analyze(new Uri(url)); if (response.StatusCode == System.Net.HttpStatusCode.OK) { var html = await response.Content.ReadAsStringAsync(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(html); var root = doc.DocumentNode; var productNodes = root.CssSelect("div.products-grid-item"); foreach (var productNode in productNodes) { string m = string.Empty; string price = string.Empty; string murl = string.Empty; string brand = string.Empty; foreach (HtmlNode infoNode in productNode.CssSelect(".pdt_brand")) { brand = WebScraper.UppercaseFirst(infoNode.InnerHtml.Trim()); } foreach (HtmlNode urlNode in productNode.CssSelect("a.title")) { murl = "https://www.stars-music.fr" + urlNode.GetAttributeValue("href"); } foreach (HtmlNode nameNode in productNode.CssSelect(".pdt_name")) { m = WebScraper.UppercaseFirst(nameNode.InnerText.Trim()); } foreach (HtmlNode priceNode in productNode.CssSelect("div.price")) { price = HttpUtility.HtmlDecode(priceNode.InnerHtml).Trim(); price = price.Replace(".00", ""); } bool canAdd = culture.CompareInfo.IndexOf(m, model, CompareOptions.IgnoreCase) >= 0; if (strictSearch) { canAdd &= culture.CompareInfo.IndexOf(brand, manufacturer, CompareOptions.IgnoreCase) >= 0; } if (canAdd) { m_refScraper.AddPrice(brand, m, ShopName, murl, price); } } } else { m_refScraper.ThrowException(ShopName, response.ReasonPhrase); } return(true); }
private async Task <bool> Search(string url, string manufacturer, string model, bool strictSearch) { CultureInfo culture = new CultureInfo("fr-FR", false); var response = await m_refScraper.Analyze(new Uri(url)); if (response.StatusCode == System.Net.HttpStatusCode.OK) { var html = await response.Content.ReadAsStringAsync(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(html); var root = doc.DocumentNode; var productNode = root.CssSelect(".product-box"); foreach (var node in productNode) { string enc = node.GetAttributeValue("data-product"); string info = HttpUtility.HtmlDecode(enc); var productInfo = JsonConvert.DeserializeObject <dynamic>(info); string brand = productInfo.brand; string murl = url; string m = productInfo.name; string price = productInfo.price; price = price.Replace(".00", ""); string currency = CurrencyTools.GetCurrencySymbol((string)productInfo.currencyCode); WebScraper.ExtractBrandFromModel(manufacturer, ref m, ref brand); foreach (HtmlNode urlNode in node.CssSelect("div.product-thumb > a")) { murl = "https://www.bax-shop.fr" + urlNode.GetAttributeValue("href"); } bool canAdd = culture.CompareInfo.IndexOf(m, model, CompareOptions.IgnoreCase) >= 0; if (strictSearch) { canAdd &= culture.CompareInfo.IndexOf(brand, manufacturer, CompareOptions.IgnoreCase) >= 0; } if (canAdd) { m_refScraper.AddPrice(brand, m, ShopName, murl, price + " " + currency); } } } else { m_refScraper.ThrowException(ShopName, response.ReasonPhrase); } return(true); }
private async Task <bool> Search(string url, string manufacturer, string model, bool strictSearch) { CultureInfo culture = new CultureInfo("fr-FR", false); var response = await m_refScraper.Analyze(new Uri(url)); if (response.StatusCode == System.Net.HttpStatusCode.OK) { var html = await response.Content.ReadAsStringAsync(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(html); var root = doc.DocumentNode; var script = root.CssSelect("script"); string enc = script.First().InnerHtml; string info = HttpUtility.HtmlDecode(enc); info = info.Replace("dataLayer = [", ""); info = info.Replace("'", "\""); info = info.Remove(info.Length - 2); var productInfo = JsonConvert.DeserializeObject <dynamic>(info); var list = productInfo.products; foreach (var product in list) { string m = product.prdName; string brand = product.prdBrand; string price = product.prdAmount; price = price.Replace(".00", "") + " €"; string murl = product.prdUrl; bool canAdd = culture.CompareInfo.IndexOf(m, model, CompareOptions.IgnoreCase) >= 0; if (strictSearch) { canAdd &= culture.CompareInfo.IndexOf(brand, manufacturer, CompareOptions.IgnoreCase) >= 0; } if (canAdd) { m_refScraper.AddPrice(brand, m, ShopName, murl, price); } } } else { m_refScraper.ThrowException(ShopName, response.ReasonPhrase); } return(true); }
private async Task <bool> Search(string url, string manufacturer, string model, bool strictSearch) { CultureInfo culture = new CultureInfo("fr-FR", false); var response = await m_refScraper.Analyze(new Uri(url)); if (response.StatusCode == System.Net.HttpStatusCode.OK) { var html = await response.Content.ReadAsStringAsync(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(html); var root = doc.DocumentNode; var productNodes = root.CssSelect("div.grilleItem"); foreach (var productNode in productNodes) { string m = string.Empty; string price = string.Empty; string murl = string.Empty; string brand = string.Empty; foreach (HtmlNode name in productNode.CssSelect("a.grilleLink")) { murl = name.GetAttributeValue("href"); } foreach (HtmlNode name in productNode.CssSelect("span.articleMarque")) { brand = WebScraper.UppercaseFirst(HttpUtility.HtmlDecode(name.InnerHtml)); } foreach (HtmlNode name in productNode.CssSelect("div.grilleDes")) { m = WebScraper.UppercaseFirst(HttpUtility.HtmlDecode(name.InnerHtml)).Trim(); int pos = m.IndexOf("</span>"); if (pos != -1) { m = WebScraper.UppercaseFirst(m.Substring(pos + 8)).Trim(); } } foreach (HtmlNode priceNode in productNode.CssSelect("span.price")) { price = HttpUtility.HtmlDecode(priceNode.InnerHtml).Trim(); price = price.Replace(".00", ""); price = price.Insert(price.Length - 1, " "); } bool canAdd = culture.CompareInfo.IndexOf(m, model, CompareOptions.IgnoreCase) >= 0; if (strictSearch) { canAdd &= culture.CompareInfo.IndexOf(brand, manufacturer, CompareOptions.IgnoreCase) >= 0; } if (canAdd) { m_refScraper.AddPrice(brand, m, ShopName, murl, price); } } } else { m_refScraper.ThrowException(ShopName, response.ReasonPhrase); } return(true); }
private async Task <bool> Search(string url, string manufacturer, string model, bool strictSearch) { CultureInfo culture = new CultureInfo("fr-FR", false); var response = await m_refScraper.Analyze(new Uri(url)); if (response.StatusCode == System.Net.HttpStatusCode.OK) { var html = await response.Content.ReadAsStringAsync(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(html); var root = doc.DocumentNode; var productNodes = root.CssSelect("div.search_product_item"); foreach (var productNode in productNodes) { string m = string.Empty; string price = string.Empty; string murl = string.Empty; string brand = string.Empty; foreach (HtmlNode urlNode in productNode.CssSelect("a")) { murl = "https://www.energyson.fr/" + urlNode.GetAttributeValue("href"); } foreach (HtmlNode nameNode in productNode.CssSelect("div.name")) { m = WebScraper.UppercaseFirst(nameNode.InnerHtml.Trim()); brand = m.Split().FirstOrDefault(); if (String.IsNullOrEmpty(brand)) { brand = string.Empty; } } foreach (HtmlNode priceNode in productNode.CssSelect("div.price")) { price = HttpUtility.HtmlDecode(priceNode.InnerHtml).Trim(); price = price.Replace(",00", ""); } bool canAdd = culture.CompareInfo.IndexOf(m, model, CompareOptions.IgnoreCase) >= 0; if (strictSearch) { canAdd &= culture.CompareInfo.IndexOf(brand, manufacturer, CompareOptions.IgnoreCase) >= 0; } if (canAdd) { m_refScraper.AddPrice(brand, m, ShopName, murl, price); } } if (productNodes.Count() == 0) { productNodes = root.CssSelect("#product_zoom"); if (productNodes.Count() == 1) { // Direct to product page string m = string.Empty; string price = string.Empty; string murl = string.Empty; string brand = string.Empty; var urlNode = root.CssSelect("div.fb-share-button"); if (urlNode.Count() == 1) { murl = urlNode.First().GetAttributeValue("data-href"); } foreach (HtmlNode dataNode in productNodes.CssSelect("h1")) { brand = dataNode.ChildNodes.First().InnerText; m = dataNode.ChildNodes.Last().InnerText; } foreach (HtmlNode priceNode in productNodes.CssSelect("span.product_zoom_price_vente > span")) { price = HttpUtility.HtmlDecode(priceNode.InnerText).Trim(); price = price.Replace(",00", ""); break; } bool canAdd = culture.CompareInfo.IndexOf(m, model, CompareOptions.IgnoreCase) >= 0; if (strictSearch) { canAdd &= culture.CompareInfo.IndexOf(brand, manufacturer, CompareOptions.IgnoreCase) >= 0; } if (canAdd) { m_refScraper.AddPrice(brand, m, ShopName, murl, price); } } } } else { m_refScraper.ThrowException(ShopName, response.ReasonPhrase); } return(true); }
private async Task <bool> Search(string url, string manufacturer, string model, bool strictSearch) { CultureInfo culture = new CultureInfo("fr-FR", false); var response = await m_refScraper.Analyze(new Uri(url)); if (response.StatusCode == System.Net.HttpStatusCode.OK) { var html = await response.Content.ReadAsStringAsync(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(html); var root = doc.DocumentNode; var productNodes = root.CssSelect("div.tile-product"); foreach (var productNode in productNodes) { string m = string.Empty; string price = string.Empty; string murl = string.Empty; string brand = string.Empty; foreach (HtmlNode urlNode in productNode.CssSelect("a.kor-product-link")) { murl = urlNode.GetAttributeValue("href"); m = urlNode.InnerText.Trim(); WebScraper.ExtractBrandFromModel(manufacturer, ref m, ref brand); } foreach (HtmlNode priceNode in productNode.CssSelect("span.kor-product-sale-price-value")) { price = HttpUtility.HtmlDecode(priceNode.InnerHtml).Trim(); price = price.Replace(",00", ""); price = price.Replace(".", ""); price = price.Replace(",", "."); } bool canAdd = culture.CompareInfo.IndexOf(m, model, CompareOptions.IgnoreCase) >= 0; if (strictSearch) { canAdd &= culture.CompareInfo.IndexOf(brand, manufacturer, CompareOptions.IgnoreCase) >= 0; } if (canAdd) { m_refScraper.AddPrice(brand, m, "Music Store", murl, price); } } if (productNodes.Count() == 0) { productNodes = root.CssSelect("div.productdata"); if (productNodes.Count() == 1) { // Direct to product page string m = string.Empty; string price = string.Empty; string murl = string.Empty; string brand = string.Empty; var urlNode = root.CssSelect("input#RedirectURL"); if (urlNode.Count() == 1) { murl = urlNode.First().GetAttributeValue("value"); } foreach (HtmlNode dataNode in productNodes.CssSelect("h1")) { brand = dataNode.ChildNodes.First().InnerText; m = dataNode.ChildNodes.Last().InnerText; } foreach (HtmlNode priceNode in productNodes.CssSelect("span.kor-product-sale-price-value")) { price = HttpUtility.HtmlDecode(priceNode.InnerHtml).Trim(); price = price.Replace(",00", ""); price = price.Replace(".", ""); price = price.Replace(",", "."); } bool canAdd = culture.CompareInfo.IndexOf(m, model, CompareOptions.IgnoreCase) >= 0; if (strictSearch) { canAdd &= culture.CompareInfo.IndexOf(brand, manufacturer, CompareOptions.IgnoreCase) >= 0; } if (canAdd) { m_refScraper.AddPrice(brand, m, ShopName, murl, price); } } } } else { m_refScraper.ThrowException(ShopName, response.ReasonPhrase); } return(true); }