private static async Task GetAmazonTrProductPrices(bool saveChanges) { const string url = "https://www.amazon.com.tr/s/ref=lp_12601909031_nr_n_1?fst=as%3Aoff&rh=n%3A12466439031%2Cn%3A%2112466440031%2Cn%3A12601909031%2Cn%3A12601984031&bbn=12601909031&ie=UTF8&qid=1548935178&rnid=12601909031"; var client = new HttpClient(); client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"); var html = await client.GetStringAsync(url); var htmlDoc = new HtmlDocument(); htmlDoc.LoadHtml(html); var productHtml = htmlDoc.DocumentNode.Descendants("ul") .Where(q => q.GetAttributeValue("class", "") .Equals( "s-result-list s-col-3 s-col-ws-4 s-result-list-hgrid s-height-equalized s-grid-view s-text-condensed")) .ToList(); var productHtmlList = productHtml[0].Descendants("li").ToList(); var list = new List <HTMLModel>(); foreach (var item in productHtmlList) { var prod = new HTMLModel { ProductName = item.Descendants("img").Select(q => q.GetAttributeValue("alt", "")).FirstOrDefault(), ProductPrice = item.Descendants("span") .FirstOrDefault(q => q.GetAttributeValue("class", "") .Equals("a-size-base a-color-price s-price a-text-bold")) ?.InnerText }; if (prod.ProductPrice == null) { prod.ProductPrice = item.Descendants("span") .FirstOrDefault(q => q.GetAttributeValue("class", "").Equals("a-size-base a-color-price a-text-bold")) ?.InnerText; } list.Add(prod); } }
private static async Task <List <HTMLModel> > GetHepsiBuradaProductPrices(bool saveChanges) { var count = 1; var goOn = true; var productList = new List <HTMLModel>(); foreach (var urlR in _json["urls"]) { while (goOn) { var url = urlR.ToString().Replace("{count}", count.ToString()); var productType = url.Split(".com/")[1].Split("-c-").First(); var client = new HttpClient(); client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"); var html = await client.GetStringAsync(url); var htmlDoc = new HtmlDocument(); htmlDoc.LoadHtml(html); var productHtml = htmlDoc.DocumentNode.Descendants("ul") .Where(q => q.GetAttributeValue("class", "").Equals("product-list results-container do-flex ")) .ToList(); var productHtmlList = productHtml[0].Descendants("a").ToList(); foreach (var item in productHtmlList) { if (item == null) { continue; } var productId = item.GetAttributeValue("data-productid", ""); var productUrl = "www.hepsiburada.com" + item.GetAttributeValue("href", ""); var productName = item.Descendants("h3") .Select(q => q.GetAttributeValue("title", "")) .FirstOrDefault(); var htmlPrice = item.Descendants("div") .FirstOrDefault(q => q.GetAttributeValue("class", "").Equals("price-value")) ?.InnerText.Replace(" ", string.Empty); var price = new StringBuilder(); if (htmlPrice == null) { htmlPrice = item.Descendants("span") .FirstOrDefault(q => q.GetAttributeValue("class", "").Equals("price product-price")) ?.InnerText; } if (htmlPrice == null) { continue; } var split = htmlPrice.Split(","); foreach (var str in split) { if (price.Length > 0) { price.Append(","); } str.Where(char.IsDigit).ToList().ForEach(q => price.Append(q)); } var model = new HTMLModel { ProductId = productId, ProductName = productName, ProductPrice = price.ToString(), ProductUrl = productUrl, ProductType = productType }; if (productList.Any(q => q.ProductName == model.ProductName && q.ProductPrice == model.ProductPrice)) { goOn = false; continue; } productList.Add(model); } count++; } } if (saveChanges) { foreach (var item in productList) { _db.Products.Add(item); } _db.SaveChanges(); } return(productList); }