private static async Task Parse() { var html = await GetData(PageUrl); var doc = new HtmlDocument(); doc.LoadHtml(html); var result = new List <PriceChartingModel>(); var brands = HtmlDocumentHelper.GetNodesByParams(doc.DocumentNode, HtmlTag.li, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "brand").Where(x => x.GetAttributeValue("id", null) != null && x.GetAttributeValue("id", null) != "tools"); foreach (var brand in brands) { try { var dropDownMenu = HtmlDocumentHelper.GetNodeByParamsUseXpathStartsWith(brand, HtmlTag.ul, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "menu-dropdown"); if (dropDownMenu != null) { var regions = HtmlDocumentHelper.GetNodesByParams(dropDownMenu, HtmlTag.ul, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "region"); if (regions != null) { foreach (var region in regions) { var regionName = HtmlDocumentHelper.GetNodeByParams(region, HtmlTag.li, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "title")?.InnerText; var categories = region.SelectNodes(".//li").Where(x => x.InnerHtml != string.Empty); foreach (var category in categories) { if (category.ChildNodes[0] == null || category.ChildNodes[0].GetAttributeValue("href", null) == null) { continue; } var url = category.ChildNodes[0].GetAttributeValue("href", null); var name = category.ChildNodes[0].InnerText; result.Add(new PriceChartingModel { Region = regionName, CategoryName = name, URL = url }); } } } else { var categories = brand.SelectNodes(".//li").Where(x => x.InnerHtml != string.Empty); foreach (var category in categories) { if (category.ChildNodes[0]?.GetAttributeValue("href", null) == null) { continue; } var url = category.ChildNodes[0].GetAttributeValue("href", null); var name = category.ChildNodes[0].InnerText; result.Add(new PriceChartingModel { Region = "NTSC (USA)", CategoryName = name, URL = url }); } } } } catch { } } foreach (var category in result) { try { var categoryData = new List <Data>(); category.Data = new List <Data>(); var response = new List <Product>(); var pageCount = 0; while (true) { try { var jUrl = PageUrlJson.Replace("{category}", category.URL.Split('/').Last()).Replace("{count}", pageCount.ToString()); var dataJson = await GetData(jUrl); var jsonObj = JsonConvert.DeserializeObject <Response>(dataJson); response.AddRange(jsonObj.products); if (jsonObj.products.Length == 50) { pageCount += 50; Thread.Sleep(100); continue; } break; } catch { } } foreach (var item in response) { categoryData.Add(new Data { Title = item.productName, LoosePrice = item.price1, CIBPrice = item.price3, NewPrice = item.price2 }); } category.Data = categoryData; } catch (Exception ex) { } // details foreach (var data in category.Data) { try { var url = $"{category.URL.Replace("console", "game")}/{ExtensionMethods.Replace(data.Title, new char[] { '[', ']', '(', ')', '/', '\\', '.', ':', ',', '?' }, " ").Replace(' ', '-')}"; Thread.Sleep(200); html = await GetData(url); doc = new HtmlDocument(); doc.LoadHtml(html); var attributeNode = HtmlDocumentHelper.GetNodeByParams(doc.DocumentNode, HtmlTag.table, Scrapping.AllPossibilities.Enums.HtmlAttribute.id, "attribute"); if (attributeNode != null) { var detailNodes = attributeNode.SelectNodes(".//tr"); var genre = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Genre:"); var releaseDate = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Release Date:"); var rating = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "ESRB Rating:"); var publisher = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Publisher:"); var developer = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Developer:"); var playerCount = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Player Count:"); var upc = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "UPC:"); var asin = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "ASIN (Amazon):"); var epid = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "ePID (eBay):"); var priceChartingId = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "PriceCharting ID:"); if (genre != null) { data.Genre = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(genre, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n')); } if (releaseDate != null) { data.ReleaseDate = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(releaseDate, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n')); } if (rating != null) { data.Rating = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(rating, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n')); } if (publisher != null) { data.Publisher = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(publisher, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n')); } if (developer != null) { data.Developer = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(developer, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n')); } if (playerCount != null) { data.PlayerCount = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(playerCount, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n')); } if (upc != null) { data.UPC = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(upc, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n')); } if (asin != null) { data.Amazon_ASIN = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(asin, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n')); } if (epid != null) { data.Ebay_ePID = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(epid, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n')); } if (priceChartingId != null) { data.PriceChartingId = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(priceChartingId, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n')); } } } catch { } } await WriteJson(JsonConvert.SerializeObject(category, Formatting.Indented), category.CategoryName); await Task.Delay(500); } await WriteJson(JsonConvert.SerializeObject(result, Formatting.Indented)); }
private static async Task Parse() { var requestHelper = new RequestHelper(); var header = HeaderBuilder.BuildOwnHeaders(new HeaderModel() { Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", Host = "gamevaluenow.com", Referer = "https://gamevaluenow.com/", User_Agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36" }); var html = await requestHelper.SendRequestAsync(PageUrl, headers : HeaderBuilder.GetDefaultHeaders()); var doc = new HtmlDocument(); doc.LoadHtml(html); var result = new List <GameValueNowModel>(); var nodes = HtmlDocumentHelper.GetNodesByParamsUseXpathStartsWith(doc.DocumentNode, HtmlTag.a, HtmlAttribute._class, "brand-"); foreach (var node in nodes) { try { var href = node.GetAttributeValue("href", null); if (href is null || href == "#") { continue; } var model = new GameValueNowModel(); var url = $"{PageUrl}{href}"; var name = node.InnerText; model.URL = url; model.PlatformName = WebUtility.HtmlDecode(name); if (result.Select(x => x.PlatformName).Contains(model.PlatformName)) { continue; } result.Add(model); } catch { //ignore; } } foreach (var item in result) { try { item.Data = new List <Data>(); var document = new HtmlDocument(); var dataHtml = await requestHelper.SendRequestAsync(item.URL, headers : header); document.LoadHtml(dataHtml); // stats var statsNode = HtmlDocumentHelper.GetNodeByParams(document.DocumentNode, HtmlTag.div, HtmlAttribute.id, "stats"); var statListNode = HtmlDocumentHelper.GetNodesByParams(statsNode, HtmlTag.div, HtmlAttribute._class, "col-100 stat"); var avgLoosePrice = statListNode.FirstOrDefault(x => x.InnerText.Contains("Avg Loose")); var avgCompletePrice = statListNode.FirstOrDefault(x => x.InnerText.Contains("Avg Complete")); var looseSetValue = statListNode.FirstOrDefault(x => x.InnerText.Contains("Loose Set")); var completeSetValue = statListNode.FirstOrDefault(x => x.InnerText.Contains("Complete Set")); var sharpOfGames = statListNode.FirstOrDefault(x => x.InnerText.Contains("#")); if (avgLoosePrice != null) { item.AvgLoosePrice = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(avgLoosePrice, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim(); } if (avgCompletePrice != null) { item.AvgCompletePrice = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(avgCompletePrice, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim(); } if (looseSetValue != null) { item.LooseSetValue = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(looseSetValue, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim(); } if (completeSetValue != null) { item.CompleteSetValue = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(completeSetValue, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim(); } if (sharpOfGames != null) { item.SharpOfGames = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(sharpOfGames, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim(); } // items var listNode = HtmlDocumentHelper.GetNodeByParams(document.DocumentNode, HtmlTag.div, HtmlAttribute.id, "item-list"); var collectionItemNodes = HtmlDocumentHelper.GetNodesByParamsUseXpathStartsWith(listNode, HtmlTag.div, HtmlAttribute._class, "item-row desktop all"); foreach (var collectionItemNode in collectionItemNodes) { try { var data = new Data(); var name = HtmlDocumentHelper .GetNodeByParams(collectionItemNode, HtmlTag.a, HtmlAttribute._class, "game-link").InnerText; var id = HtmlDocumentHelper .GetNodeByParams(collectionItemNode, HtmlTag.div, HtmlAttribute._class, "item-number").InnerText .Replace("\n", "").Replace(" ", ""); data.Title = name; data.Id = id; var priceContainer = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(collectionItemNode, HtmlTag.div, HtmlAttribute._class, "price-col-container"); if (priceContainer is null) { continue; } var prices = HtmlDocumentHelper.GetNodesByParamsUseXpathStartsWith(priceContainer, HtmlTag.a, HtmlAttribute._class, "game-link"); if (prices is null) { continue; } var priceCount = prices.Count; if (priceCount > 0) { data.Loose = prices[0].InnerText; } if (priceCount > 1) { data.Complete = prices[1].InnerText; } if (priceCount > 2) { data.New = prices[2].InnerText; } if (priceCount > 3) { data.Graded = prices[3].InnerText; } data.PlatformName = item.PlatformName; item.Data.Add(data); } catch { //ignore } } } catch (Exception e) { //ignore } await Task.Delay(500); } await SaveToJsonFile(result); //await SaveToSql(result); }