private static async Task Parse()
{
var html = await GetData(PageUrl);
var doc = new HtmlDocument();
doc.LoadHtml(html);
var result = new List <PriceChartingModel>();
var brands = HtmlDocumentHelper.GetNodesByParams(doc.DocumentNode, HtmlTag.li, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "brand").Where(x => x.GetAttributeValue("id", null) != null && x.GetAttributeValue("id", null) != "tools");
foreach (var brand in brands)
{
try
{
var dropDownMenu = HtmlDocumentHelper.GetNodeByParamsUseXpathStartsWith(brand, HtmlTag.ul, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "menu-dropdown");
if (dropDownMenu != null)
{
var regions = HtmlDocumentHelper.GetNodesByParams(dropDownMenu, HtmlTag.ul, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "region");
if (regions != null)
{
foreach (var region in regions)
{
var regionName = HtmlDocumentHelper.GetNodeByParams(region, HtmlTag.li, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "title")?.InnerText;
var categories = region.SelectNodes(".//li").Where(x => x.InnerHtml != string.Empty);
foreach (var category in categories)
{
if (category.ChildNodes[0] == null || category.ChildNodes[0].GetAttributeValue("href", null) == null)
{
continue;
}
var url = category.ChildNodes[0].GetAttributeValue("href", null);
var name = category.ChildNodes[0].InnerText;
result.Add(new PriceChartingModel
{
Region = regionName,
CategoryName = name,
URL = url
});
}
}
}
else
{
var categories = brand.SelectNodes(".//li").Where(x => x.InnerHtml != string.Empty);
foreach (var category in categories)
{
if (category.ChildNodes[0]?.GetAttributeValue("href", null) == null)
{
continue;
}
var url = category.ChildNodes[0].GetAttributeValue("href", null);
var name = category.ChildNodes[0].InnerText;
result.Add(new PriceChartingModel
{
Region = "NTSC (USA)",
CategoryName = name,
URL = url
});
}
}
}
}
catch { }
}
foreach (var category in result)
{
try
{
var categoryData = new List <Data>();
category.Data = new List <Data>();
var response = new List <Product>();
var pageCount = 0;
while (true)
{
try
{
var jUrl = PageUrlJson.Replace("{category}", category.URL.Split('/').Last()).Replace("{count}", pageCount.ToString());
var dataJson = await GetData(jUrl);
var jsonObj = JsonConvert.DeserializeObject <Response>(dataJson);
response.AddRange(jsonObj.products);
if (jsonObj.products.Length == 50)
{
pageCount += 50;
Thread.Sleep(100);
continue;
}
break;
}
catch
{
}
}
foreach (var item in response)
{
categoryData.Add(new Data
{
Title = item.productName,
LoosePrice = item.price1,
CIBPrice = item.price3,
NewPrice = item.price2
});
}
category.Data = categoryData;
}
catch (Exception ex)
{
}
// details
foreach (var data in category.Data)
{
try
{
var url = $"{category.URL.Replace("console", "game")}/{ExtensionMethods.Replace(data.Title, new char[] { '[', ']', '(', ')', '/', '\\', '.', ':', ',', '?' }, " ").Replace(' ', '-')}";
Thread.Sleep(200);
html = await GetData(url);
doc = new HtmlDocument();
doc.LoadHtml(html);
var attributeNode = HtmlDocumentHelper.GetNodeByParams(doc.DocumentNode, HtmlTag.table, Scrapping.AllPossibilities.Enums.HtmlAttribute.id, "attribute");
if (attributeNode != null)
{
var detailNodes = attributeNode.SelectNodes(".//tr");
var genre = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Genre:");
var releaseDate = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Release Date:");
var rating = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "ESRB Rating:");
var publisher = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Publisher:");
var developer = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Developer:");
var playerCount = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Player Count:");
var upc = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "UPC:");
var asin = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "ASIN (Amazon):");
var epid = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "ePID (eBay):");
var priceChartingId = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "PriceCharting ID:");
if (genre != null)
{
data.Genre = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(genre, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (releaseDate != null)
{
data.ReleaseDate = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(releaseDate, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (rating != null)
{
data.Rating = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(rating, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (publisher != null)
{
data.Publisher = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(publisher, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (developer != null)
{
data.Developer = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(developer, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (playerCount != null)
{
data.PlayerCount = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(playerCount, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (upc != null)
{
data.UPC = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(upc, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (asin != null)
{
data.Amazon_ASIN = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(asin, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (epid != null)
{
data.Ebay_ePID = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(epid, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (priceChartingId != null)
{
data.PriceChartingId = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(priceChartingId, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
}
}
catch
{
}
}
await WriteJson(JsonConvert.SerializeObject(category, Formatting.Indented), category.CategoryName);
await Task.Delay(500);
}
await WriteJson(JsonConvert.SerializeObject(result, Formatting.Indented));
}