public async Task ParseAmazon()
{
try
{
var helper = new RequestHelper();
var data = await helper.SendRequestAsync(LinkAmazon, headers : HeaderBuilder.GetDefaultHeaders(), automaticDecompression : true);
if (!data.Contains("Currently unavailable"))
{
var document = new HtmlDocument();
document.LoadHtml(data);
var text = HtmlDocumentHelper.GetNodeByParams(document.DocumentNode, HtmlTag.div, HtmlAttribute.id, "centerCol").InnerText.Replace("\n", " ");
_emailHelper.SendEmail(text);
}
else
{
File.AppendAllText(@"D:\playstationLog.log", $"{DateTime.Now}: Not available in Amazon{Environment.NewLine}");
}
}
catch (Exception e)
{
File.AppendAllText(@"D:\playstationLog.log", $"{DateTime.Now}: {e}{Environment.NewLine}");
}
}
public static async Task <List <DataModelBase> > Start(string link)
{
var result = new List <DataModelBase>();
try
{
Driver.Navigate().GoToUrl(string.IsNullOrEmpty(link) ? Link : link);
var doc = new HtmlDocument();
doc.LoadHtml(Driver.PageSource);
var nodes = doc.DocumentNode;
var pageNumber = Convert.ToInt32(HtmlDocumentHelper.GetNodeByParams(nodes, "nav", "class", "be-pagination u-margin-t-large").ChildNodes[4].InnerText);
// iterate through every page
try
{
for (var i = 1; i <= pageNumber; i++)
{
var sb = new StringBuilder(); // list of ids
var listPerPage = new List <DataModel>(); // list of lots per page
Driver.Navigate().GoToUrl(Link + $"?page={i}");
doc.LoadHtml(Driver.PageSource);
var lots = HtmlDocumentHelper.GetNodeByParams(doc.DocumentNode, "div", "class", "be-lot-list gallery");
foreach (var lot in lots.ChildNodes)
{
try
{
var name = HtmlDocumentHelper.GetNodeByParams(lot, "h2", "class", "c-card__title be-lot__title").InnerText;
var url = HtmlDocumentHelper.GetNodeByParams(lot, "a", "class", "c-card").GetAttributeValue("href", null);
var id = new Uri(url).Segments[2].Split('-')[0];
listPerPage.Add(new DataModel {
Name = name, Url = url, CurrentBid = int.Parse(id)
});
sb.Append($",{id}");
}
catch
{
//ignore
}
}
var ids = sb.ToString().TrimStart(',');
var helper = new RequestHelper();
var response = await helper.SendRequestAsync($"{JsonUrl}?ids={ids}", automaticDecompression : true, headers : HeaderBuilder.GetDefaultHeaders());
var list = JsonConvert.DeserializeObject <JsonResult>(response);
foreach (var item in listPerPage)
{
try
{
var correspondentItem = list.Lots.FirstOrDefault(x => x.id == item.CurrentBid);
item.BiddingEndTime = correspondentItem.bidding_end_time;
var amount = correspondentItem.current_bid_amount;
item.CurrentBidAmount = amount.EUR;
item.ReservedPrice = correspondentItem.reserve_price_met.HasValue && correspondentItem.reserve_price_met.Value ? "Has reserve price" : "No reserve price";
result.Add(item);
}
catch
{
//ignore
}
}
;
}
}
catch
{
//ignore
}
}
catch (Exception e)
{
//ignore
}
return(result);
}
private static async Task Parse()
{
var html = await GetData(PageUrl);
var doc = new HtmlDocument();
doc.LoadHtml(html);
var result = new List <PriceChartingModel>();
var brands = HtmlDocumentHelper.GetNodesByParams(doc.DocumentNode, HtmlTag.li, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "brand").Where(x => x.GetAttributeValue("id", null) != null && x.GetAttributeValue("id", null) != "tools");
foreach (var brand in brands)
{
try
{
var dropDownMenu = HtmlDocumentHelper.GetNodeByParamsUseXpathStartsWith(brand, HtmlTag.ul, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "menu-dropdown");
if (dropDownMenu != null)
{
var regions = HtmlDocumentHelper.GetNodesByParams(dropDownMenu, HtmlTag.ul, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "region");
if (regions != null)
{
foreach (var region in regions)
{
var regionName = HtmlDocumentHelper.GetNodeByParams(region, HtmlTag.li, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "title")?.InnerText;
var categories = region.SelectNodes(".//li").Where(x => x.InnerHtml != string.Empty);
foreach (var category in categories)
{
if (category.ChildNodes[0] == null || category.ChildNodes[0].GetAttributeValue("href", null) == null)
{
continue;
}
var url = category.ChildNodes[0].GetAttributeValue("href", null);
var name = category.ChildNodes[0].InnerText;
result.Add(new PriceChartingModel
{
Region = regionName,
CategoryName = name,
URL = url
});
}
}
}
else
{
var categories = brand.SelectNodes(".//li").Where(x => x.InnerHtml != string.Empty);
foreach (var category in categories)
{
if (category.ChildNodes[0]?.GetAttributeValue("href", null) == null)
{
continue;
}
var url = category.ChildNodes[0].GetAttributeValue("href", null);
var name = category.ChildNodes[0].InnerText;
result.Add(new PriceChartingModel
{
Region = "NTSC (USA)",
CategoryName = name,
URL = url
});
}
}
}
}
catch { }
}
foreach (var category in result)
{
try
{
var categoryData = new List <Data>();
category.Data = new List <Data>();
var response = new List <Product>();
var pageCount = 0;
while (true)
{
try
{
var jUrl = PageUrlJson.Replace("{category}", category.URL.Split('/').Last()).Replace("{count}", pageCount.ToString());
var dataJson = await GetData(jUrl);
var jsonObj = JsonConvert.DeserializeObject <Response>(dataJson);
response.AddRange(jsonObj.products);
if (jsonObj.products.Length == 50)
{
pageCount += 50;
Thread.Sleep(100);
continue;
}
break;
}
catch
{
}
}
foreach (var item in response)
{
categoryData.Add(new Data
{
Title = item.productName,
LoosePrice = item.price1,
CIBPrice = item.price3,
NewPrice = item.price2
});
}
category.Data = categoryData;
}
catch (Exception ex)
{
}
// details
foreach (var data in category.Data)
{
try
{
var url = $"{category.URL.Replace("console", "game")}/{ExtensionMethods.Replace(data.Title, new char[] { '[', ']', '(', ')', '/', '\\', '.', ':', ',', '?' }, " ").Replace(' ', '-')}";
Thread.Sleep(200);
html = await GetData(url);
doc = new HtmlDocument();
doc.LoadHtml(html);
var attributeNode = HtmlDocumentHelper.GetNodeByParams(doc.DocumentNode, HtmlTag.table, Scrapping.AllPossibilities.Enums.HtmlAttribute.id, "attribute");
if (attributeNode != null)
{
var detailNodes = attributeNode.SelectNodes(".//tr");
var genre = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Genre:");
var releaseDate = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Release Date:");
var rating = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "ESRB Rating:");
var publisher = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Publisher:");
var developer = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Developer:");
var playerCount = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Player Count:");
var upc = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "UPC:");
var asin = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "ASIN (Amazon):");
var epid = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "ePID (eBay):");
var priceChartingId = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "PriceCharting ID:");
if (genre != null)
{
data.Genre = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(genre, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (releaseDate != null)
{
data.ReleaseDate = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(releaseDate, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (rating != null)
{
data.Rating = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(rating, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (publisher != null)
{
data.Publisher = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(publisher, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (developer != null)
{
data.Developer = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(developer, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (playerCount != null)
{
data.PlayerCount = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(playerCount, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (upc != null)
{
data.UPC = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(upc, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (asin != null)
{
data.Amazon_ASIN = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(asin, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (epid != null)
{
data.Ebay_ePID = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(epid, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (priceChartingId != null)
{
data.PriceChartingId = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(priceChartingId, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
}
}
catch
{
}
}
await WriteJson(JsonConvert.SerializeObject(category, Formatting.Indented), category.CategoryName);
await Task.Delay(500);
}
await WriteJson(JsonConvert.SerializeObject(result, Formatting.Indented));
}
private static async Task Parse()
{
var requestHelper = new RequestHelper();
var header = HeaderBuilder.BuildOwnHeaders(new HeaderModel()
{
Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
Host = "gamevaluenow.com",
Referer = "https://gamevaluenow.com/",
User_Agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36"
});
var html = await requestHelper.SendRequestAsync(PageUrl, headers : HeaderBuilder.GetDefaultHeaders());
var doc = new HtmlDocument();
doc.LoadHtml(html);
var result = new List <GameValueNowModel>();
var nodes = HtmlDocumentHelper.GetNodesByParamsUseXpathStartsWith(doc.DocumentNode, HtmlTag.a, HtmlAttribute._class, "brand-");
foreach (var node in nodes)
{
try
{
var href = node.GetAttributeValue("href", null);
if (href is null || href == "#")
{
continue;
}
var model = new GameValueNowModel();
var url = $"{PageUrl}{href}";
var name = node.InnerText;
model.URL = url;
model.PlatformName = WebUtility.HtmlDecode(name);
if (result.Select(x => x.PlatformName).Contains(model.PlatformName))
{
continue;
}
result.Add(model);
}
catch
{
//ignore;
}
}
foreach (var item in result)
{
try
{
item.Data = new List <Data>();
var document = new HtmlDocument();
var dataHtml = await requestHelper.SendRequestAsync(item.URL, headers : header);
document.LoadHtml(dataHtml);
// stats
var statsNode = HtmlDocumentHelper.GetNodeByParams(document.DocumentNode, HtmlTag.div, HtmlAttribute.id, "stats");
var statListNode = HtmlDocumentHelper.GetNodesByParams(statsNode, HtmlTag.div, HtmlAttribute._class, "col-100 stat");
var avgLoosePrice = statListNode.FirstOrDefault(x => x.InnerText.Contains("Avg Loose"));
var avgCompletePrice = statListNode.FirstOrDefault(x => x.InnerText.Contains("Avg Complete"));
var looseSetValue = statListNode.FirstOrDefault(x => x.InnerText.Contains("Loose Set"));
var completeSetValue = statListNode.FirstOrDefault(x => x.InnerText.Contains("Complete Set"));
var sharpOfGames = statListNode.FirstOrDefault(x => x.InnerText.Contains("#"));
if (avgLoosePrice != null)
{
item.AvgLoosePrice = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(avgLoosePrice, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim();
}
if (avgCompletePrice != null)
{
item.AvgCompletePrice = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(avgCompletePrice, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim();
}
if (looseSetValue != null)
{
item.LooseSetValue = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(looseSetValue, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim();
}
if (completeSetValue != null)
{
item.CompleteSetValue = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(completeSetValue, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim();
}
if (sharpOfGames != null)
{
item.SharpOfGames = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(sharpOfGames, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim();
}
// items
var listNode =
HtmlDocumentHelper.GetNodeByParams(document.DocumentNode, HtmlTag.div, HtmlAttribute.id, "item-list");
var collectionItemNodes = HtmlDocumentHelper.GetNodesByParamsUseXpathStartsWith(listNode, HtmlTag.div,
HtmlAttribute._class, "item-row desktop all");
foreach (var collectionItemNode in collectionItemNodes)
{
try
{
var data = new Data();
var name = HtmlDocumentHelper
.GetNodeByParams(collectionItemNode, HtmlTag.a, HtmlAttribute._class, "game-link").InnerText;
var id = HtmlDocumentHelper
.GetNodeByParams(collectionItemNode, HtmlTag.div, HtmlAttribute._class, "item-number").InnerText
.Replace("\n", "").Replace(" ", "");
data.Title = name;
data.Id = id;
var priceContainer = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(collectionItemNode, HtmlTag.div, HtmlAttribute._class, "price-col-container");
if (priceContainer is null)
{
continue;
}
var prices = HtmlDocumentHelper.GetNodesByParamsUseXpathStartsWith(priceContainer, HtmlTag.a, HtmlAttribute._class, "game-link");
if (prices is null)
{
continue;
}
var priceCount = prices.Count;
if (priceCount > 0)
{
data.Loose = prices[0].InnerText;
}
if (priceCount > 1)
{
data.Complete = prices[1].InnerText;
}
if (priceCount > 2)
{
data.New = prices[2].InnerText;
}
if (priceCount > 3)
{
data.Graded = prices[3].InnerText;
}
data.PlatformName = item.PlatformName;
item.Data.Add(data);
}
catch
{
//ignore
}
}
}
catch (Exception e)
{
//ignore
}
await Task.Delay(500);
}
await SaveToJsonFile(result);
//await SaveToSql(result);
}