/// <summary>
/// 解析下一个链接
/// </summary>
/// <returns></returns>
protected override string ParseNextUrl()
{
if (this.IsEmptyPage())
{
return(null);
}
var currentUrl = this.CurrentUrl;
if (StringExtension.IsNullOrWhiteSpace(currentUrl))
{
return(null);
}
var nextpageUrl = HtmlDocumentHelper.GetValueByXPath(this.HtmlSource, @"//a[text()='下一页']/@href");
if (StringExtension.IsNullOrWhiteSpace(nextpageUrl))
{
return(null);
}
string baseUrl;
NameValueCollection collection;
Url.ParseUrl(currentUrl, out baseUrl, out collection);
var pageNo = int.Parse(collection[@"pageNo"]);
collection["@pageNo"] = $"{pageNo + 1}";
return(Url.CombinUrl(baseUrl, collection));
}
public async Task BankIdAuthentication_Login_Returns_Form_With_Resolved_Cancel_Url()
{
// Arrange
var options = new BankIdLoginOptions(new List <string>(), null, false, true, false, false, "~/cru", DefaultStateCookieName);
var mockProtector = new Mock <IBankIdLoginOptionsProtector>();
mockProtector
.Setup(protector => protector.Unprotect(It.IsAny <string>()))
.Returns(options);
using var server = CreateServer(o =>
{
o.UseSimulatedEnvironment()
.AddSameDevice();
},
DefaultAppConfiguration(async context =>
{
await context.ChallengeAsync(BankIdDefaults.SameDeviceAuthenticationScheme);
}),
services =>
{
services.AddTransient(s => mockProtector.Object);
});
// Act
var request = CreateRequestWithStateCookie(server, "/BankIdAuthentication/Login?returnUrl=%2F&loginOptions=X&orderRef=Y");
var transaction = await request.GetAsync();
// Assert
Assert.Equal(HttpStatusCode.OK, transaction.StatusCode);
var document = await HtmlDocumentHelper.FromContent(transaction.Content);
Assert.Equal("/cru", document.GetInputValue("input[name='CancelReturnUrl']"));
}
public static async Task <HtmlDocument> CreateHtmlDocumentAsync(this IRestClient restClient, Uri uri)
{
return(HtmlDocumentHelper.CreateNew
(
await restClient.GetAsync(uri)
));
}
public async Task BankIdAuthentication_Login_Returns_Form_And_Status()
{
// Arrange
using var server = CreateServer(o =>
{
o.UseSimulatedEnvironment()
.AddSameDevice();
},
DefaultAppConfiguration(async context =>
{
await context.ChallengeAsync(BankIdDefaults.SameDeviceAuthenticationScheme);
}),
services =>
{
services.AddTransient(s => _bankIdLoginOptionsProtector.Object);
});
// Act
var request = CreateRequestWithStateCookie(server, "/BankIdAuthentication/Login?returnUrl=%2F&loginOptions=X&orderRef=Y");
var transaction = await request.GetAsync();
// Assert
Assert.Equal(HttpStatusCode.OK, transaction.StatusCode);
var document = await HtmlDocumentHelper.FromContent(transaction.Content);
Assert.NotNull(document.GetElement <IHtmlFormElement>("form[id='bankIdLoginForm']"));
Assert.NotNull(document.GetElement <IHtmlDivElement>("div[id='bankIdLoginStatus']"));
Assert.NotNull(document.GetElement <IHtmlImageElement>("img.qr-code-image"));
Assert.Equal("/", document.GetInputValue("input[name='ReturnUrl']"));
Assert.Equal("/", document.GetInputValue("input[name='CancelReturnUrl']"));
Assert.Equal("X", document.GetInputValue("input[name='LoginOptions']"));
Assert.Equal("true", document.GetInputValue("input[name='AutoLogin']"));
}
public static async Task <HtmlDocument> CreateHtmlDocumentAsync(this IRestClient restClient, Uri uri, HttpContent content)
{
return(HtmlDocumentHelper.CreateNew
(
await restClient.PostAsync(uri, content)
));
}
private static List <string> GetTagsFromPostContent(HtmlDocument doc)
{
// Parse tags from the content of a blog post that contains HTML
// similar to the following:
//
// <h3>
// Tags</h3>
// <ul>
// <li><a href="..." rel="tag"> My
// System </a></li>
// <li><a href="..." rel="tag">Toolbox</a></li>
// </ul>
//
// For the example HTML above, a list containing "My System" and
// "Toolbox" would be returned.
var tagLinks = doc.DocumentNode.SelectNodes(
"//h3[normalize-space() = 'Tags']/following-sibling::ul/li/a");
var tags = new List <string>();
if (tagLinks != null)
{
foreach (var tagLink in tagLinks)
{
var tag = HtmlDocumentHelper.NormalizeWhitespace(
tagLink.InnerText).Trim();
tags.Add(tag);
}
}
return(tags);
}
/// <summary>
/// 解析当前总页数
/// </summary>
/// <param name="content">The content.</param>
/// <returns></returns>
private int ParseAmountPage(string content)
{
if (content == null)
{
return(-1);
}
var navigator = HtmlDocumentHelper.CreateNavigator(content);
var node = navigator.SelectSingleNode(@"//div[@class='jPage']/em");
if (node == null)
{
return(-1);
}
/*
* <div class="jPage">
* <em>共104条记录</em>
* <span>上一页</span>
* <a class="current">1</a>
* <a href="//mall.jd.com/view_search-504028-1000007084-1000007084-0-5-0-0-1-2-24.html?isGlobalSearch=0">2</a>
* <a href="//mall.jd.com/view_search-504028-1000007084-1000007084-0-5-0-0-1-3-24.html?isGlobalSearch=0">3</a>
* <span>...</span>
* <a href="//mall.jd.com/view_search-504028-1000007084-1000007084-0-5-0-0-1-5-24.html?isGlobalSearch=0">5</a>
* <a href="//mall.jd.com/view_search-504028-1000007084-1000007084-0-5-0-0-1-2-24.html?isGlobalSearch=0">下一页</a>
* </div>
*/
var matchResults = Regex.Match(node.Value, @"(?<=共)\d+(?=条记录)");
return(matchResults.Success ? int.Parse(matchResults.Value) : -1);
}
/// <summary>
/// 推荐是不是商品页面
/// </summary>
/// <param name="webContent"></param>
/// <returns></returns>
private bool GuessIsSearchWebContent(string webContent)
{
if (string.IsNullOrEmpty(webContent))
{
return(false);
}
var navigator = HtmlDocumentHelper.CreateNavigator(webContent);
var iterator = navigator.Select(@"//a");
var itemCount = 0;
foreach (XPathNavigator item in iterator)
{
var href = item.GetAttribute(@"href", string.Empty);
if (href == null)
{
continue;
}
if (Regex.IsMatch(href, @"^//item\.jd\.com/\d+\.html", RegexOptions.IgnoreCase))
{
itemCount++;
}
if (itemCount >= 8)
{
return(true);
}
}
return(false);
}
private void GetPersonPageUrls(string localDir, string listPageUrl, List<Dictionary<string, string>> allPersonPageUrlInfos)
{
try
{
string listPageLocalPath = this.RunPage.GetFilePath(listPageUrl, localDir);
HtmlAgilityPack.HtmlDocument pageHtmlDoc = HtmlDocumentHelper.Load(listPageLocalPath);
HtmlNodeCollection allLiNodes = pageHtmlDoc.DocumentNode.SelectNodes("//ol[@class=\"search-results\"]/li");
foreach (HtmlNode liNode in allLiNodes)
{
if (liNode.GetAttributeValue("class", "").Contains("people"))
{
HtmlNode personLinkNode = liNode.SelectSingleNode("./div[@class=\"bd\"]/h3/a");
string personUrl = CommonUtil.UrlDecodeSymbolAnd(personLinkNode.GetAttributeValue("href", ""));
string personName = personLinkNode.InnerText.Trim();
Dictionary<string, string> personPageUrlInfo = new Dictionary<string, string>();
personPageUrlInfo.Add("personUrl", personUrl);
personPageUrlInfo.Add("personName", personName);
allPersonPageUrlInfos.Add(personPageUrlInfo);
}
}
}
catch (Exception ex)
{
this.RunPage.InvokeAppendLogText(ex.Message, LogLevelType.Error, true);
throw new Exception("解析列表页出错, listPageUrl = +" + listPageUrl, ex);
}
}
/// <summary>
/// 解析出商品
/// </summary>
/// <param name="htmlSource">The HTML source.</param>
/// <param name="listOnly">仅解析出列表，不解析价格等需要再次访问网络的内容.</param>
/// <returns></returns>
private IResut[] ParseCurrentItems(string htmlSource, bool listOnly = false)
{
/*
#if DEBUG
* htmlSource = "";
* var htmlSources = File.ReadAllLines(@"C:\Users\Administrator\Desktop\htmlSource.txt",System.Text.Encoding.UTF8);
* for (int i=0;i< htmlSources.Length;i++)
* {
* htmlSource += htmlSources[i];
* }
*
#endif
*/
const string SkuIdKey = "ProductSku";
var resultList = new List <IResut>();
var navigator = HtmlDocumentHelper.CreateNavigator(htmlSource);
var iterator = navigator.Select(@"//ul/li");
foreach (XPathNavigator item in iterator)
{
var title = HtmlDocumentHelper.GetNodeValue(item, ".//div[@class='jDesc']/a/text()");
var href = HtmlDocumentHelper.GetNodeValue(item, ".//div[@class='jDesc']//@href");
if (string.IsNullOrEmpty(title))
{
title = HtmlDocumentHelper.GetNodeValue(item, ".//div[@class='jTitle']/a/text()");
href = HtmlDocumentHelper.GetNodeValue(item, ".//div[@class='jTitle']//@href");
}
var imgSrc = HtmlDocumentHelper.GetNodeValue(item, ".//div[@class='jPic']//@original");
var skuMatchResults = Regex.Match(href, @"(?<=/)\d+(?=\.html)");
var sku = skuMatchResults.Success ? skuMatchResults.Value : string.Empty;
if (string.IsNullOrEmpty(sku))
{
continue;
}
// 评价数据
var comments = ParseComments(item);
IResut resut = new Resut();
resut[SkuIdKey] = sku;
resut["ShopId"] = ShopUrl;
resut["ProductName"] = title;
resut["ProductUrl"] = href;
resut["ProductImage"] = imgSrc;
resut["ProductComments"] = comments;
resultList.Add(resut);
}
if (!listOnly)
{
this.UpdateResultsPrices(resultList, SkuIdKey);
}
return(resultList.ToArray());
}
/// <summary>
/// 解析当前页的所有产品信息
/// </summary>
/// <param name="htmlSource"></param>
/// <param name="listOnly"></param>
/// <returns></returns>
private IResut[] ParseCurrentItems(string htmlSource, bool listOnly = false)
{
const string SkuIdKey = "ProductSku";
var resultList = new List <IResut>();
// 返回xpath查询器
var navigator = HtmlDocumentHelper.CreateNavigator(htmlSource);
var iterator = navigator.Select(@"//ul/li");
foreach (XPathNavigator item in iterator)
{
var title = HtmlDocumentHelper.GetNodeValue(item, ".//div[@class='jDesc']/a/text()");
var href = HtmlDocumentHelper.GetNodeValue(item, ".//div[@class='jDesc']//@href");
if (string.IsNullOrEmpty(title))
{
title = HtmlDocumentHelper.GetNodeValue(item, ".//div[@class='jTitle']/a/text()");
href = HtmlDocumentHelper.GetNodeValue(item, ".//div[@class='jTitle']//@href");
}
//HtmlDocumentHelper.GetNodeValue(item,".//div[@class='jPic']//@original")
var imgSrc = HtmlDocumentHelper.GetNodeValue(item, ".//div[@class='jPic']//@original");
if (imgSrc.Equals(string.Empty))
{
imgSrc = HtmlDocumentHelper.GetNodeValue(htmlSource, ".//div[@class='jPic']//@src");
}
var skuMatchResults = Regex.Match(href, @"(?<=/)\d+(?=.html)");
var sku = skuMatchResults.Success ? skuMatchResults.Value : string.Empty;
if (string.IsNullOrEmpty(sku))
{
continue;
}
// 评价数据
var comments = ParseComments(item);
IResut resut = new Resut();
resut[SkuIdKey] = sku;
resut["ShopUrl"] = this.ShopUrl;
resut["ProductName"] = title;
resut["ProductUrl"] = href;
resut["ProductImage"] = imgSrc;
resut["ProductComments"] = comments;
resultList.Add(resut);
}
if (!listOnly)
{
this.UpdateResultsPrices(resultList, SkuIdKey);
}
return(resultList.ToArray());
}
/// <summary>
/// 开始解析
/// </summary>
/// <param name="webContent">Content of the web.</param>
/// <returns></returns>
public IResut Parse(string webContent)
{
IResut resut = new Resut();
var navigator = HtmlDocumentHelper.CreateNavigator(webContent);
ParseShopScoreResult(resut, navigator);
throw new NotImplementedException();
}
/* /// <summary>
* /// 返回自营店的店铺编号列表
* /// </summary>
* /// <param name="shopDictionary">The shop dictionary.</param>
* /// <returns></returns>
* private string[] GetSelfSupportShopIds(IDictionary<string, IResut> shopDictionary)
* {
* var shopIdList = new List<string>();
*
* foreach (var shopItem in shopDictionary)
* {
* var shopId = shopItem.Key;
* if (shopItem.Value.GetStringValue(@"vender_type") == "0")
* {
* shopIdList.Add(shopId);
* }
* }
*
* return shopIdList.ToArray();
* }*/
/// <summary>
/// Parses the identifier from xpath.
/// </summary>
/// <param name="htmlSource">The HTML source.</param>
/// <param name="xpath">The xpath.</param>
/// <returns></returns>
private string[] ParseIdFromXpath(string htmlSource, string xpath)
{
var navigator = HtmlDocumentHelper.CreateNavigator(htmlSource);
var iterator = navigator.Select(xpath);
var xItems = HtmlDocumentHelper.CopyNodeToArray(iterator);
var pids = Array.ConvertAll(xItems, item => item.Value);
return(DictionaryHelper.Distinct(pids));
}
private string BuildAjaxSearchUrl(string webContent, IDictionary <string, string> renderStructure)
{
var matchResults = Regex.Match(webContent, "(?<=var params = ){[^}]+}");
if (!matchResults.Success)
{
throw new NotSupportedException("无法从页面中解析出搜索参数");
}
var jObject = JObject.Parse(matchResults.Value);
var navigator = HtmlDocumentHelper.CreateNavigator(webContent);
//System.Func<string, string> readJsonFunc = key => JsonHelper.TryReadJobjectValue(jObject, key,(string)null);
//System.Func<string, string> readHtmlFunc = key =>
//{
// string value;
// renderStructure.TryGetValue(key,out value);
// return value;
//};
//System.Func<string, string> readInputFunc = key =>
//HtmlDocumentHelper.GetNodeValue(navigator,$@"//input[@id='{key}']/@value");
var collection = Url.CreateQueryCollection();
collection[@"appId"] = ReadJsonFunc(jObject, "appId");
collection[@"orderBy"] = "5";
collection[@"pageNo"] = "1";
collection[@"direction"] = "1";
collection[@"categoryId"] = ReadJsonFunc(jObject, @"categoryId");
collection[@"pageSize"] = @"24";
collection[@"pagePrototypeId"] = ReadJsonFunc(jObject, @"pagePrototypeId");
collection[@"pageInstanceId"] = ReadHtmlFunc(renderStructure, @"m_render_pageInstance_id");
collection[@"moduleInstanceId"] = ReadHtmlFunc(renderStructure, "m_render_instance_id");
collection[@"prototypeId"] = ReadHtmlFunc(renderStructure, @"m_render_prototype_id");
collection[@"templateId"] = ReadHtmlFunc(renderStructure, @"m_render_template_id");
collection[@"layoutInstanceId"] = ReadHtmlFunc(renderStructure, @"m_render_layout_instance_id");
collection[@"origin"] = ReadHtmlFunc(renderStructure, @"m_render_origin");
collection[@"shopId"] = ReadInputFunc(navigator, @"shop_id");
collection[@"verderId"] = ReadInputFunc(navigator, @"vender_id");
collection[@"_"] = $"{JsCodeHelper.GetDateTime()}";
var baseUrl = renderStructure[@"m_render_is_search"] == "true"
? @"http://module-jshop.jd.com/module/getModuleHtml.html"
: @"http://mall.jd.com/view/getModuleHtml.html";
return(Url.CombinUrl(baseUrl, collection));
}
/// <summary>
/// 构造搜索 ajax url
/// </summary>
/// <param name="webContent">Content of the web.</param>
/// <param name="renderStructure">The render structure.</param>
/// <returns></returns>
/// <exception cref="System.NotSupportedException">$无法从页面中解析出搜索参数</exception>
private string BuildAjaxSearchUrl(string webContent, IDictionary <string, string> renderStructure)
{
/*webContent = File.ReadAllText(@"C:\Users\sinoX\Desktop\京东搜索页.html");*/
var matchResults = Regex.Match(webContent, "(?<=var params = ){[^}]+}");
if (!matchResults.Success)
{
throw new NotSupportedException("无法从页面中解析出搜索参数");
}
// {"appId":"435517","orderBy":"5","direction":"0","categoryId":"0","pageSize":"24","venderId":"1000004373","isGlobalSearch":"0","maxPrice":"0","pagePrototypeId":"17","pageNo":"1","shopId":"1000004373","minPrice":"0"}
var jObject = JObject.Parse(matchResults.Value);
var navigator = HtmlDocumentHelper.CreateNavigator(webContent);
System.Func <string, string> readJsonFunc = key => JsonHelper.TryReadJobjectValue(jObject, key, (string)null);
System.Func <string, string> readHtmlFunc = key =>
{
string value;
renderStructure.TryGetValue(key, out value);
return(value);
};
System.Func <string, string> readInputFunc =
key => HtmlDocumentHelper.GetNodeValue(navigator, $@"//input[@id='{key}']/@value");
var collection = Url.CreateQueryCollection();
collection[@"appId"] = readJsonFunc("appId");
collection[@"orderBy"] = "5";
collection[@"pageNo"] = "1";
collection[@"direction"] = "1";
collection[@"categoryId"] = readJsonFunc(@"categoryId");
collection[@"pageSize"] = @"24";
collection[@"pagePrototypeId"] = readJsonFunc(@"pagePrototypeId");
collection[@"pageInstanceId"] = readHtmlFunc(@"m_render_pageInstance_id");
collection[@"moduleInstanceId"] = readHtmlFunc(@"m_render_instance_id");
collection[@"prototypeId"] = readHtmlFunc(@"m_render_prototype_id");
collection[@"templateId"] = readHtmlFunc(@"m_render_template_id");
collection[@"layoutInstanceId"] = readHtmlFunc(@"m_render_layout_instance_id");
collection[@"origin"] = readHtmlFunc(@"m_render_origin");
collection[@"shopId"] = readInputFunc(@"shop_id");
collection[@"venderId"] = readInputFunc(@"vender_id");
/*collection[@"callback"] = @"jshop_module_render_callback"; // 不用这个直接返回一个 json 结构 */
collection[@"_"] = $"{JsCodeHelper.GetDateTime()}";
var baseUrl = renderStructure[@"m_render_is_search"] == "true"
? @"http://module-jshop.jd.com/module/getModuleHtml.html"
: @"http://mall.jd.com/view/getModuleHtml.html";
return(Url.CombinUrl(baseUrl, collection));
}
/// <summary>
/// 处理特例
/// </summary>
/// <param name="shopUrl"></param>
/// <returns></returns>
private string GetSpecialSearchPageContent(string shopUrl)
{
var webContent = this.GetWebContent(shopUrl);
var pageAppId = HtmlDocumentHelper.GetNodeValue(webContent, @"//input[@id='pageInstance_appId']/@value");
var vender_id = HtmlDocumentHelper.GetNodeValue(webContent, @"//input[@id='vender_id']/@value");
var shop_id = HtmlDocumentHelper.GetNodeValue(webContent, @"//input[@id='shop_id']/@value");
var searchUrl = $"http://mall.jd.com/advance_search-{pageAppId}-{vender_id}-{shop_id}-5-0-0-1-1-24.html";
this.CurrentUrl = searchUrl;
return(this.GetWebContent(searchUrl));
}
private void GetPersonPageUrls(string localDir, string listPageUrl, List <Dictionary <string, string> > allPersonPageUrlInfos, string keyWords)
{
try
{
string listPageLocalPath = this.RunPage.GetFilePath(listPageUrl, localDir);
HtmlAgilityPack.HtmlDocument pageHtmlDoc = HtmlDocumentHelper.Load(listPageLocalPath);
HtmlNodeCollection allDivNodes = pageHtmlDoc.DocumentNode.SelectNodes("//div[@class=\"result c-container \"]");
foreach (HtmlNode divNode in allDivNodes)
{
string linkedinUrlPart = HtmlDocumentHelper.TryGetNodeInnerText(divNode, "./div[@class=\"f13\"]/a", true, true, null, null);
if (linkedinUrlPart == null)
{
linkedinUrlPart = HtmlDocumentHelper.TryGetNodeInnerText(divNode, "./div/div[@class=\"f13\"]/a", true, true, null, null);
}
string abstractText = HtmlDocumentHelper.TryGetNodeInnerText(divNode, true, true, null, null);
if (linkedinUrlPart != null && linkedinUrlPart.Contains(".linkedin.com/in/") && abstractText != null && abstractText.ToLower().Contains(keyWords.ToLower()))
{
try
{
string personName = HtmlDocumentHelper.TryGetNodeInnerText(divNode, "./h3/a", true, true, null, null);
string personUrl = HtmlDocumentHelper.TryGetNodeAttributeValue(divNode, "./h3/a", "href", true, true, null, null);
foreach (string postfix in this.BaiduLinkedinItemPostfix)
{
personName = personName.Replace(postfix, "").Trim();
}
Dictionary <string, string> personPageUrlInfo = new Dictionary <string, string>();
personPageUrlInfo.Add("personUrl", personUrl);
personPageUrlInfo.Add("personName", personName.Trim());
allPersonPageUrlInfos.Add(personPageUrlInfo);
}
catch (Exception ex)
{
throw new Exception("获取个人网页地址时出错", ex);
}
}
}
}
catch (Exception ex)
{
this.RunPage.InvokeAppendLogText(ex.Message, LogLevelType.Error, true);
throw new Exception("解析Baidu列表页出错, listPageUrl = +" + listPageUrl, ex);
}
}
private async Task <HttpResponseMessage> GetInitializeResponse(HttpClient client, object initializeRequestBody)
{
// Arrange csrf info
var loginResponse = await client.GetAsync("/BankIdAuthentication/Login?returnUrl=%2F&loginOptions=X&orderRef=Y");
var loginCookies = loginResponse.Headers.GetValues("set-cookie");
var document = await HtmlDocumentHelper.FromContent(loginResponse.Content);
var csrfToken = document.GetRequestVerificationToken();
// Arrange acting request
var initializeRequest = new JsonContent(initializeRequestBody);
initializeRequest.Headers.Add("Cookie", loginCookies);
initializeRequest.Headers.Add("RequestVerificationToken", csrfToken);
return(await client.PostAsync("/BankIdAuthentication/Api/Initialize", initializeRequest));
}
private void _handleHeaderBodyFooter(string path, string htmlContent, string idFooter, string idHeader = null)
{
_pathTempSaveFile = path;
var htmlDocumentHelper = new HtmlDocumentHelper(htmlContent);
_headerContent = null;
if (!string.IsNullOrEmpty(_url))
{
htmlDocumentHelper.MergeUrlOnImage(_url);
}
if (!string.IsNullOrEmpty(idFooter))
{
_footerContent = htmlDocumentHelper.GetContentById(idFooter);
}
_bodyContent = htmlDocumentHelper.RemoveContentById(idFooter);
}
/// <summary>
/// 解析出总共的页数
/// </summary>
/// <param name="content"></param>
/// <returns></returns>
private int ParseAmountPage(string content)
{
if (content == null)
{
return(-1);
}
var navigator = HtmlDocumentHelper.CreateNavigator(content);
var node = navigator.SelectSingleNode(@"//div[@class='jPage']/em");
if (node == null)
{
return(-1);
}
var matchResults = Regex.Match(node.Value, @"(?<=共)\d+(?=条记录)");
return(matchResults.Success ? int.Parse(matchResults.Value) : -1);
}
private void GetPersonPageUrls(string localDir, string listPageUrl, List <Dictionary <string, string> > allPersonPageUrlInfos)
{
try
{
string listPageLocalPath = this.RunPage.GetFilePath(listPageUrl, localDir);
HtmlAgilityPack.HtmlDocument pageHtmlDoc = HtmlDocumentHelper.Load(listPageLocalPath);
HtmlNodeCollection allANodes = pageHtmlDoc.DocumentNode.SelectNodes("//div[@class=\"rc\"]/h3/a");
foreach (HtmlNode aNode in allANodes)
{
string personUrl = HtmlDocumentHelper.TryGetNodeAttributeValue(aNode, "data-href", true, true, null, null);
if (personUrl == null)
{
personUrl = HtmlDocumentHelper.TryGetNodeAttributeValue(aNode, "href", true, true, null, null);
}
if (personUrl.Contains(".linkedin.com/in/"))
{
try
{
string personName = aNode.InnerText.Trim();
foreach (string postfix in this.GoogleLinkedinItemPostfix)
{
personName = personName.Replace(postfix, "");
}
Dictionary <string, string> personPageUrlInfo = new Dictionary <string, string>();
personUrl = CommonUtil.UrlDecode(personUrl);
personPageUrlInfo.Add("personUrl", personUrl);
personPageUrlInfo.Add("personName", personName.Trim());
allPersonPageUrlInfos.Add(personPageUrlInfo);
}
catch (Exception ex)
{
throw new Exception("获取个人网页地址时出错", ex);
}
}
}
}
catch (Exception ex)
{
this.RunPage.InvokeAppendLogText(ex.Message, LogLevelType.Error, true);
throw new Exception("解析Google列表页出错, listPageUrl = +" + listPageUrl, ex);
}
}
/// <summary>
///解析评论人数以及评价
/// </summary>
/// <param name="item"></param>
/// <returns></returns>
private static string ParseComments(XPathNavigator item)
{
var matchResults = Regex.Match(item.Value, @"\d+(?= *人评价)");
if (matchResults.Success)
{
return(matchResults.Value);
}
// 这里还没有找到例子
var commentsNode = HtmlDocumentHelper.GetNodeValue(item, ".//span[@class='evaluate']");
matchResults = Regex.Match(commentsNode, @"(?<=\()\d+(?=\))");
if (matchResults.Success)
{
return(matchResults.Value);
}
return("-1");
}
/// <summary>
/// 同步考次(考试时间)
/// </summary>
/// <returns></returns>
private List <DictionaryModel> SysTimes()
{
List <DictionaryModel> datas = new List <DictionaryModel>();
var options = new HttpClientOptions();
options.URL = AppHelper.UrlApplyPage;
options.Method = "GET";
options.CookieCollection = CurrentCookies;
var result = new HttpWebClientUtility().Request(options);
if (!VerifyHelper.IsEmpty(result.Content))
{
var doc = HtmlDocumentHelper.Load(result.Content);
if (doc != null)
{
var timesNode = HtmlDocumentHelper.FindChildNodes(doc, AppHelper.XPathTimes);
if (timesNode != null)
{
foreach (var item in timesNode.Where(x => x.OriginalName == "a"))
{
var valueAttribute = item.Attributes["attrval"];
if (valueAttribute != null)
{
datas.Add(new DictionaryModel()
{
Id = Guid.NewGuid(),
Genre = AppConst.DictionaryTimes,
Name = StringHelper.Get(item.InnerText),
Value = StringHelper.Get(valueAttribute.Value),
Parent = "",
Sort = 0
});
}
}
}
}
}
return(datas);
}
/// <summary>
/// 返回搜索页面的内容
/// </summary>
/// <param name="shopUrl"></param>
/// <returns></returns>
private string GetSearchPageContent(string shopUrl)
{
var webContent = this.GetWebContent(shopUrl);
// 取出<input type="hidden" value="504028" id="pageInstance_appId"/>中的value
var pageAppId = HtmlDocumentHelper.GetNodeValue(webContent, @"//input[@id='pageInstance_appId']/@value");
/*
* // view_search-店铺页面编号-0-排序类型-排序方向-每页条数-页码.html
* 排序类型： 5 销量 4 价格 3 收藏 2 时间
* 每页条数： 最大 24
* 排序方向： 1 从大到时小 0 从小到大
* 页码： 从 1 开始
* 查找pageInstance_appId找到value的值
* http://mall.jd.com//view_search-337310-0-5-0-24-5.html
*/
var searchUrl = $"http://mall.jd.com/view_search-{pageAppId}-0-5-0-24-1.html";
return(this.GetWebContent(searchUrl));
}
public async Task ParseAmazon()
{
try
{
var helper = new RequestHelper();
var data = await helper.SendRequestAsync(LinkAmazon, headers : HeaderBuilder.GetDefaultHeaders(), automaticDecompression : true);
if (!data.Contains("Currently unavailable"))
{
var document = new HtmlDocument();
document.LoadHtml(data);
var text = HtmlDocumentHelper.GetNodeByParams(document.DocumentNode, HtmlTag.div, HtmlAttribute.id, "centerCol").InnerText.Replace("\n", " ");
_emailHelper.SendEmail(text);
}
else
{
File.AppendAllText(@"D:\playstationLog.log", $"{DateTime.Now}: Not available in Amazon{Environment.NewLine}");
}
}
catch (Exception e)
{
File.AppendAllText(@"D:\playstationLog.log", $"{DateTime.Now}: {e}{Environment.NewLine}");
}
}
public static async Task <List <DataModelBase> > Start(string link)
{
var result = new List <DataModelBase>();
try
{
Driver.Navigate().GoToUrl(string.IsNullOrEmpty(link) ? Link : link);
var doc = new HtmlDocument();
doc.LoadHtml(Driver.PageSource);
var nodes = doc.DocumentNode;
var pageNumber = Convert.ToInt32(HtmlDocumentHelper.GetNodeByParams(nodes, "nav", "class", "be-pagination u-margin-t-large").ChildNodes[4].InnerText);
// iterate through every page
try
{
for (var i = 1; i <= pageNumber; i++)
{
var sb = new StringBuilder(); // list of ids
var listPerPage = new List <DataModel>(); // list of lots per page
Driver.Navigate().GoToUrl(Link + $"?page={i}");
doc.LoadHtml(Driver.PageSource);
var lots = HtmlDocumentHelper.GetNodeByParams(doc.DocumentNode, "div", "class", "be-lot-list gallery");
foreach (var lot in lots.ChildNodes)
{
try
{
var name = HtmlDocumentHelper.GetNodeByParams(lot, "h2", "class", "c-card__title be-lot__title").InnerText;
var url = HtmlDocumentHelper.GetNodeByParams(lot, "a", "class", "c-card").GetAttributeValue("href", null);
var id = new Uri(url).Segments[2].Split('-')[0];
listPerPage.Add(new DataModel {
Name = name, Url = url, CurrentBid = int.Parse(id)
});
sb.Append($",{id}");
}
catch
{
//ignore
}
}
var ids = sb.ToString().TrimStart(',');
var helper = new RequestHelper();
var response = await helper.SendRequestAsync($"{JsonUrl}?ids={ids}", automaticDecompression : true, headers : HeaderBuilder.GetDefaultHeaders());
var list = JsonConvert.DeserializeObject <JsonResult>(response);
foreach (var item in listPerPage)
{
try
{
var correspondentItem = list.Lots.FirstOrDefault(x => x.id == item.CurrentBid);
item.BiddingEndTime = correspondentItem.bidding_end_time;
var amount = correspondentItem.current_bid_amount;
item.CurrentBidAmount = amount.EUR;
item.ReservedPrice = correspondentItem.reserve_price_met.HasValue && correspondentItem.reserve_price_met.Value ? "Has reserve price" : "No reserve price";
result.Add(item);
}
catch
{
//ignore
}
}
;
}
}
catch
{
//ignore
}
}
catch (Exception e)
{
//ignore
}
return(result);
}
public async Task Cancel_Calls_CancelApi()
{
// Arrange mocks
var autoLaunchOptions = new BankIdLoginOptions(new List <string>(), null, false, true, false, false, string.Empty, DefaultStateCookieName);
var mockProtector = new Mock <IBankIdLoginOptionsProtector>();
mockProtector
.Setup(protector => protector.Unprotect(It.IsAny <string>()))
.Returns(autoLaunchOptions);
var testBankIdApi = new TestBankIdApi(new BankIdSimulatedApiClient());
using var server = CreateServer(
o =>
{
o.UseSimulatedEnvironment().AddSameDevice();
o.AuthenticationBuilder.Services.AddTransient <IBankIdLauncher, TestBankIdLauncher>();
},
DefaultAppConfiguration(async context =>
{
await context.ChallengeAsync(BankIdDefaults.SameDeviceAuthenticationScheme);
}),
services =>
{
services.AddTransient(s => mockProtector.Object);
services.AddSingleton <IBankIdApiClient>(s => testBankIdApi);
});
// Arrange csrf info
var loginRequest = CreateRequestWithStateCookie(server, "/BankIdAuthentication/Login?returnUrl=%2F&loginOptions=X&orderRef=Y");
var loginResponse = await loginRequest.GetAsync();
var loginCookies = loginResponse.Headers.GetValues("set-cookie");
var document = await HtmlDocumentHelper.FromContent(loginResponse.Content);
var csrfToken = document.GetRequestVerificationToken();
// Arrange acting request
var testReturnUrl = "/TestReturnUrl";
var testOptions = "TestOptions";
var initializeRequest = new JsonContent(new { returnUrl = testReturnUrl, loginOptions = testOptions });
initializeRequest.Headers.Add("Cookie", loginCookies);
initializeRequest.Headers.Add("RequestVerificationToken", csrfToken);
// Act
var client = server.CreateClient();
var initializeTransaction = await client.PostAsync("/BankIdAuthentication/Api/Initialize", initializeRequest);
var initializeResponseContent = await initializeTransaction.Content.ReadAsStringAsync();
var initializeObject = JsonConvert.DeserializeAnonymousType(initializeResponseContent, new { RedirectUri = "", OrderRef = "", IsAutoLaunch = false });
var cancelRequest = new JsonContent(new
{
orderRef = initializeObject.OrderRef,
loginOptions = "TestOptions",
cancelReturnUrl = "/"
});
cancelRequest.Headers.Add("Cookie", loginCookies);
cancelRequest.Headers.Add("RequestVerificationToken", csrfToken);
// Act
var cancelTransaction = await client.PostAsync("/BankIdAuthentication/Api/Cancel", cancelRequest);
// Assert
Assert.Equal(HttpStatusCode.OK, cancelTransaction.StatusCode);
Assert.True(testBankIdApi.CancelAsyncIsCalled);
}
private static async Task Parse()
{
var requestHelper = new RequestHelper();
var header = HeaderBuilder.BuildOwnHeaders(new HeaderModel()
{
Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
Host = "gamevaluenow.com",
Referer = "https://gamevaluenow.com/",
User_Agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36"
});
var html = await requestHelper.SendRequestAsync(PageUrl, headers : HeaderBuilder.GetDefaultHeaders());
var doc = new HtmlDocument();
doc.LoadHtml(html);
var result = new List <GameValueNowModel>();
var nodes = HtmlDocumentHelper.GetNodesByParamsUseXpathStartsWith(doc.DocumentNode, HtmlTag.a, HtmlAttribute._class, "brand-");
foreach (var node in nodes)
{
try
{
var href = node.GetAttributeValue("href", null);
if (href is null || href == "#")
{
continue;
}
var model = new GameValueNowModel();
var url = $"{PageUrl}{href}";
var name = node.InnerText;
model.URL = url;
model.PlatformName = WebUtility.HtmlDecode(name);
if (result.Select(x => x.PlatformName).Contains(model.PlatformName))
{
continue;
}
result.Add(model);
}
catch
{
//ignore;
}
}
foreach (var item in result)
{
try
{
item.Data = new List <Data>();
var document = new HtmlDocument();
var dataHtml = await requestHelper.SendRequestAsync(item.URL, headers : header);
document.LoadHtml(dataHtml);
// stats
var statsNode = HtmlDocumentHelper.GetNodeByParams(document.DocumentNode, HtmlTag.div, HtmlAttribute.id, "stats");
var statListNode = HtmlDocumentHelper.GetNodesByParams(statsNode, HtmlTag.div, HtmlAttribute._class, "col-100 stat");
var avgLoosePrice = statListNode.FirstOrDefault(x => x.InnerText.Contains("Avg Loose"));
var avgCompletePrice = statListNode.FirstOrDefault(x => x.InnerText.Contains("Avg Complete"));
var looseSetValue = statListNode.FirstOrDefault(x => x.InnerText.Contains("Loose Set"));
var completeSetValue = statListNode.FirstOrDefault(x => x.InnerText.Contains("Complete Set"));
var sharpOfGames = statListNode.FirstOrDefault(x => x.InnerText.Contains("#"));
if (avgLoosePrice != null)
{
item.AvgLoosePrice = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(avgLoosePrice, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim();
}
if (avgCompletePrice != null)
{
item.AvgCompletePrice = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(avgCompletePrice, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim();
}
if (looseSetValue != null)
{
item.LooseSetValue = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(looseSetValue, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim();
}
if (completeSetValue != null)
{
item.CompleteSetValue = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(completeSetValue, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim();
}
if (sharpOfGames != null)
{
item.SharpOfGames = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(sharpOfGames, HtmlTag.div, HtmlAttribute._class, "stat-value")?.InnerText?.Trim();
}
// items
var listNode =
HtmlDocumentHelper.GetNodeByParams(document.DocumentNode, HtmlTag.div, HtmlAttribute.id, "item-list");
var collectionItemNodes = HtmlDocumentHelper.GetNodesByParamsUseXpathStartsWith(listNode, HtmlTag.div,
HtmlAttribute._class, "item-row desktop all");
foreach (var collectionItemNode in collectionItemNodes)
{
try
{
var data = new Data();
var name = HtmlDocumentHelper
.GetNodeByParams(collectionItemNode, HtmlTag.a, HtmlAttribute._class, "game-link").InnerText;
var id = HtmlDocumentHelper
.GetNodeByParams(collectionItemNode, HtmlTag.div, HtmlAttribute._class, "item-number").InnerText
.Replace("\n", "").Replace(" ", "");
data.Title = name;
data.Id = id;
var priceContainer = HtmlDocumentHelper.GetNodeByParamsUseXpathContains(collectionItemNode, HtmlTag.div, HtmlAttribute._class, "price-col-container");
if (priceContainer is null)
{
continue;
}
var prices = HtmlDocumentHelper.GetNodesByParamsUseXpathStartsWith(priceContainer, HtmlTag.a, HtmlAttribute._class, "game-link");
if (prices is null)
{
continue;
}
var priceCount = prices.Count;
if (priceCount > 0)
{
data.Loose = prices[0].InnerText;
}
if (priceCount > 1)
{
data.Complete = prices[1].InnerText;
}
if (priceCount > 2)
{
data.New = prices[2].InnerText;
}
if (priceCount > 3)
{
data.Graded = prices[3].InnerText;
}
data.PlatformName = item.PlatformName;
item.Data.Add(data);
}
catch
{
//ignore
}
}
}
catch (Exception e)
{
//ignore
}
await Task.Delay(500);
}
await SaveToJsonFile(result);
//await SaveToSql(result);
}
/// <summary>
/// GetShopDetail
/// </summary>
/// <param name="listSheet"></param>
/// <param name="pageSourceDir"></param>
/// <param name="resultEW"></param>
private void GetShopDetail(IListSheet listSheet, string pageSourceDir, ExcelWriter resultEW)
{
Dictionary <string, string> shopDic = new Dictionary <string, string>();
for (int i = 0; i < listSheet.RowCount; i++)
{
//listSheet中只有一条记录
Dictionary <string, string> row = listSheet.GetRow(i);
bool giveUp = "Y".Equals(row[SysConfig.GiveUpGrabFieldName]);
if (!giveUp)
{
string pageUrl = row[SysConfig.DetailPageUrlFieldName];
string provinceName = row["provinceName"];
string cityCode = row["cityCode"];
string cityName = row["cityName"];
string shopCode = row["shopCode"];
string shopName = row["shopName"];
string serviceTime = "";
string tel = "";
string address = "";
Nullable <decimal> lng = null;
Nullable <decimal> lat = null;
string serviceItems = "";
string localFilePath = this.RunPage.GetFilePath(pageUrl, pageSourceDir);
HtmlAgilityPack.HtmlDocument htmlDoc = this.RunPage.GetLocalHtmlDocument(listSheet, i);
HtmlNodeCollection allInfoNameNodes = htmlDoc.DocumentNode.SelectNodes("//div[@class=\"shopInfo\"]/dl/dt");
if (allInfoNameNodes != null)
{
foreach (HtmlNode infoNameNode in allInfoNameNodes)
{
string infoName = infoNameNode.InnerText;
if (infoName.StartsWith("服务时间"))
{
HtmlNode infoNode = HtmlDocumentHelper.GetNextNode(infoNameNode, "dd");
if (infoNode != null)
{
serviceTime = infoNode.InnerText;
}
}
/*
* else if (infoName.StartsWith("服务地址"))
* {
* HtmlNode infoNode = HtmlDocumentHelper.GetNextNode(infoNameNode, "dd");
* if (infoNode != null)
* {
* HtmlNode infoSpanNode = infoNode.SelectSingleNode("./span");
* if (infoSpanNode != null)
* {
* address = infoSpanNode.InnerText;
* }
* }
* }
* else if (infoName.StartsWith("服务电话"))
* {
* HtmlNode infoNode = HtmlDocumentHelper.GetNextNode(infoNameNode, "dd");
* if (infoNode != null)
* {
* HtmlNode infoSpanNode = infoNode.SelectSingleNode("./span");
* if (infoSpanNode != null)
* {
* tel = infoSpanNode.InnerText;
* }
* }
* }
*/
}
}
HtmlNodeCollection allScriptNodes = htmlDoc.DocumentNode.SelectNodes("//script");
if (allScriptNodes != null)
{
foreach (HtmlNode scriptNode in allScriptNodes)
{
string script = scriptNode.InnerText;
if (script.Contains("var lng = \""))
{
int lngBeginIndex = script.IndexOf("var lng = \"") + 11;
int lngEndIndex = script.IndexOf("\";", lngBeginIndex);
int latBeginIndex = script.IndexOf("var lat = \"") + 11;
int latEndIndex = script.IndexOf("\";", latBeginIndex);
lng = decimal.Parse(script.Substring(lngBeginIndex, lngEndIndex - lngBeginIndex));
lat = decimal.Parse(script.Substring(latBeginIndex, latEndIndex - latBeginIndex));
break;
}
}
}
HtmlNode telNode = htmlDoc.DocumentNode.SelectSingleNode("//input[@id=\"HiddenStrPhone\"]");
if (telNode != null)
{
tel = telNode.Attributes["value"].Value;
}
HtmlNode keyWordNode = htmlDoc.DocumentNode.SelectSingleNode("//meta[@name='keywords']");
string keyWord = keyWordNode.Attributes["content"].Value;
string[] kws = keyWord.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries);
address = kws[1];
StringBuilder serviceItemSB = new StringBuilder();
HtmlNodeCollection allServiceGroupNodes = htmlDoc.DocumentNode.SelectNodes("//div[@class=\"shopProject\"]/table/tbody/tr");
if (allServiceGroupNodes != null)
{
for (int j = 0; j < allServiceGroupNodes.Count; j++)
{
HtmlNode serviceGroupNode = allServiceGroupNodes[j];
HtmlNodeCollection allServiceNodes = serviceGroupNode.SelectNodes("./td");
foreach (HtmlNode serviceNode in allServiceNodes)
{
string serviceText = serviceNode.InnerText.Trim();
if (serviceText.EndsWith("："))
{
serviceItemSB.Append(serviceText);
}
else
{
serviceItemSB.Append(serviceText + ";");
}
}
}
serviceItems = serviceItemSB.ToString();
}
Dictionary <string, object> f2vs = new Dictionary <string, object>();
f2vs.Add("provinceName", provinceName);
f2vs.Add("cityCode", cityCode);
f2vs.Add("cityName", cityName);
f2vs.Add("shopCode", shopCode);
f2vs.Add("shopName", shopName);
f2vs.Add("serviceTime", serviceTime);
f2vs.Add("address", address);
f2vs.Add("lng", lng);
f2vs.Add("lat", lat);
f2vs.Add("tel", tel);
f2vs.Add("serviceItems", serviceItems);
resultEW.AddRow(f2vs);
}
}
}
private static async Task Parse()
{
var html = await GetData(PageUrl);
var doc = new HtmlDocument();
doc.LoadHtml(html);
var result = new List <PriceChartingModel>();
var brands = HtmlDocumentHelper.GetNodesByParams(doc.DocumentNode, HtmlTag.li, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "brand").Where(x => x.GetAttributeValue("id", null) != null && x.GetAttributeValue("id", null) != "tools");
foreach (var brand in brands)
{
try
{
var dropDownMenu = HtmlDocumentHelper.GetNodeByParamsUseXpathStartsWith(brand, HtmlTag.ul, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "menu-dropdown");
if (dropDownMenu != null)
{
var regions = HtmlDocumentHelper.GetNodesByParams(dropDownMenu, HtmlTag.ul, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "region");
if (regions != null)
{
foreach (var region in regions)
{
var regionName = HtmlDocumentHelper.GetNodeByParams(region, HtmlTag.li, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "title")?.InnerText;
var categories = region.SelectNodes(".//li").Where(x => x.InnerHtml != string.Empty);
foreach (var category in categories)
{
if (category.ChildNodes[0] == null || category.ChildNodes[0].GetAttributeValue("href", null) == null)
{
continue;
}
var url = category.ChildNodes[0].GetAttributeValue("href", null);
var name = category.ChildNodes[0].InnerText;
result.Add(new PriceChartingModel
{
Region = regionName,
CategoryName = name,
URL = url
});
}
}
}
else
{
var categories = brand.SelectNodes(".//li").Where(x => x.InnerHtml != string.Empty);
foreach (var category in categories)
{
if (category.ChildNodes[0]?.GetAttributeValue("href", null) == null)
{
continue;
}
var url = category.ChildNodes[0].GetAttributeValue("href", null);
var name = category.ChildNodes[0].InnerText;
result.Add(new PriceChartingModel
{
Region = "NTSC (USA)",
CategoryName = name,
URL = url
});
}
}
}
}
catch { }
}
foreach (var category in result)
{
try
{
var categoryData = new List <Data>();
category.Data = new List <Data>();
var response = new List <Product>();
var pageCount = 0;
while (true)
{
try
{
var jUrl = PageUrlJson.Replace("{category}", category.URL.Split('/').Last()).Replace("{count}", pageCount.ToString());
var dataJson = await GetData(jUrl);
var jsonObj = JsonConvert.DeserializeObject <Response>(dataJson);
response.AddRange(jsonObj.products);
if (jsonObj.products.Length == 50)
{
pageCount += 50;
Thread.Sleep(100);
continue;
}
break;
}
catch
{
}
}
foreach (var item in response)
{
categoryData.Add(new Data
{
Title = item.productName,
LoosePrice = item.price1,
CIBPrice = item.price3,
NewPrice = item.price2
});
}
category.Data = categoryData;
}
catch (Exception ex)
{
}
// details
foreach (var data in category.Data)
{
try
{
var url = $"{category.URL.Replace("console", "game")}/{ExtensionMethods.Replace(data.Title, new char[] { '[', ']', '(', ')', '/', '\\', '.', ':', ',', '?' }, " ").Replace(' ', '-')}";
Thread.Sleep(200);
html = await GetData(url);
doc = new HtmlDocument();
doc.LoadHtml(html);
var attributeNode = HtmlDocumentHelper.GetNodeByParams(doc.DocumentNode, HtmlTag.table, Scrapping.AllPossibilities.Enums.HtmlAttribute.id, "attribute");
if (attributeNode != null)
{
var detailNodes = attributeNode.SelectNodes(".//tr");
var genre = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Genre:");
var releaseDate = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Release Date:");
var rating = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "ESRB Rating:");
var publisher = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Publisher:");
var developer = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Developer:");
var playerCount = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "Player Count:");
var upc = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "UPC:");
var asin = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "ASIN (Amazon):");
var epid = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "ePID (eBay):");
var priceChartingId = detailNodes.FirstOrDefault(x => x.SelectSingleNode(".//td")?.InnerText == "PriceCharting ID:");
if (genre != null)
{
data.Genre = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(genre, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (releaseDate != null)
{
data.ReleaseDate = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(releaseDate, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (rating != null)
{
data.Rating = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(rating, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (publisher != null)
{
data.Publisher = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(publisher, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (developer != null)
{
data.Developer = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(developer, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (playerCount != null)
{
data.PlayerCount = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(playerCount, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (upc != null)
{
data.UPC = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(upc, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (asin != null)
{
data.Amazon_ASIN = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(asin, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (epid != null)
{
data.Ebay_ePID = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(epid, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
if (priceChartingId != null)
{
data.PriceChartingId = WebUtility.HtmlDecode(HtmlDocumentHelper.GetNodeByParams(priceChartingId, HtmlTag.td, Scrapping.AllPossibilities.Enums.HtmlAttribute._class, "details")?.InnerText?.Trim(' ', '\n'));
}
}
}
catch
{
}
}
await WriteJson(JsonConvert.SerializeObject(category, Formatting.Indented), category.CategoryName);
await Task.Delay(500);
}
await WriteJson(JsonConvert.SerializeObject(result, Formatting.Indented));
}
