///// <summary>
///// GetWebContent
///// </summary>
///// <param name="url"></param>
///// <param name="cookies"></param>
///// <returns></returns>
//private string GetWebContent(string url,ref string cookies)
//{
// MyHttpHelper myHttpHelper = new MyHttpHelper();
// HttpItem httpItem = new HttpItem
// {
// URL = url,
// MaximumAutomaticRedirections = 10,
// Timeout = 60000,
// Allowautoredirect = false,
// UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"
// };
// if (!string.IsNullOrEmpty(Cookies))
// httpItem.Cookie = cookies;
// HttpResult httpResult = myHttpHelper.GetHtml(httpItem);
// if (string.IsNullOrEmpty(cookies))
// {
// cookies = httpResult.Cookie;
// CookieCollection cookie = httpResult.CookieCollection;
// }
// return httpResult.Html;
//}
/// <summary>
/// GetMainWebContent
/// </summary>
/// <param name="nextUrl"></param>
/// <param name="postData"></param>
/// <param name="cookies"></param>
/// <param name="currentUrl"></param>
/// <returns></returns>
protected override string GetMainWebContent(string nextUrl, byte[] postData, ref string cookies, string currentUrl)
{
WebRequestCtrl.GetWebContentParam @default = WebRequestCtrl.GetWebContentParam.Default;
@default.Refere = currentUrl;
@default.MaxRedirect = 10;
@default.TimeOut = 60000;
cookies = cookies ?? string.Empty;
string html = string.Empty;
bool isSuccess = false;
while (!isSuccess)
{
try
{
html = WebRequestCtrl.GetWebContent(nextUrl, postData, ref cookies, 1, @default);
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
if (e.ToString().Contains("操作超时") || e.ToString().Contains("操作已超时"))
{
continue;
}
}
isSuccess = true;
}
return(html);
}
/// <summary>
/// GetMainWebContent
/// </summary>
/// <param name="nextUrl"></param>
/// <param name="postData"></param>
/// <param name="cookies"></param>
/// <param name="currentUrl"></param>
/// <returns></returns>
protected override string GetMainWebContent(string nextUrl, byte[] postData, ref string cookies, string currentUrl)
{
WebRequestCtrl.GetWebContentParam @default = WebRequestCtrl.GetWebContentParam.Default;
@default.Refere = currentUrl;
@default.MaxRedirect = 20;
cookies = cookies ?? string.Empty;
return(WebRequestCtrl.GetWebContent(nextUrl, postData, ref cookies, 1, @default));
}
/* /// <summary>
* /// 获取用户数据
* /// </summary>
* /// <param name="shopId">The shop identifier.</param>
* /// <returns></returns>
* private IResut GetCompanyInfo(string shopId)
* {
* IResut resut;
* companyInfoDictionary.TryGetValue(shopId, out resut);
* if (resut != null)
* {
* return resut;
* }
*
* resut = this.GetCompanyInfoOnline(shopId);
* companyInfoDictionary[shopId] = resut;
*
* return resut;
* }*/
/* /// <summary>
* /// 在线获取用户公司信息
* /// </summary>
* /// <param name="shopId">The shop identifier.</param>
* /// <returns></returns>
* private IResut GetCompanyInfoOnline(string shopId)
* {
* var url = $"http://mall.jd.com/shopLevel-{shopId}.html";
* var cookies = this.Cookies ?? string.Empty;
* var webContent = WebRequestCtrl.GetWebContent(url, null, ref cookies, 2);
*
* var parser = new CompanyInfoParser();
* return parser.Parse(webContent);
* }*/
/* /// <summary>
* /// 获取公司信息
* /// </summary>
* /// <param name="shopIds">The shop ids.</param>
* /// <returns></returns>
* private IDictionary<string, IResut> GetCompanyInfos(string[] shopIds)
* {
* IDictionary<string, IResut> resultDictionary = new Dictionary<string, IResut>();
* foreach (var shopId in shopIds)
* {
* var resut = this.GetCompanyInfo(shopId);
* resultDictionary[shopId] = resut;
* }
*
* return resultDictionary;
* }*/
/// <summary>
/// 获取店铺信息
/// </summary>
/// <returns></returns>
private IDictionary <string, IResut> GetListShopItemInfo()
{
var htmlSource = this.HtmlSource;
IDictionary <string, IResut> resultDictionary = new Dictionary <string, IResut>();
var shopIds = this.ParseShopIds(htmlSource);
if (shopIds.Length == 0)
{
return(resultDictionary);
}
// http://search.jd.com/shop_new.php?ids=1000004042,1000000922,1000015485,1000015427,1000010664
var collection = Url.CreateQueryCollection();
collection.Add(@"ids", string.Join(",", shopIds));
var url = Url.CombinUrl(@"http://search.jd.com/shop_new.php", collection);
var cookies = this.Cookies ?? string.Empty;
var param = WebRequestCtrl.GetWebContentParam.Default;
param.Refere = this.CurrentUrl;
var webContent = WebRequestCtrl.GetWebContent(url, null, ref cookies, 2, param);
this.TryUpdateHasMoreFlag(webContent);
var keyword = this.InnerParam.Keyword;
var page = this.CurrentPage;
var rank = 0;
var jArray = JArray.Parse(webContent);
foreach (var jToken in jArray)
{
rank++;
var shopId = jToken[@"shop_id"]?.Value <string>();
if (shopId == null)
{
continue;
}
var jObject = (JObject)jToken;
var result = this.ConvertToResult(jObject);
result[@"ShopUrl"] = $"http://mall.jd.com/index-{shopId}.html";
result[@"SearchKeyword"] = keyword;
this.SetResultSearchPageRank(result, rank);
this.SetResultSearchPageIndex(result, page);
resultDictionary[shopId] = result;
}
return(resultDictionary);
}
/// <summary>
/// 返回指定 url 的页面内容
/// </summary>
/// <param name="url">The URL.</param>
/// <param name="postData">The post data.</param>
/// <param name="cookies">The cookies.</param>
/// <param name="refere">The refere.</param>
/// <param name="isAjax">是不是ajax请求.</param>
/// <returns></returns>
protected string GetWebContent(
string url,
byte[] postData,
ref string cookies,
string refere = "",
bool isAjax = false)
{
var param = WebRequestCtrl.GetWebContentParam.Default;
param.Refere = refere;
param.IsAjax = isAjax;
cookies = cookies ?? string.Empty;
var webContent = WebRequestCtrl.GetWebContent(url, null, ref cookies, 1, param);
return(webContent);
}