///// <summary> ///// GetWebContent ///// </summary> ///// <param name="url"></param> ///// <param name="cookies"></param> ///// <returns></returns> //private string GetWebContent(string url,ref string cookies) //{ // MyHttpHelper myHttpHelper = new MyHttpHelper(); // HttpItem httpItem = new HttpItem // { // URL = url, // MaximumAutomaticRedirections = 10, // Timeout = 60000, // Allowautoredirect = false, // UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" // }; // if (!string.IsNullOrEmpty(Cookies)) // httpItem.Cookie = cookies; // HttpResult httpResult = myHttpHelper.GetHtml(httpItem); // if (string.IsNullOrEmpty(cookies)) // { // cookies = httpResult.Cookie; // CookieCollection cookie = httpResult.CookieCollection; // } // return httpResult.Html; //} /// <summary> /// GetMainWebContent /// </summary> /// <param name="nextUrl"></param> /// <param name="postData"></param> /// <param name="cookies"></param> /// <param name="currentUrl"></param> /// <returns></returns> protected override string GetMainWebContent(string nextUrl, byte[] postData, ref string cookies, string currentUrl) { WebRequestCtrl.GetWebContentParam @default = WebRequestCtrl.GetWebContentParam.Default; @default.Refere = currentUrl; @default.MaxRedirect = 10; @default.TimeOut = 60000; cookies = cookies ?? string.Empty; string html = string.Empty; bool isSuccess = false; while (!isSuccess) { try { html = WebRequestCtrl.GetWebContent(nextUrl, postData, ref cookies, 1, @default); } catch (Exception e) { Console.WriteLine(e.ToString()); if (e.ToString().Contains("操作超时") || e.ToString().Contains("操作已超时")) { continue; } } isSuccess = true; } return(html); }
/// <summary> /// GetMainWebContent /// </summary> /// <param name="nextUrl"></param> /// <param name="postData"></param> /// <param name="cookies"></param> /// <param name="currentUrl"></param> /// <returns></returns> protected override string GetMainWebContent(string nextUrl, byte[] postData, ref string cookies, string currentUrl) { WebRequestCtrl.GetWebContentParam @default = WebRequestCtrl.GetWebContentParam.Default; @default.Refere = currentUrl; @default.MaxRedirect = 20; cookies = cookies ?? string.Empty; return(WebRequestCtrl.GetWebContent(nextUrl, postData, ref cookies, 1, @default)); }
/* /// <summary> * /// 获取用户数据 * /// </summary> * /// <param name="shopId">The shop identifier.</param> * /// <returns></returns> * private IResut GetCompanyInfo(string shopId) * { * IResut resut; * companyInfoDictionary.TryGetValue(shopId, out resut); * if (resut != null) * { * return resut; * } * * resut = this.GetCompanyInfoOnline(shopId); * companyInfoDictionary[shopId] = resut; * * return resut; * }*/ /* /// <summary> * /// 在线获取用户公司信息 * /// </summary> * /// <param name="shopId">The shop identifier.</param> * /// <returns></returns> * private IResut GetCompanyInfoOnline(string shopId) * { * var url = $"http://mall.jd.com/shopLevel-{shopId}.html"; * var cookies = this.Cookies ?? string.Empty; * var webContent = WebRequestCtrl.GetWebContent(url, null, ref cookies, 2); * * var parser = new CompanyInfoParser(); * return parser.Parse(webContent); * }*/ /* /// <summary> * /// 获取公司信息 * /// </summary> * /// <param name="shopIds">The shop ids.</param> * /// <returns></returns> * private IDictionary<string, IResut> GetCompanyInfos(string[] shopIds) * { * IDictionary<string, IResut> resultDictionary = new Dictionary<string, IResut>(); * foreach (var shopId in shopIds) * { * var resut = this.GetCompanyInfo(shopId); * resultDictionary[shopId] = resut; * } * * return resultDictionary; * }*/ /// <summary> /// 获取店铺信息 /// </summary> /// <returns></returns> private IDictionary <string, IResut> GetListShopItemInfo() { var htmlSource = this.HtmlSource; IDictionary <string, IResut> resultDictionary = new Dictionary <string, IResut>(); var shopIds = this.ParseShopIds(htmlSource); if (shopIds.Length == 0) { return(resultDictionary); } // http://search.jd.com/shop_new.php?ids=1000004042,1000000922,1000015485,1000015427,1000010664 var collection = Url.CreateQueryCollection(); collection.Add(@"ids", string.Join(",", shopIds)); var url = Url.CombinUrl(@"http://search.jd.com/shop_new.php", collection); var cookies = this.Cookies ?? string.Empty; var param = WebRequestCtrl.GetWebContentParam.Default; param.Refere = this.CurrentUrl; var webContent = WebRequestCtrl.GetWebContent(url, null, ref cookies, 2, param); this.TryUpdateHasMoreFlag(webContent); var keyword = this.InnerParam.Keyword; var page = this.CurrentPage; var rank = 0; var jArray = JArray.Parse(webContent); foreach (var jToken in jArray) { rank++; var shopId = jToken[@"shop_id"]?.Value <string>(); if (shopId == null) { continue; } var jObject = (JObject)jToken; var result = this.ConvertToResult(jObject); result[@"ShopUrl"] = $"http://mall.jd.com/index-{shopId}.html"; result[@"SearchKeyword"] = keyword; this.SetResultSearchPageRank(result, rank); this.SetResultSearchPageIndex(result, page); resultDictionary[shopId] = result; } return(resultDictionary); }
/// <summary> /// 返回指定 url 的页面内容 /// </summary> /// <param name="url">The URL.</param> /// <param name="postData">The post data.</param> /// <param name="cookies">The cookies.</param> /// <param name="refere">The refere.</param> /// <param name="isAjax">是不是ajax请求.</param> /// <returns></returns> protected string GetWebContent( string url, byte[] postData, ref string cookies, string refere = "", bool isAjax = false) { var param = WebRequestCtrl.GetWebContentParam.Default; param.Refere = refere; param.IsAjax = isAjax; cookies = cookies ?? string.Empty; var webContent = WebRequestCtrl.GetWebContent(url, null, ref cookies, 1, param); return(webContent); }