/// <summary>
/// 刷新 淘宝h5 SDK 的cookie
/// </summary>
public void RefreshH5Api_Cookies()
{
try
{
// 生成时间戳
string timestamp = JavascriptContext.getUnixTimestamp();
//加载Cookie
var ckVisitor = new LazyCookieVistor();
///淘宝域下 的cookie
string etao_appkey = "12574478";
var data = "{\"isSec\":0}";//固定数据 只为刷token
var sign_getToken = JavascriptContext.getEtaoJSSDKSign("", timestamp, etao_appkey, data);
string h5TokenUrl = string.Format("https://api.m.taobao.com/h5/mtop.user.getusersimple/1.0/?appKey={0}&t={1}&sign={2}&api=mtop.user.getUserSimple&v=1.0&H5Request=true&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data={3}", etao_appkey, timestamp, sign_getToken, data);
this.AutoRefeshCookie(h5TokenUrl);//从新刷新页面 获取 服务器颁发的私钥
var cks_taobao = ckVisitor.LoadCookies(TaobaoSiteUrl);
var _m_h5_tk_cookie = cks_taobao.FirstOrDefault(x => x.Name == "_m_h5_tk");
if (null == _m_h5_tk_cookie)
{
throw new Exception("未能成功刷新 淘宝h5 SDK! _m_h5_tk");
}
}
catch (Exception ex)
{
Logger.Error(ex);
}
}
public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
{
string keyWord = queryParas.KeyWord;
if (string.IsNullOrEmpty(keyWord))
{
return(string.Empty);
}
//加载Cookie
var ckVisitor = new LazyCookieVistor();
var cks = ckVisitor.LoadCookies(GuomeiSiteUrl);
//优先使用格式化好的查询地址
string searchUrl = "";
if (null != queryParas.ResolvedUrl)
{
searchUrl = queryParas.ResolvedUrl.Url;
}
else
{
searchUrl = string.Format(templateOfSearchUrl, keyWord);
}
var client = GuomeiHttpClient;
//设置Host
client.Client.DefaultRequestHeaders.Host = new Uri(searchUrl).Host;
//设置跳转头 Referrer
string enCodeKeyword = HttpUtility.UrlEncode(keyWord);
client.Client.DefaultRequestHeaders.Referrer = new Uri(string.Format(templateOfSearchUrl, keyWord));
////加载cookies
////获取当前站点的Cookie
client.ChangeGlobleCookies(cks, GuomeiSiteUrl);
// 4 发送请求
var clientProxy = new HttpServerProxy()
{
Client = client.Client, KeepAlive = true
};
//注意：对于响应的内容 不要使用内置的文本 工具打开，这个工具有bug.看到的文本不全面
//使用json格式打开即可看到里面所有的字符串
string content = clientProxy.GetRequestTransfer(searchUrl, null);
return(content);
}
public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
{
string keyWord = queryParas.KeyWord;
if (string.IsNullOrEmpty(keyWord))
{
return(string.Empty);
}
//加载Cookie
var ckVisitor = new LazyCookieVistor();
var cks = ckVisitor.LoadCookies(tmallSiteUrl);
//天猫应该是有解析的搜索url的，如果没有，那么使用基于拼接的默认关键词的检索地址
string searchUrl = "";
if (null != queryParas.ResolvedUrl)
{
searchUrl = queryParas.ResolvedUrl.Url;
}
else
{
string sortValue = "s";//综合排序
if (null != queryParas.OrderFiled)
{
sortValue = queryParas.OrderFiled.FieldValue;
}
searchUrl = string.Format(templateOfSearchUrl, keyWord, sortValue);
}
var client = TmallHttpClient;
////加载cookies
////获取当前站点的Cookie
client.ChangeGlobleCookies(cks, tmallSiteUrl);
// 4 发送请求
var clientProxy = new HttpServerProxy()
{
Client = client.Client, KeepAlive = true
};
//注意：对于响应的内容 不要使用内置的文本 工具打开，这个工具有bug.看到的文本不全面
//使用json格式打开即可看到里面所有的字符串
string content = clientProxy.GetRequestTransfer(searchUrl, null);
return(content);
}
public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
{
string keyWord = queryParas.KeyWord;
if (string.IsNullOrEmpty(keyWord))
{
return(string.Empty);
}
//加载Cookie
var ckVisitor = new LazyCookieVistor();
var cks = ckVisitor.LoadCookies(PddSiteUrl);
//优先使用格式化好的查询地址
string searchUrl = "";
if (null != queryParas.ResolvedUrl)
{
searchUrl = queryParas.ResolvedUrl.Url;
}
else
{
searchUrl = string.Format(templateOfSearchUrl
, keyWord, queryParas.PageIndex + 1
, queryParas.OrderFiled.FieldValue);
}
var client = PddHttpClient;
//设置跳转头 Referrer
client.Client.DefaultRequestHeaders.Referrer = new Uri(string.Format("http://mobile.yangkeduo.com/search_result.html?search_key={0}&search_src=new&search_met_track=manual&refer_page_name=search&refer_page_id=search_1515546776782_Zeq7a7qvHN&refer_page_sn=10031&page_id=search_result_1515546785426_9C9j8sIOep&sort_type=2", keyWord));
////加载cookies
////获取当前站点的Cookie
client.ChangeGlobleCookies(cks, PddSiteUrl);
// 4 发送请求
var clientProxy = new HttpServerProxy()
{
Client = client.Client, KeepAlive = true
};
//注意：对于响应的内容 不要使用内置的文本 工具打开，这个工具有bug.看到的文本不全面
//使用json格式打开即可看到里面所有的字符串
string content = clientProxy.GetRequestTransfer(searchUrl, null);
return(content);
}
/// <summary>
/// 加载Cookies
/// </summary>
protected void LoadCookies()
{
if (null == this.DomainIdentity || this.DomainIdentity.Length <= 0)
{
return;
}
var ckVistor = new LazyCookieVistor();
_PageCooies = ckVistor.LoadCookies(this.DomainIdentity);
}
public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
{
string keyWord = queryParas.KeyWord;
if (string.IsNullOrEmpty(keyWord))
{
return(string.Empty);
}
//加载Cookie
var ckVisitor = new LazyCookieVistor();
var cks = ckVisitor.LoadCookies(MogujieSiteUrl);
string timeToken = JavascriptContext.getUnixTimestamp();
//优先使用格式化好的查询地址
string searchUrl = "";
if (null != queryParas.ResolvedUrl)
{
searchUrl = queryParas.ResolvedUrl.Url;
}
else
{
searchUrl = string.Format(templateOfSearchUrl, timeToken, keyWord);
}
var client = MogujieHttpClient;
client.Client.DefaultRequestHeaders.Referrer = new Uri(string.Format("http://list.mogujie.com/s?q={0}&ptp=1.eW5XD.0.0.qJUTT", keyWord));
////加载cookies
////获取当前站点的Cookie
client.ChangeGlobleCookies(cks, MogujieSiteUrl);
// 4 发送请求
var clientProxy = new HttpServerProxy()
{
Client = client.Client, KeepAlive = true
};
//注意：对于响应的内容 不要使用内置的文本 工具打开，这个工具有bug.看到的文本不全面
//使用json格式打开即可看到里面所有的字符串
string content = clientProxy.GetRequestTransfer(searchUrl, null);
return(content);
}
public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
{
string keyWord = queryParas.KeyWord;
if (string.IsNullOrEmpty(keyWord))
{
return(string.Empty);
}
//加载Cookie
var ckVisitor = new LazyCookieVistor();
var cks = ckVisitor.LoadCookies(QingTaokeSiteUrl);
string searchUrl = string.Format(templateOfSearchUrl, keyWord);
var client = QingTaokeHttpClient;
//设置跳转头 Referrer
//client.Client.DefaultRequestHeaders.Referrer = new Uri(string.Format("http://mobile.yangkeduo.com/search_result.html?search_key={0}&search_src=new&refer_page_name=search&refer_page_id=search_1500439537429_yr7sonlWB0", keyWord));
////加载cookies
////获取当前站点的Cookie
client.ChangeGlobleCookies(cks, QingTaokeSiteUrl);
// 4 发送请求
var clientProxy = new HttpServerProxy()
{
Client = client.Client, KeepAlive = true
};
//注意：对于响应的内容 不要使用内置的文本 工具打开，这个工具有bug.看到的文本不全面
//使用json格式打开即可看到里面所有的字符串
string content = clientProxy.GetRequestTransfer(searchUrl, null);
return(content);
}
public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
{
string keyWord = queryParas.KeyWord;
if (string.IsNullOrEmpty(keyWord))
{
return(string.Empty);
}
//加载Cookie
var ckVisitor = new LazyCookieVistor();
var cks = ckVisitor.LoadCookies(SuningSiteUrl);
//优先使用格式化好的查询地址
string searchUrl = "";
if (null != queryParas.ResolvedUrl)
{
searchUrl = queryParas.ResolvedUrl.Url;
}
else
{
searchUrl = string.Format(templateOfSearchUrl, keyWord);
}
var client = SuningHttpClient;
//设置Host
client.Client.DefaultRequestHeaders.Host = new Uri(searchUrl).Host;
//设置跳转头 Referrer
string enCodeKeyword = HttpUtility.UrlEncode(keyWord);
client.Client.DefaultRequestHeaders.Referrer = new Uri(string.Format(templateOfSearchUrl, keyWord));
//如果是ajax 请求 那么附带这两个头信息
if (searchUrl.Contains("searchProductList.do"))
{
client.Client.DefaultRequestHeaders.Add("X-Requested-With", "XMLHttpRequest");
client.Client.DefaultRequestHeaders.Add("X-Tingyun-Id", "p35OnrDoP8k;r=57681924");
}
////加载cookies
////获取当前站点的Cookie
client.ChangeGlobleCookies(cks, SuningSiteUrl);
// 4 发送请求
var clientProxy = new HttpServerProxy()
{
Client = client.Client, KeepAlive = true
};
//注意：对于响应的内容 不要使用内置的文本 工具打开，这个工具有bug.看到的文本不全面
//使用json格式打开即可看到里面所有的字符串
string content = clientProxy.GetRequestTransfer(searchUrl, null);
return(content);
}
/// <summary>
/// 加载淘宝 h5 webapi 优惠券查询地址-详细
/// 注意： 访问过于频繁并发的时候会被 ban 掉；以后设计为N个淘宝账户使用不同 的cookie进行抓取
/// </summary>
/// <param name="sellerId"></param>
/// <param name="activityId"></param>
/// <returns></returns>
public async Task <string> LoadH5Api_YouhuiquanDetailAsync(long sellerId, string activityId)
{
try
{
if (string.IsNullOrEmpty(activityId))
{
return(null);
}
//生成时间戳
string timestamp = JavascriptContext.getUnixTimestamp();
//加载Cookie
var ckVisitor = new LazyCookieVistor();
///淘宝域下 的cookie
var cks_taobao = ckVisitor.LoadCookies(TaobaoSiteUrl);
string etao_appkey = "12574478";
//淘宝sdk 下的token
//List<Cookie> cks_h5 = ckVisitor.LoadCookies(TaobaoH5WebApiUrl);
var _m_h5_tk_cookie = cks_taobao.FirstOrDefault(x => x.Name == "_m_h5_tk");
string _m_h5_tk_valueString = string.Empty;
if (null == _m_h5_tk_cookie)
{
this.RefreshH5Api_Cookies();
cks_taobao = ckVisitor.LoadCookies(TaobaoSiteUrl);
_m_h5_tk_cookie = cks_taobao.FirstOrDefault(x => x.Name == "_m_h5_tk");
}
if (null == _m_h5_tk_cookie || string.IsNullOrEmpty(_m_h5_tk_cookie.Value))
{
throw new Exception("加载授权私钥失败！无法获取对应的cookie:_m_h5_tk ");
}
_m_h5_tk_valueString = _m_h5_tk_cookie.Value.Split('_')[0];
string paras = string.Concat("{\"uuid\":\"", activityId, "\",\"sellerId\":\"", sellerId, "\",\"queryShop\":true}");
string sign = JavascriptContext.getEtaoJSSDKSign(_m_h5_tk_valueString, timestamp, etao_appkey, paras);
string apiUrl = string.Format("https://api.m.taobao.com/h5/mtop.taobao.couponmtopreadservice.findshopbonusactivitys/2.0/?appKey={0}&t={1}&sign={2}&api=mtop.taobao.couponMtopReadService.findShopBonusActivitys&v=2.0&AntiFlood=false&ecode=0&type=jsonp&dataType=jsonp&callback=mtopjsonp2&data={3}", etao_appkey, timestamp, sign, paras);
var client = TaobaoH5ApiHttpClient;
////加载cookies
////获取当前站点的Cookie
client.ChangeGlobleCookies(cks_taobao, TaobaoH5WebApiUrl);
//修改client 的refer 头
client.Client.DefaultRequestHeaders.Referrer = new Uri(apiUrl);
// 4 发送请求
var clientProxy = new HttpServerProxy()
{
Client = client.Client, KeepAlive = true
};
var taskMsg = await clientProxy.GetResponseTransferAsync(apiUrl, null);
string content = await taskMsg.Content.ReadAsStringAsync();
if (!string.IsNullOrEmpty(content) && content.Contains("FAIL_SYS_USER_VALIDATE"))
{
//需要身份验证的时候 ;FAIL_SYS_USER_VALIDATE::哎哟喂,被挤爆啦,请稍后重试; 被踢的时候 如何处理？？
//清除taobao 的cookie 从客户端的rediscookie 从新加载
//ckVisitor.DeleteCookies(TaobaoSiteUrl);
int x = 0;
}
return(content);
}
catch (Exception ex)
{
Logger.Error(ex);
}
return(string.Empty);
}
public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
{
VipSearchResultBag resultBag = new VipSearchResultBag();
string keyWord = queryParas.KeyWord;
if (string.IsNullOrEmpty(keyWord))
{
return(string.Empty);
}
try
{
//加载Cookie
var ckVisitor = new LazyCookieVistor();
var cks = ckVisitor.LoadCookies(VipSiteUrl);
string searchUrl = string.Format(templateOfSearchUrl, keyWord);
var client = VipHttpClient;
client.Client.DefaultRequestHeaders.Referrer = new Uri(string.Format("https://m.vip.com/searchlist.html?q={0}&channel_id=", keyWord));
////加载cookies
////获取当前站点的Cookie
client.ChangeGlobleCookies(cks, VipSiteUrl);
// 4 发送请求
var clientProxy = new HttpServerProxy()
{
Client = client.Client, KeepAlive = true
};
//注意：对于响应的内容 不要使用内置的文本 工具打开，这个工具有bug.看到的文本不全面
//使用json格式打开即可看到里面所有的字符串
Task <HttpResponseMessage> brandTask;
Task <HttpResponseMessage> categoryTreeTask;
Task <HttpResponseMessage> searchListTask;
string para_brandJson = "";
string para_categoryTreeJson = "";
string para_searchListJson = "";
if (null != queryParas.ResolvedUrl && null != queryParas.ResolvedUrl.ParasPost)
{
para_brandJson = queryParas.ResolvedUrl.ParasPost["para_brand"].ToString();
para_categoryTreeJson = queryParas.ResolvedUrl.ParasPost["para_categoryTree"].ToString();
para_searchListJson = queryParas.ResolvedUrl.ParasPost["para_searchList"].ToString();
}
else
{
para_brandJson = new VipSearchParaBrand(keyWord).ToJson();
para_categoryTreeJson = new VipSearchParaCategoryTree(keyWord).ToJson();
//插件不解析的话，那么使用最简单的基本关键词过滤分页，不进行复杂过滤，复杂过滤筛选应该在插件实现
var paraDetais = new VipSearchParaSearchList(keyWord);
//分页
paraDetais.paramsDetails.np = queryParas.PageIndex + 1;
para_searchListJson = paraDetais.ToJson();
}
if (queryParas.IsNeedResolveHeaderTags == true)
{
//1 查询品牌
var brandPara = new Dictionary <string, string>();
brandPara.Add("para_brand", para_brandJson);
brandTask = clientProxy.PostRequestTransferAsync(queryBrandUrl, PostDataContentType.Json, brandPara, null);
// 2 查询分类
var categoryTreePara = new Dictionary <string, string>();
categoryTreePara.Add("para_categoryTree", para_categoryTreeJson);
categoryTreeTask = clientProxy.PostRequestTransferAsync(queryCategoryUrl, PostDataContentType.Json, categoryTreePara, null);
}
else
{
brandTask = Task.FromResult <HttpResponseMessage>(null);
categoryTreeTask = Task.FromResult <HttpResponseMessage>(null);
}
//3检索内容
var searchListPara = new Dictionary <string, string>();
searchListPara.Add("para_searchList", para_searchListJson);
searchListTask = clientProxy.PostRequestTransferAsync(templateOfSearchUrl, PostDataContentType.Json, searchListPara, null);
//等待任务完毕
Task.WaitAll(new Task[] { brandTask, categoryTreeTask, searchListTask });
if (brandTask.Result != null)
{
resultBag.BrandStoreList = brandTask.Result.Content.ReadAsStringAsync().Result;
}
if (categoryTreeTask.Result != null)
{
resultBag.CategoryTree = categoryTreeTask.Result.Content.ReadAsStringAsync().Result;
}
if (searchListTask.Result != null)
{
resultBag.SearchList = searchListTask.Result.Content.ReadAsStringAsync().Result;
}
}
catch (Exception ex)
{
throw ex;
}
return(resultBag.ToJson());
}
public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
{
string keyWord = queryParas.KeyWord;
if (string.IsNullOrEmpty(keyWord))
{
return(string.Empty);
}
//生成时间戳
string timestamp = JavascriptContext.getUnixTimestamp();
//加载Cookie
var ckVisitor = new LazyCookieVistor();
var cks = ckVisitor.LoadCookies(eTaoSiteUrl);
var _m_h5_tk_cookie = cks.FirstOrDefault(x => x.Name == "_m_h5_tk");
if (null == _m_h5_tk_cookie)
{
this.AutoRefeshCookie(this.RefreshCookieUrl);//从新刷新页面 获取 服务器颁发的私钥
cks = ckVisitor.LoadCookies(eTaoSiteUrl);
_m_h5_tk_cookie = cks.FirstOrDefault(x => x.Name == "_m_h5_tk");
}
if (null == _m_h5_tk_cookie || string.IsNullOrEmpty(_m_h5_tk_cookie.Value))
{
throw new Exception("加载授权私钥失败！无法获取对应的cookie:_m_h5_tk ");
}
string _m_h5_tk_valueString = _m_h5_tk_cookie.Value.Split('_')[0];
string etao_appkey = "12574478";
if (null == queryParas.OrderFiled)
{
queryParas.OrderFiled = queryParas.GetCurrentPlatformSupportOrderFields().First();
}
int pageNum = queryParas.PageIndex * 40;
string paras = string.Concat("{\"s\":", pageNum, ",\"n\":40,\"q\":\"", keyWord, "\",\"needEncode\":false,\"sort\":\"", queryParas.OrderFiled.FieldValue, "\",\"maxPrice\":10000000,\"minPrice\":0,\"serviceList\":\"\",\"navigator\":\"all\",\"urlType\":2}");
//生成参数戳
string sign = JavascriptContext.getEtaoJSSDKSign(_m_h5_tk_valueString, timestamp, etao_appkey, paras);
string url = string.Format("https://apie.m.etao.com/h5/mtop.etao.fe.search/1.0/?type=jsonp&api=mtop.etao.fe.search&v=1.0&appKey=12574478&data={0}&t={1}&sign={2}&callback=jsonp28861232595120323", paras, timestamp, sign);
string searchUrl = string.Format(templateOfSearchUrl, keyWord);
var client = etaoHttpClient;
////加载cookies
////获取当前站点的Cookie
client.ChangeGlobleCookies(cks, eTaoSiteUrl);
//修改client 的refer 头
client.Client.DefaultRequestHeaders.Referrer = new Uri(searchUrl);
// 4 发送请求
var clientProxy = new HttpServerProxy()
{
Client = client.Client, KeepAlive = true
};
string content = clientProxy.GetRequestTransfer(url, null);
return(content);
}
