Пример #1
0
            public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
            {
                string keyWord = queryParas.KeyWord;

                if (string.IsNullOrEmpty(keyWord))
                {
                    return(string.Empty);
                }


                //加载Cookie
                var ckVisitor = new LazyCookieVistor();
                var cks       = ckVisitor.LoadCookies(GuomeiSiteUrl);



                //优先使用格式化好的查询地址
                string searchUrl = "";

                if (null != queryParas.ResolvedUrl)
                {
                    searchUrl = queryParas.ResolvedUrl.Url;
                }
                else
                {
                    searchUrl = string.Format(templateOfSearchUrl, keyWord);
                }


                var client = GuomeiHttpClient;

                //设置Host
                client.Client.DefaultRequestHeaders.Host = new Uri(searchUrl).Host;

                //设置跳转头 Referrer
                string enCodeKeyword = HttpUtility.UrlEncode(keyWord);

                client.Client.DefaultRequestHeaders.Referrer = new Uri(string.Format(templateOfSearchUrl, keyWord));
                ////加载cookies
                ////获取当前站点的Cookie
                client.ChangeGlobleCookies(cks, GuomeiSiteUrl);

                // 4 发送请求
                var clientProxy = new HttpServerProxy()
                {
                    Client = client.Client, KeepAlive = true
                };

                //注意:对于响应的内容 不要使用内置的文本 工具打开,这个工具有bug.看到的文本不全面
                //使用json格式打开即可看到里面所有的字符串

                string content = clientProxy.GetRequestTransfer(searchUrl, null);



                return(content);
            }
Пример #2
0
            public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
            {
                string keyWord = queryParas.KeyWord;

                if (string.IsNullOrEmpty(keyWord))
                {
                    return(string.Empty);
                }



                //加载Cookie
                var ckVisitor = new LazyCookieVistor();
                var cks       = ckVisitor.LoadCookies(PddSiteUrl);



                //优先使用格式化好的查询地址
                string searchUrl = "";

                if (null != queryParas.ResolvedUrl)
                {
                    searchUrl = queryParas.ResolvedUrl.Url;
                }
                else
                {
                    searchUrl = string.Format(templateOfSearchUrl
                                              , keyWord, queryParas.PageIndex + 1
                                              , queryParas.OrderFiled.FieldValue);
                }



                var client = PddHttpClient;

                //设置跳转头 Referrer
                client.Client.DefaultRequestHeaders.Referrer = new Uri(string.Format("http://mobile.yangkeduo.com/search_result.html?search_key={0}&search_src=new&search_met_track=manual&refer_page_name=search&refer_page_id=search_1515546776782_Zeq7a7qvHN&refer_page_sn=10031&page_id=search_result_1515546785426_9C9j8sIOep&sort_type=2", keyWord));
                ////加载cookies
                ////获取当前站点的Cookie
                client.ChangeGlobleCookies(cks, PddSiteUrl);

                // 4 发送请求
                var clientProxy = new HttpServerProxy()
                {
                    Client = client.Client, KeepAlive = true
                };

                //注意:对于响应的内容 不要使用内置的文本 工具打开,这个工具有bug.看到的文本不全面
                //使用json格式打开即可看到里面所有的字符串

                string content = clientProxy.GetRequestTransfer(searchUrl, null);



                return(content);
            }
Пример #3
0
            public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
            {
                string keyWord = queryParas.KeyWord;

                if (string.IsNullOrEmpty(keyWord))
                {
                    return(string.Empty);
                }


                //加载Cookie
                var ckVisitor = new LazyCookieVistor();
                var cks       = ckVisitor.LoadCookies(tmallSiteUrl);



                //天猫应该是有解析的搜索url的,如果没有,那么使用基于拼接的默认关键词的检索地址
                string searchUrl = "";

                if (null != queryParas.ResolvedUrl)
                {
                    searchUrl = queryParas.ResolvedUrl.Url;
                }
                else
                {
                    string sortValue = "s";//综合排序
                    if (null != queryParas.OrderFiled)
                    {
                        sortValue = queryParas.OrderFiled.FieldValue;
                    }

                    searchUrl = string.Format(templateOfSearchUrl, keyWord, sortValue);
                }


                var client = TmallHttpClient;

                ////加载cookies
                ////获取当前站点的Cookie
                client.ChangeGlobleCookies(cks, tmallSiteUrl);

                // 4 发送请求
                var clientProxy = new HttpServerProxy()
                {
                    Client = client.Client, KeepAlive = true
                };

                //注意:对于响应的内容 不要使用内置的文本 工具打开,这个工具有bug.看到的文本不全面
                //使用json格式打开即可看到里面所有的字符串

                string content = clientProxy.GetRequestTransfer(searchUrl, null);



                return(content);
            }
Пример #4
0
        /// <summary>
        /// 静态构造函数
        /// </summary>
        static TmallAutoMapping()
        {
            //初始化头信息
            var tmallRequestHeaders = BaseAutoMappingWord.GetCommonHttpRequestHeaders();

            tmallRequestHeaders.Add("Referer", "https://www.tmall.com/");
            tmallHttpClient         = new HttpClient();
            tmallHttpClient.Timeout = TimeSpan.FromMilliseconds(2000);
            HttpServerProxy.FormatRequestHeader(tmallHttpClient.DefaultRequestHeaders, tmallRequestHeaders);
        }
        /// <summary>
        /// 静态构造函数 进行一次初始化
        /// </summary>
        static JingdongAutoMaping()
        {
            //初始化头信息
            var jdRequestHeaders = BaseAutoMappingWord.GetCommonHttpRequestHeaders();

            jdRequestHeaders.Add("Referer", "https://www.jd.com/");

            jdHttpClient         = new HttpClient();
            jdHttpClient.Timeout = TimeSpan.FromMilliseconds(2000);
            HttpServerProxy.FormatRequestHeader(jdHttpClient.DefaultRequestHeaders, jdRequestHeaders);
        }
            /// <summary>
            /// 静态构造函数
            /// </summary>
            static VipMixReuestLoader()
            {
                //初始化头信息
                var requestHeaders = BaseRequest.GetCommonRequestHeaders(true);

                requestHeaders.Add("Accept-Encoding", "gzip, deflate");//接受gzip流 减少通信body体积
                requestHeaders.Add("Host", "m.vip.com");
                requestHeaders.Add("Referer", VipSiteUrl);


                VipHttpClient = new CookedHttpClient();
                HttpServerProxy.FormatRequestHeader(VipHttpClient.Client.DefaultRequestHeaders, requestHeaders);
            }
Пример #7
0
            public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
            {
                string keyWord = queryParas.KeyWord;

                if (string.IsNullOrEmpty(keyWord))
                {
                    return(string.Empty);
                }

                //加载Cookie
                var ckVisitor = new LazyCookieVistor();
                var cks       = ckVisitor.LoadCookies(MogujieSiteUrl);

                string timeToken = JavascriptContext.getUnixTimestamp();

                //优先使用格式化好的查询地址
                string searchUrl = "";

                if (null != queryParas.ResolvedUrl)
                {
                    searchUrl = queryParas.ResolvedUrl.Url;
                }
                else
                {
                    searchUrl = string.Format(templateOfSearchUrl, timeToken, keyWord);
                }



                var client = MogujieHttpClient;

                client.Client.DefaultRequestHeaders.Referrer = new Uri(string.Format("http://list.mogujie.com/s?q={0}&ptp=1.eW5XD.0.0.qJUTT", keyWord));
                ////加载cookies
                ////获取当前站点的Cookie
                client.ChangeGlobleCookies(cks, MogujieSiteUrl);

                // 4 发送请求
                var clientProxy = new HttpServerProxy()
                {
                    Client = client.Client, KeepAlive = true
                };

                //注意:对于响应的内容 不要使用内置的文本 工具打开,这个工具有bug.看到的文本不全面
                //使用json格式打开即可看到里面所有的字符串

                string content = clientProxy.GetRequestTransfer(searchUrl, null);



                return(content);
            }
            /// <summary>
            /// 静态构造函数
            /// </summary>
            static TaoUlandMixReuestLoader()
            {
                //静态创建请求客户端
                //TaoUlandHttpClient = new CookiedCefBrowser().BindingHttpClient;

                //初始化头信息
                var requestHeaders = BaseRequest.GetCommonRequestHeaders();

                requestHeaders.Add("Accept-Encoding", "gzip, deflate");//接受gzip流 减少通信body体积
                requestHeaders.Add("upgrade-insecure-requests", "1");
                requestHeaders.Add("Host", "uland.taobao.com");
                TaoUlandHttpClient = new CookedHttpClient();
                HttpServerProxy.FormatRequestHeader(TaoUlandHttpClient.Client.DefaultRequestHeaders, requestHeaders);
            }
Пример #9
0
            /// <summary>
            /// 静态构造函数
            /// </summary>
            static TmallMixReuestLoader()
            {
                //静态创建请求客户端
                //TmallHttpClient = new CookiedCefBrowser().BindingHttpClient;

                //初始化头信息
                var requestHeaders = BaseRequest.GetCommonRequestHeaders();

                requestHeaders.Add("Accept-Encoding", "gzip, deflate");//接受gzip流 减少通信body体积
                requestHeaders.Add("Host", "list.tmall.com");
                //requestHeaders.Add("Referer", TmallSiteUrl);
                TmallHttpClient = new CookedHttpClient();
                HttpServerProxy.FormatRequestHeader(TmallHttpClient.Client.DefaultRequestHeaders, requestHeaders);
            }
            /// <summary>
            /// 静态构造函数
            /// </summary>
            static QingTaokeMixReuestLoader()
            {
                //静态创建请求客户端
                // QingTaokeHttpClient = new CookiedCefBrowser().BindingHttpClient;

                //初始化头信息
                var requestHeaders = BaseRequest.GetCommonRequestHeaders();

                requestHeaders.Add("Accept-Encoding", "gzip, deflate");//接受gzip流 减少通信body体积
                requestHeaders.Add("Host", "www.qingtaoke.com");
                requestHeaders.Add("Upgrade-Insecure-Requests", "1");
                QingTaokeHttpClient = new CookedHttpClient();
                HttpServerProxy.FormatRequestHeader(QingTaokeHttpClient.Client.DefaultRequestHeaders, requestHeaders);
            }
Пример #11
0
            /// <summary>
            /// 静态构造函数
            /// </summary>
            static PddMixReuestLoader()
            {
                //静态创建请求客户端
                // PddHttpClient = new CookiedCefBrowser().BindingHttpClient;

                //初始化头信息
                var requestHeaders = BaseRequest.GetCommonRequestHeaders();

                requestHeaders.Add("Accept-Encoding", "gzip, deflate");//接受gzip流 减少通信body体积
                requestHeaders.Add("Host", "apiv2.yangkeduo.com");
                requestHeaders.Add("Origin", PddSiteUrl);
                PddHttpClient = new CookedHttpClient();
                HttpServerProxy.FormatRequestHeader(PddHttpClient.Client.DefaultRequestHeaders, requestHeaders);
            }
Пример #12
0
            /// <summary>
            /// 静态构造函数
            /// </summary>
            static GuomeiMixReuestLoader()
            {
                //静态创建请求客户端
                //GuomeiHttpClient = new CookiedCefBrowser().BindingHttpClient;

                //初始化头信息
                var requestHeaders = BaseRequest.GetCommonRequestHeaders();

                requestHeaders.Add("Accept-Encoding", "gzip, deflate");//接受gzip流 减少通信body体积
                requestHeaders.Add("Host", "search.gome.com.cn");
                requestHeaders.Add("Referer", GuomeiSiteUrl);
                requestHeaders.Add("Upgrade-Insecure-Requests", "1");
                GuomeiHttpClient = new CookedHttpClient();
                HttpServerProxy.FormatRequestHeader(GuomeiHttpClient.Client.DefaultRequestHeaders, requestHeaders);
            }
Пример #13
0
            /// <summary>
            /// 静态构造函数
            /// </summary>
            static ETaoMixReuestLoader()
            {
                //静态创建请求客户端
                // etaoHttpClient = new CookiedCefBrowser().BindingHttpClient;

                //初始化头信息
                var requestHeaders = BaseRequest.GetCommonRequestHeaders();

                requestHeaders.Add("Accept-Encoding", "gzip, deflate");//接受gzip流 减少通信body体积
                requestHeaders.Add("Host", "apie.m.etao.com");
                requestHeaders.Add("Upgrade-Insecure-Requests", "1");
                //requestHeaders.Add("Referer", eTaoSiteUrl);
                etaoHttpClient = new CookedHttpClient();
                HttpServerProxy.FormatRequestHeader(etaoHttpClient.Client.DefaultRequestHeaders, requestHeaders);
            }
Пример #14
0
        /// <summary>
        /// 发送异步请求
        /// </summary>
        /// <returns></returns>
        protected Task <string> SendHttpRequesntAsync(HttpClient client, CookieContainer cookies = null)
        {
            var clientProxy = new HttpServerProxy()
            {
                Client = client, KeepAlive = true, Cookies = cookies
            };
            var resp = clientProxy.GetResponseTransferAsync(TargetUrl, null).Result;

            if (null != resp && resp.StatusCode != HttpStatusCode.InternalServerError)
            {
                return(resp.Content.ReadAsStringAsync());
            }
            else
            {
                return(null);
            }
        }
            public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
            {
                string keyWord = queryParas.KeyWord;

                if (string.IsNullOrEmpty(keyWord))
                {
                    return(string.Empty);
                }



                //加载Cookie
                var ckVisitor = new LazyCookieVistor();
                var cks       = ckVisitor.LoadCookies(QingTaokeSiteUrl);



                string searchUrl = string.Format(templateOfSearchUrl, keyWord);

                var client = QingTaokeHttpClient;

                //设置跳转头 Referrer
                //client.Client.DefaultRequestHeaders.Referrer = new Uri(string.Format("http://mobile.yangkeduo.com/search_result.html?search_key={0}&search_src=new&refer_page_name=search&refer_page_id=search_1500439537429_yr7sonlWB0", keyWord));
                ////加载cookies
                ////获取当前站点的Cookie
                client.ChangeGlobleCookies(cks, QingTaokeSiteUrl);

                // 4 发送请求
                var clientProxy = new HttpServerProxy()
                {
                    Client = client.Client, KeepAlive = true
                };

                //注意:对于响应的内容 不要使用内置的文本 工具打开,这个工具有bug.看到的文本不全面
                //使用json格式打开即可看到里面所有的字符串

                string content = clientProxy.GetRequestTransfer(searchUrl, null);



                return(content);
            }
            /// <summary>
            /// 静态构造函数
            /// </summary>
            static TaobaoMixReuestLoader()
            {
                //静态创建请求客户端
                //TaobaoHttpClient = new CookiedCefBrowser().BindingHttpClient;

                //初始化头信息
                var requestHeaders = BaseRequest.GetCommonRequestHeaders();

                requestHeaders.Add("Accept-Encoding", "gzip, deflate");//接受gzip流 减少通信body体积
                // requestHeaders.Add("upgrade-insecure-requests", "1");
                requestHeaders.Add("Referer", TaobaoSiteUrl);
                requestHeaders.Add("Host", "s.taobao.com");
                TaobaoHttpClient = new CookedHttpClient();
                HttpServerProxy.FormatRequestHeader(TaobaoHttpClient.Client.DefaultRequestHeaders, requestHeaders);

                ////淘宝h5client
                //var requestHeadersH5 = BaseRequest.GetCommonRequestHeaders();
                //requestHeaders.Add("Accept-Encoding", "gzip, deflate");//接受gzip流 减少通信body体积
                //requestHeaders.Add("upgrade-insecure-requests", "1");
                //requestHeaders.Add("Pragma", "no-cache");
                //requestHeaders.Add("Host", "api.m.taobao.com");
                //TaobaoH5ApiHttpClient = new CookedHttpClient();
                //HttpServerProxy.FormatRequestHeader(TaobaoH5ApiHttpClient.Client.DefaultRequestHeaders, requestHeadersH5);
            }
Пример #17
0
            public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
            {
                string keyWord = queryParas.KeyWord;

                if (string.IsNullOrEmpty(keyWord))
                {
                    return(string.Empty);
                }
                //生成时间戳
                string timestamp = JavascriptContext.getUnixTimestamp();

                //加载Cookie
                var ckVisitor = new LazyCookieVistor();
                var cks       = ckVisitor.LoadCookies(eTaoSiteUrl);

                var _m_h5_tk_cookie = cks.FirstOrDefault(x => x.Name == "_m_h5_tk");

                if (null == _m_h5_tk_cookie)
                {
                    this.AutoRefeshCookie(this.RefreshCookieUrl);//从新刷新页面 获取 服务器颁发的私钥
                    cks             = ckVisitor.LoadCookies(eTaoSiteUrl);
                    _m_h5_tk_cookie = cks.FirstOrDefault(x => x.Name == "_m_h5_tk");
                }
                if (null == _m_h5_tk_cookie || string.IsNullOrEmpty(_m_h5_tk_cookie.Value))
                {
                    throw new Exception("加载授权私钥失败!无法获取对应的cookie:_m_h5_tk ");
                }
                string _m_h5_tk_valueString = _m_h5_tk_cookie.Value.Split('_')[0];

                string etao_appkey = "12574478";

                if (null == queryParas.OrderFiled)
                {
                    queryParas.OrderFiled = queryParas.GetCurrentPlatformSupportOrderFields().First();
                }
                int pageNum = queryParas.PageIndex * 40;

                string paras = string.Concat("{\"s\":", pageNum, ",\"n\":40,\"q\":\"", keyWord, "\",\"needEncode\":false,\"sort\":\"", queryParas.OrderFiled.FieldValue, "\",\"maxPrice\":10000000,\"minPrice\":0,\"serviceList\":\"\",\"navigator\":\"all\",\"urlType\":2}");

                //生成参数戳
                string sign = JavascriptContext.getEtaoJSSDKSign(_m_h5_tk_valueString, timestamp, etao_appkey, paras);

                string url = string.Format("https://apie.m.etao.com/h5/mtop.etao.fe.search/1.0/?type=jsonp&api=mtop.etao.fe.search&v=1.0&appKey=12574478&data={0}&t={1}&sign={2}&callback=jsonp28861232595120323", paras, timestamp, sign);


                string searchUrl = string.Format(templateOfSearchUrl, keyWord);
                var    client    = etaoHttpClient;

                ////加载cookies
                ////获取当前站点的Cookie
                client.ChangeGlobleCookies(cks, eTaoSiteUrl);
                //修改client 的refer 头
                client.Client.DefaultRequestHeaders.Referrer = new Uri(searchUrl);
                // 4 发送请求
                var clientProxy = new HttpServerProxy()
                {
                    Client = client.Client, KeepAlive = true
                };
                string content = clientProxy.GetRequestTransfer(url, null);

                return(content);
            }
Пример #18
0
 public TaobaoAutoMapping(string userAgent)
 {
     HttpServerProxy.ChangeHttpClientUserAgent(taobaoHttpClient, userAgent);
 }
        /// <summary>
        /// 请求使用的Cookies
        /// </summary>
        //private static CookieContainer Cookies;
        /// <summary>
        /// 京东请求需要的头信息
        /// </summary>
        // private  NameValueCollection jdRequestHeaders;

        public JingdongAutoMaping(string userAgent)
        {
            HttpServerProxy.ChangeHttpClientUserAgent(jdHttpClient, userAgent);
        }
Пример #20
0
            public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
            {
                string keyWord = queryParas.KeyWord;

                if (string.IsNullOrEmpty(keyWord))
                {
                    return(string.Empty);
                }



                //加载Cookie
                var ckVisitor = new LazyCookieVistor();
                var cks       = ckVisitor.LoadCookies(SuningSiteUrl);


                //优先使用格式化好的查询地址
                string searchUrl = "";

                if (null != queryParas.ResolvedUrl)
                {
                    searchUrl = queryParas.ResolvedUrl.Url;
                }
                else
                {
                    searchUrl = string.Format(templateOfSearchUrl, keyWord);
                }



                var client = SuningHttpClient;

                //设置Host
                client.Client.DefaultRequestHeaders.Host = new Uri(searchUrl).Host;

                //设置跳转头 Referrer
                string enCodeKeyword = HttpUtility.UrlEncode(keyWord);

                client.Client.DefaultRequestHeaders.Referrer = new Uri(string.Format(templateOfSearchUrl, keyWord));

                //如果是ajax 请求 那么附带这两个头信息
                if (searchUrl.Contains("searchProductList.do"))
                {
                    client.Client.DefaultRequestHeaders.Add("X-Requested-With", "XMLHttpRequest");
                    client.Client.DefaultRequestHeaders.Add("X-Tingyun-Id", "p35OnrDoP8k;r=57681924");
                }
                ////加载cookies
                ////获取当前站点的Cookie
                client.ChangeGlobleCookies(cks, SuningSiteUrl);

                // 4 发送请求
                var clientProxy = new HttpServerProxy()
                {
                    Client = client.Client, KeepAlive = true
                };

                //注意:对于响应的内容 不要使用内置的文本 工具打开,这个工具有bug.看到的文本不全面
                //使用json格式打开即可看到里面所有的字符串

                string content = clientProxy.GetRequestTransfer(searchUrl, null);



                return(content);
            }
            /// <summary>
            /// 加载淘宝 h5 webapi 优惠券查询地址-详细
            /// 注意: 访问过于频繁并发的时候会被 ban 掉;以后设计为N个淘宝账户使用不同 的cookie进行抓取
            /// </summary>
            /// <param name="sellerId"></param>
            /// <param name="activityId"></param>
            /// <returns></returns>
            public async Task <string> LoadH5Api_YouhuiquanDetailAsync(long sellerId, string activityId)
            {
                try
                {
                    if (string.IsNullOrEmpty(activityId))
                    {
                        return(null);
                    }
                    //生成时间戳
                    string timestamp = JavascriptContext.getUnixTimestamp();

                    //加载Cookie
                    var ckVisitor = new LazyCookieVistor();
                    ///淘宝域下 的cookie
                    var    cks_taobao  = ckVisitor.LoadCookies(TaobaoSiteUrl);
                    string etao_appkey = "12574478";
                    //淘宝sdk 下的token
                    //List<Cookie> cks_h5 = ckVisitor.LoadCookies(TaobaoH5WebApiUrl);

                    var    _m_h5_tk_cookie      = cks_taobao.FirstOrDefault(x => x.Name == "_m_h5_tk");
                    string _m_h5_tk_valueString = string.Empty;


                    if (null == _m_h5_tk_cookie)
                    {
                        this.RefreshH5Api_Cookies();

                        cks_taobao      = ckVisitor.LoadCookies(TaobaoSiteUrl);
                        _m_h5_tk_cookie = cks_taobao.FirstOrDefault(x => x.Name == "_m_h5_tk");
                    }
                    if (null == _m_h5_tk_cookie || string.IsNullOrEmpty(_m_h5_tk_cookie.Value))
                    {
                        throw new Exception("加载授权私钥失败!无法获取对应的cookie:_m_h5_tk ");
                    }
                    _m_h5_tk_valueString = _m_h5_tk_cookie.Value.Split('_')[0];



                    string paras = string.Concat("{\"uuid\":\"", activityId, "\",\"sellerId\":\"", sellerId, "\",\"queryShop\":true}");

                    string sign = JavascriptContext.getEtaoJSSDKSign(_m_h5_tk_valueString, timestamp, etao_appkey, paras);

                    string apiUrl = string.Format("https://api.m.taobao.com/h5/mtop.taobao.couponmtopreadservice.findshopbonusactivitys/2.0/?appKey={0}&t={1}&sign={2}&api=mtop.taobao.couponMtopReadService.findShopBonusActivitys&v=2.0&AntiFlood=false&ecode=0&type=jsonp&dataType=jsonp&callback=mtopjsonp2&data={3}", etao_appkey, timestamp, sign, paras);


                    var client = TaobaoH5ApiHttpClient;

                    ////加载cookies
                    ////获取当前站点的Cookie
                    client.ChangeGlobleCookies(cks_taobao, TaobaoH5WebApiUrl);
                    //修改client 的refer 头
                    client.Client.DefaultRequestHeaders.Referrer = new Uri(apiUrl);
                    // 4 发送请求
                    var clientProxy = new HttpServerProxy()
                    {
                        Client = client.Client, KeepAlive = true
                    };
                    var taskMsg = await clientProxy.GetResponseTransferAsync(apiUrl, null);

                    string content = await taskMsg.Content.ReadAsStringAsync();


                    if (!string.IsNullOrEmpty(content) && content.Contains("FAIL_SYS_USER_VALIDATE"))
                    {
                        //需要身份验证的时候 ;FAIL_SYS_USER_VALIDATE::哎哟喂,被挤爆啦,请稍后重试; 被踢的时候 如何处理??
                        //清除taobao 的cookie 从客户端的rediscookie 从新加载
                        //ckVisitor.DeleteCookies(TaobaoSiteUrl);

                        int x = 0;
                    }

                    return(content);
                }
                catch (Exception ex)
                {
                    Logger.Error(ex);
                }

                return(string.Empty);
            }
Пример #22
0
 public TmallAutoMapping(string userAgent)
 {
     HttpServerProxy.ChangeHttpClientUserAgent(tmallHttpClient, userAgent);
 }
            public override string LoadUrlGetSearchApiContent(IFetchWebPageArgument queryParas)
            {
                VipSearchResultBag resultBag = new VipSearchResultBag();

                string keyWord = queryParas.KeyWord;

                if (string.IsNullOrEmpty(keyWord))
                {
                    return(string.Empty);
                }

                try
                {
                    //加载Cookie
                    var ckVisitor = new LazyCookieVistor();
                    var cks       = ckVisitor.LoadCookies(VipSiteUrl);



                    string searchUrl = string.Format(templateOfSearchUrl, keyWord);

                    var client = VipHttpClient;
                    client.Client.DefaultRequestHeaders.Referrer = new Uri(string.Format("https://m.vip.com/searchlist.html?q={0}&channel_id=", keyWord));
                    ////加载cookies
                    ////获取当前站点的Cookie
                    client.ChangeGlobleCookies(cks, VipSiteUrl);

                    // 4 发送请求
                    var clientProxy = new HttpServerProxy()
                    {
                        Client = client.Client, KeepAlive = true
                    };

                    //注意:对于响应的内容 不要使用内置的文本 工具打开,这个工具有bug.看到的文本不全面
                    //使用json格式打开即可看到里面所有的字符串
                    Task <HttpResponseMessage> brandTask;
                    Task <HttpResponseMessage> categoryTreeTask;
                    Task <HttpResponseMessage> searchListTask;
                    string para_brandJson        = "";
                    string para_categoryTreeJson = "";
                    string para_searchListJson   = "";
                    if (null != queryParas.ResolvedUrl && null != queryParas.ResolvedUrl.ParasPost)
                    {
                        para_brandJson        = queryParas.ResolvedUrl.ParasPost["para_brand"].ToString();
                        para_categoryTreeJson = queryParas.ResolvedUrl.ParasPost["para_categoryTree"].ToString();
                        para_searchListJson   = queryParas.ResolvedUrl.ParasPost["para_searchList"].ToString();
                    }
                    else
                    {
                        para_brandJson        = new VipSearchParaBrand(keyWord).ToJson();
                        para_categoryTreeJson = new VipSearchParaCategoryTree(keyWord).ToJson();

                        //插件不解析的话,那么使用最简单的基本关键词过滤分页,不进行复杂过滤,复杂过滤筛选应该在插件实现
                        var paraDetais = new VipSearchParaSearchList(keyWord);
                        //分页
                        paraDetais.paramsDetails.np = queryParas.PageIndex + 1;

                        para_searchListJson = paraDetais.ToJson();
                    }
                    if (queryParas.IsNeedResolveHeaderTags == true)
                    {
                        //1 查询品牌
                        var brandPara = new Dictionary <string, string>();
                        brandPara.Add("para_brand", para_brandJson);
                        brandTask = clientProxy.PostRequestTransferAsync(queryBrandUrl, PostDataContentType.Json, brandPara, null);
                        // 2 查询分类
                        var categoryTreePara = new Dictionary <string, string>();
                        categoryTreePara.Add("para_categoryTree", para_categoryTreeJson);
                        categoryTreeTask = clientProxy.PostRequestTransferAsync(queryCategoryUrl, PostDataContentType.Json, categoryTreePara, null);
                    }
                    else
                    {
                        brandTask        = Task.FromResult <HttpResponseMessage>(null);
                        categoryTreeTask = Task.FromResult <HttpResponseMessage>(null);
                    }

                    //3检索内容
                    var searchListPara = new Dictionary <string, string>();
                    searchListPara.Add("para_searchList", para_searchListJson);
                    searchListTask = clientProxy.PostRequestTransferAsync(templateOfSearchUrl, PostDataContentType.Json, searchListPara, null);

                    //等待任务完毕
                    Task.WaitAll(new Task[] { brandTask, categoryTreeTask, searchListTask });
                    if (brandTask.Result != null)
                    {
                        resultBag.BrandStoreList = brandTask.Result.Content.ReadAsStringAsync().Result;
                    }
                    if (categoryTreeTask.Result != null)
                    {
                        resultBag.CategoryTree = categoryTreeTask.Result.Content.ReadAsStringAsync().Result;
                    }
                    if (searchListTask.Result != null)
                    {
                        resultBag.SearchList = searchListTask.Result.Content.ReadAsStringAsync().Result;
                    }
                }
                catch (Exception ex)
                {
                    throw ex;
                }
                return(resultBag.ToJson());
            }