private void ProcessTradeDataMulti() { while (true) { if (_conQueue.Count > 0 && taskThreshold < taskScale) { try { IPMetaDataItem mdi = null; IPItem di = null; lock (objec) { if (_conQueue.TryDequeue(out di)) { mdi = di.Convert2IPMetaDataItemMulti(); } } Task.Run( () => { taskThreshold++; try { SpiderProcessMulti(mdi); } catch (Exception ex1) { //_logger.Error("优信拍 数据抓取任务异常:{0} mdi{1}", ex1, JsonConvert.SerializeObject(mdi)); } finally { taskThreshold--; } } ); //Thread.Sleep(30); if (ConfigUtls.proxy_rate_open == 0) { Thread.Sleep(random.Next(30)); } } catch (Exception ex) { //_logger.Error("优信拍 抓取主过程 异常:{0}", ex); } //if (internalSingleton.RequestCount == internalSingleton.RequestTotal || internalSingleton.RequestCount > internalSingleton.RequestTotal) //{ // Console.ReadKey(); //} } else { Thread.Sleep(procInterval); } } }
private void ProcessWork() { while (true) { if (listConQueue.Count > 0) { try { IPMetaDataItem mdi = null; IPItem di = null; lock (objec) { if (listConQueue.TryDequeue(out di)) { RefreshIESettings(di.ipwithport); //webBrowser1.Navigate("https://https://www.baidu.com/", null, null, null); webBrowser1.Navigate("https://heycar.m.autohome.com.cn/NewYearH5/Index?ssor=1851197957", null, null, null); //webBrowser2.Navigate("https://heycar.m.autohome.com.cn/AnnualMeetingAnswer/AnswerIndex"); //webBrowser3.Navigate("https://heycar.m.autohome.com.cn/young/SchoolZmActivity"); //webBrowser4.Navigate("https://club.m.autohome.com.cn/bbs/thread/922a3efdd3f63a4c/91344584-1.html#pvareaid=3460109"); //webBrowser5.Navigate("https://club.m.autohome.com.cn/partner/uc/thread/91341569"); //webBrowser6.Navigate("https://club.m.autohome.com.cn/partner/yidian/thread/91341569"); //webBrowser7.Navigate("https://club.m.autohome.com.cn/partner/qutoutiao/thread/91341569"); //webBrowser8.Navigate("https://club.m.autohome.com.cn/partner/oppo/thread/91341569"); //webBrowser9.Navigate("https://club.autohome.com.cn/hongrenzhuanti#pvareaid=3454633"); //webBrowser10.Navigate("https://heycar.m.autohome.com.cn/Act/haowu/index.htm"); //webBrowser11.Navigate("https://heycar.m.autohome.com.cn/Act/haowu/liebian.htm"); //webBrowser12.Navigate("https://heycar.m.autohome.com.cn/Act/oneyear/index.htm"); Thread.Sleep(8000); RefreshIESettings(string.Empty); } } } catch (Exception ex) { //_logger.Error("优信拍 抓取主过程 异常:{0}", ex); } } else { yxpProducer.Process(null); } } }
private void SetCookie(HttpClient httpClient, RequestUrlAndReferer requestUrlAndReferer, IPMetaDataItem mdi) { httpClient.DefaultRequestHeaders.Remove("Cookie"); //this.guidStr = Guid.NewGuid().ToString().ToUpper(); ////string guidStr1 = Guid.NewGuid().ToString().ToUpper(); //string dateTime = DateTime.Now.AddDays(-random.Next(10)).ToString(); //string cookie = string.Format("fvlid={1}; " + // "sessionid={2}{6} " + // "autoid={7}; " + // "ahpau=1; __ah_uuid_ng=c_{3}; " + // "sessionip={0}; " + // "sessionvid=681B9A1B-2EC5-4194-8437-33DEFB091DEA; " + // "area=119999; v_no=0; visit_info_ad={4}||681B9A1B-2EC5-4194-8437-33DEFB091DEA||-1||-1||4; ref={5}" // , mdi.outip, mdi.fvlid, // this.guidStr, this.guidStr, this.guidStr, // HttpUtility.UrlEncode(string.Format("0|0|0|0|{0}|{1}", DateTime.Now.ToString(), dateTime)) // , HttpUtility.UrlEncode(string.Format("||{0}||0; ", dateTime)), // Z.GetMD5LowerString(this.guidStr) // ); //string cookie = string.Format("fvlid={1}; " + // "sessionid=C0B882EA-FEDC-43CA-8E9F-B322F35528C8%7C%7C2020-07-25+12%3A38%3A50.381%7C%7C0; " + // "autoid=c933fac8868713f3f0e2d3d4b83f16b0; " + // "ahpau=1; __ah_uuid_ng=c_C0B882EA-FEDC-43CA-8E9F-B322F35528C8; " + // "sessionip={0}; " + // "sessionvid=681B9A1B-2EC5-4194-8437-33DEFB091DEA; " + // "area=119999; v_no=0; visit_info_ad=C0B882EA-FEDC-43CA-8E9F-B322F35528C8||681B9A1B-2EC5-4194-8437-33DEFB091DEA||-1||-1||4; ref=0%7C0%7C0%7C0%7C2020-08-02+08%3A00%3A14.918%7C2020-07-25+12%3A38%3A50.381" // , mdi.outip, requestUrlAndReferer.fvlid); //this.guidStr = Guid.NewGuid().ToString().ToUpper(); //this.guidStr1 = Guid.NewGuid().ToString().ToUpper(); //string dateTime = DateTime.Now.AddDays(-random.Next(10)).ToString(); //string cookie = string.Format("fvlid={1}; " + // "sessionid={2}{6} " + // "autoid={7}; " + // "ahpau=1; __ah_uuid_ng=c_{3}; " + // "sessionip={0}; " + // "sessionvid={8}; " + // "area=330199; v_no=1; visit_info_ad={4}||{9}||-1||-1||1; ref={5}" // , mdi.outip, mdi.fvlid, // this.guidStr, this.guidStr, this.guidStr, // HttpUtility.UrlEncode(string.Format("0|0|0|0|{0}|{1}", DateTime.Now.ToString(), dateTime)) // , HttpUtility.UrlEncode(string.Format("||{0}||0; ", dateTime)), // Z.GetMD5LowerString(this.guidStr), // this.guidStr1, this.guidStr1 // ); //string cookie = string.Format("fvlid={1}; " + // "sessionid=EC51C026-F4BB-4AB0-9872-993F6CFF34A8%7C%7C2020-10-14+18%3A20%3A47.654%7C%7C0; " + // "autoid=31bf984e655c9fe48f14c7176521ee08; " + // "ahpau=1; __ah_uuid_ng=c_EC51C026-F4BB-4AB0-9872-993F6CFF34A8; " + // "sessionip={0}; " + // "sessionvid=52CD437F-1054-4AD5-A7F4-345B803C41AA; " + // "area=330199; v_no=1; visit_info_ad=EC51C026-F4BB-4AB0-9872-993F6CFF34A8||52CD437F-1054-4AD5-A7F4-345B803C41AA||-1||-1||1; ref=0%7C0%7C0%7C0%7C2020-12-13+15%3A16%3A28.794%7C2020-10-14+18%3A20%3A47.654" // , mdi.outip, mdi.fvlid); //httpClient.DefaultRequestHeaders.Add("Cookie", cookie); httpClient.DefaultRequestHeaders.Remove("Referer"); //httpClient.DefaultRequestHeaders.Add("Referer", requestUrlAndReferer.referer); }
/// <summary> /// 获取动态代理 /// </summary> /// <param name="proxyIPport"></param> /// <returns></returns> private HttpClient GetProxyHttpClientMulti(IPMetaDataItem mdi) { var proxy = new WebProxy(string.Format("http://{0}", mdi.ipwithport)); HttpClientHandler httpClientHandler = new HttpClientHandler() { Proxy = proxy }; var httpCient = new HttpClient(httpClientHandler); // 增加头部 httpCient.DefaultRequestHeaders.Add("Accept", "image/webp,image/apng,image/*,*/*;q=0.8"); httpCient.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); httpCient.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9"); ////string guidStr = Guid.NewGuid().ToString().ToUpper(); ////string guidStr1 = Guid.NewGuid().ToString().ToUpper(); //string dateTime = DateTime.Now.AddDays(-random.Next(10)).ToString(); //string cookie = string.Format("fvlid={1}; " + // "sessionid={2}{6} " + // "autoid={7}; " + // "ahpau=1; __ah_uuid_ng=c_{3}; " + // "sessionip={0}; " + // "sessionvid=681B9A1B-2EC5-4194-8437-33DEFB091DEA; " + // "area=119999; v_no=0; visit_info_ad={4}||681B9A1B-2EC5-4194-8437-33DEFB091DEA||-1||-1||4; ref={5}" // , mdi.outip, mdi.fvlid, // this.guidStr, this.guidStr, this.guidStr, // HttpUtility.UrlEncode(string.Format("0|0|0|0|{0}|{1}", DateTime.Now.ToString(), dateTime)) // , HttpUtility.UrlEncode(string.Format("||{0}||0; ", dateTime)), // Z.GetMD5LowerString(this.guidStr) // ); //string cookie = string.Format("fvlid={1}; " + // "sessionid=C0B882EA-FEDC-43CA-8E9F-B322F35528C8%7C%7C2020-07-25+12%3A38%3A50.381%7C%7C0; " + // "autoid=c933fac8868713f3f0e2d3d4b83f16b0; " + // "ahpau=1; __ah_uuid_ng=c_C0B882EA-FEDC-43CA-8E9F-B322F35528C8; " + // "sessionip={0}; " + // "sessionvid=681B9A1B-2EC5-4194-8437-33DEFB091DEA; " + // "area=119999; v_no=0; visit_info_ad=C0B882EA-FEDC-43CA-8E9F-B322F35528C8||681B9A1B-2EC5-4194-8437-33DEFB091DEA||-1||-1||4; ref=0%7C0%7C0%7C0%7C2020-08-02+08%3A00%3A14.918%7C2020-07-25+12%3A38%3A50.381" // , mdi.outip, mdi.fvlid); //this.guidStr = Guid.NewGuid().ToString().ToUpper(); //this.guidStr1 = Guid.NewGuid().ToString().ToUpper(); //string dateTime = DateTime.Now.AddDays(-random.Next(10)).ToString(); ////string cookie = string.Format("fvlid={1}; " + // "sessionid={2}{6} " + // "autoid={7}; " + // "ahpau=1; __ah_uuid_ng=c_{3}; " + // "sessionip={0}; " + // "sessionvid={8}; " + // "area=330199; v_no=1; visit_info_ad={4}||{9}||-1||-1||1; ref={5}" // , mdi.outip, mdi.fvlid, // this.guidStr, this.guidStr, this.guidStr, // HttpUtility.UrlEncode(string.Format("0|0|0|0|{0}|{1}", DateTime.Now.ToString(), dateTime)) // , HttpUtility.UrlEncode(string.Format("||{0}||0; ", dateTime)), // Z.GetMD5LowerString(this.guidStr), // this.guidStr1, this.guidStr1 // ); //string cookie = string.Format("fvlid={1}; " + // "sessionid=EC51C026-F4BB-4AB0-9872-993F6CFF34A8%7C%7C2020-10-14+18%3A20%3A47.654%7C%7C0; " + // "autoid=31bf984e655c9fe48f14c7176521ee08; " + // "ahpau=1; __ah_uuid_ng=c_EC51C026-F4BB-4AB0-9872-993F6CFF34A8; " + // "sessionip={0}; " + // "sessionvid=52CD437F-1054-4AD5-A7F4-345B803C41AA; " + // "area=330199; v_no=1; visit_info_ad=EC51C026-F4BB-4AB0-9872-993F6CFF34A8||52CD437F-1054-4AD5-A7F4-345B803C41AA||-1||-1||1; ref=0%7C0%7C0%7C0%7C2020-12-13+15%3A16%3A28.794%7C2020-10-14+18%3A20%3A47.654" // , mdi.outip, mdi.fvlid); //httpCient.DefaultRequestHeaders.Add("Cookie", cookie); //httpCient.DefaultRequestHeaders.Add("Host", "al.autohome.com.cn"); //httpCient.DefaultRequestHeaders.Add("Referer", ConfigUtls.mda_pv_init_referer); httpCient.DefaultRequestHeaders.Add("User-Agent", string.Format("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.{0}.77 Safari/537.36", 3000 + random1.Next())); return(httpCient); }
/// <summary> /// 获取动态代理 /// </summary> /// <param name="proxyIPport"></param> /// <returns></returns> private HttpClient GetProxyHttpClient(IPMetaDataItem mdi) { var proxy = new WebProxy(string.Format("http://{0}", mdi.ipwithport)); HttpClientHandler httpClientHandler = new HttpClientHandler() { Proxy = proxy }; var httpCient = new HttpClient(httpClientHandler); // 增加头部 httpCient.DefaultRequestHeaders.Add("Accept", "image/webp,image/apng,image/*,*/*;q=0.8"); httpCient.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); httpCient.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9"); string areaid = "119999"; if (DateTime.Now < dt)//如果相加后的时间大于现在的时间 { string url = string.Format("http://heycar.api.autohome.com.cn/OpenFlipboard/GetIP2Area?ip={0}", mdi.outip); areaid = Z.WebRequest(url); } this.guidStr = Guid.NewGuid().ToString().ToUpper(); this.guidStr1 = Guid.NewGuid().ToString().ToUpper(); string dateTime = DateTime.Now.AddDays(-random.Next(10)).ToString(); string cookie = string.Format("fvlid={1}; " + "sessionid={2}{6} " + "autoid={7}; " + "ahpau=1; __ah_uuid_ng=c_{3}; " + "sessionip={0}; " + "sessionvid={8}; " + "area={10}; v_no=1; visit_info_ad={4}||{9}||-1||-1||1; ref={5}" , mdi.outip, mdi.fvlid, this.guidStr, this.guidStr, this.guidStr, HttpUtility.UrlEncode(string.Format("0|0|0|0|{0}|{1}", DateTime.Now.ToString(), dateTime)) , HttpUtility.UrlEncode(string.Format("||{0}||0; ", dateTime)), Z.GetMD5LowerString(this.guidStr), this.guidStr1, this.guidStr1, areaid ); //string cookie = string.Format("fvlid={1}; " + // "sessionid=C0B882EA-FEDC-43CA-8E9F-B322F35528C8%7C%7C2020-07-25+12%3A38%3A50.381%7C%7C0; " + // "autoid=c933fac8868713f3f0e2d3d4b83f16b0; " + // "ahpau=1; __ah_uuid_ng=c_C0B882EA-FEDC-43CA-8E9F-B322F35528C8; " + // "sessionip={0}; " + // "sessionvid=681B9A1B-2EC5-4194-8437-33DEFB091DEA; " + // "area={2}; v_no=0; visit_info_ad=C0B882EA-FEDC-43CA-8E9F-B322F35528C8||681B9A1B-2EC5-4194-8437-33DEFB091DEA||-1||-1||4; ref=0%7C0%7C0%7C0%7C2020-08-02+08%3A00%3A14.918%7C2020-07-25+12%3A38%3A50.381" // , mdi.outip, mdi.fvlid, areaid); //string cookie = string.Format("fvlid={1}; " + // "sessionid=C0B882EA-FEDC-43CA-8E9F-B322F35528C8%7C%7C2020-07-25+12%3A38%3A50.381%7C%7C0; " + // "autoid=c933fac8868713f3f0e2d3d4b83f16b0; " + // "ahpau=1; __ah_uuid_ng=c_C0B882EA-FEDC-43CA-8E9F-B322F35528C8; " + // "sessionip={0}; " + // "sessionvid=681B9A1B-2EC5-4194-8437-33DEFB091DEA; " + // "area=119999; v_no=0; visit_info_ad=C0B882EA-FEDC-43CA-8E9F-B322F35528C8||681B9A1B-2EC5-4194-8437-33DEFB091DEA||-1||-1||4; ref=0%7C0%7C0%7C0%7C2020-08-02+08%3A00%3A14.918%7C2020-07-25+12%3A38%3A50.381" // , mdi.outip, mdi.fvlid); httpCient.DefaultRequestHeaders.Add("Cookie", cookie); httpCient.DefaultRequestHeaders.Add("Host", "al.autohome.com.cn"); httpCient.DefaultRequestHeaders.Add("Referer", ConfigUtls.mda_pv_init_referer); httpCient.DefaultRequestHeaders.Add("User-Agent", string.Format("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.{0}.77 Safari/537.36", 3000 + random1.Next())); return(httpCient); }
private void SpiderProcessMulti(IPMetaDataItem mdi) { if (mdi != null) { if (string.IsNullOrWhiteSpace(mdi.ipwithport) || mdi.ipwithport.Contains("{")) { return; } //string proxy = mdi.ipwithport; //RefreshIESettings(proxy); //IEProxy ie = new IEProxy(proxy); //browser.RequestWeb(mdi.requesturl); //if (GatherTradeData(mdi, null)) //{ // GatherCarData(mdi, null); // _storeQueue.Enqueue(mdi); // //if (internalSingleton.RequestIpCount > internalSingleton.RequestTotal && // (_conQueue.Count == 0)) //{ // return; //} if (!string.IsNullOrWhiteSpace(mdi.ipwithport)) { HttpClient httpClient = GetProxyHttpClientMulti(mdi); if (ConfigUtls.proxy_rate_open > 0) { if (mdi.requestUrlAndReferers != null && mdi.requestUrlAndReferers.Count > 0) { try { #region pv方式 //foreach (var item in mdi.requestUrlAndReferers) //{ // if (item != null && !string.IsNullOrWhiteSpace(item.requesturl)) // { // string httpResult = ""; // string tempRequest = HttpUtility.UrlDecode(item.requesturl, System.Text.Encoding.UTF8); // try // { // SetCookie(httpClient, item, mdi); // httpClient.GetAsync(tempRequest); // httpResult = "dddd" + httpResult; // this.internalSingleton.RequestCount++; // //Console.WriteLine(string.Format("*执行总数totalCount:{0}/{1}当前执行数量 /获取资源数量{2}/消耗资源数量{5}/开始时间{3} 结束时间{4}", // // this.internalSingleton.RequestTotal, // // internalSingleton.RequestCount, // // internalSingleton.RequestIpCount, // // internalSingleton.BeginTime.ToString("yyyy-MM-dd hh:mm:ss fff"), // // DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss fff"), // // this.internalSingleton.ConsumerIpCount)); // } // catch (Exception e1) // { // //Console.WriteLine(string.Format("{0}\n{1}", mdi.ipwithport, e1.StackTrace.ToString())); // } // //参照间隔时间Thread.Sleep(20); // //Thread.Sleep(6); // Thread.Sleep(ConfigUtls.time_space_multi); // } //} #endregion //string item = pageurls[random2.Next(3)]; foreach (var item in pageurls) { //if (item != null && !string.IsNullOrWhiteSpace(item.requesturl)) //{ string httpResult = ""; string tempRequest = HttpUtility.UrlDecode(item, System.Text.Encoding.UTF8); try { //SetCookie(httpClient, item, mdi); httpClient.GetAsync(tempRequest); httpResult = "dddd" + httpResult; this.internalSingleton.RequestCount++; //Console.WriteLine(string.Format("*执行总数totalCount:{0}/{1}当前执行数量 /获取资源数量{2}/消耗资源数量{5}/开始时间{3} 结束时间{4}", // this.internalSingleton.RequestTotal, // internalSingleton.RequestCount, // internalSingleton.RequestIpCount, // internalSingleton.BeginTime.ToString("yyyy-MM-dd hh:mm:ss fff"), // DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss fff"), // this.internalSingleton.ConsumerIpCount)); } catch (Exception e1) { Console.WriteLine(string.Format("{0}\n{1}", mdi.ipwithport, e1.StackTrace.ToString())); } //参照间隔时间Thread.Sleep(20); //Thread.Sleep(6); //Thread.Sleep(ConfigUtls.time_space_multi); //} } //if (ConfigUtls.probe_switch > 0 && DateTime.Now.Minute == 29 && 28 < DateTime.Now.Second && DateTime.Now.Second < 31) //{ //string tempRequest1 = HttpUtility.UrlDecode("https://club.m.autohome.com.cn/partner/oppo/thread/91341569", System.Text.Encoding.UTF8); //var response = httpClient.GetAsync(tempRequest1).Result; //mdi.result = string.Format("Ipport {3} -{4} Datetime {2} IsSuccessStatusCode {0} StatusCode {1} Cookie {5} httpContent{6}" // , response.IsSuccessStatusCode, response.StatusCode, DateTime.Now, mdi.ipwithport, mdi.outip, // JsonConvert.SerializeObject(response), JsonConvert.SerializeObject(response.Content)); //this._storeQueue.Enqueue(mdi); string tempRequest1 = HttpUtility.UrlDecode("https://club.m.autohome.com.cn/partner/oppo/thread/91341569", System.Text.Encoding.UTF8); var response = httpClient.GetStringAsync(tempRequest1).Result; mdi.result = string.Format("Ipport {2} -{3} Datetime {1} response {0} " , response, DateTime.Now, mdi.ipwithport, mdi.outip); this._storeQueue.Enqueue(mdi); //} Console.WriteLine(string.Format("*执行总数totalCount:{0}/{1}当前执行数量 /获取资源数量{2}/消耗资源数量{5}/队列长度{6}/开始时间{3} 结束时间{4}", this.internalSingleton.RequestTotal, internalSingleton.RequestCount, internalSingleton.RequestIpCount, internalSingleton.BeginTime.ToString("yyyy-MM-dd hh:mm:ss fff"), DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss fff"), this.internalSingleton.ConsumerIpCount, this._conQueue.Count)); } catch (Exception e2) { Console.WriteLine(string.Format("{0}\n{1}", mdi.ipwithport, e2.StackTrace.ToString())); } finally { this.internalSingleton.ConsumerIpCount++; httpClient.Dispose(); httpClient = null; mdi = null; } } } else { if (mdi.requestUrlAndReferers != null && mdi.requestUrlAndReferers.Count > 0) { try { foreach (var item in mdi.requestUrlAndReferers) { if (item != null && !string.IsNullOrWhiteSpace(item.requesturl)) { string httpResult = ""; string tempRequest = HttpUtility.UrlDecode(item.requesturl, System.Text.Encoding.UTF8); try { SetCookie(httpClient, item, mdi); httpClient.GetAsync(tempRequest); httpResult = "dddd" + httpResult; this.internalSingleton.RequestCount++; //Console.WriteLine(string.Format("*执行总数totalCount:{0}/{1}当前执行数量 /获取资源数量{2}/消耗资源数量{5}/开始时间{3} 结束时间{4}", // this.internalSingleton.RequestTotal, // internalSingleton.RequestCount, // internalSingleton.RequestIpCount, // internalSingleton.BeginTime.ToString("yyyy-MM-dd hh:mm:ss fff"), // DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss fff"), // this.internalSingleton.ConsumerIpCount)); } catch (Exception e1) { //Console.WriteLine(string.Format("{0}\n{1}", mdi.ipwithport, e1.StackTrace.ToString())); } } } Console.WriteLine(string.Format("*执行总数totalCount:{0}/{1}当前执行数量 /获取资源数量{2}/消耗资源数量{5}/队列长度{6}/开始时间{3} 结束时间{4}", this.internalSingleton.RequestTotal, internalSingleton.RequestCount, internalSingleton.RequestIpCount, internalSingleton.BeginTime.ToString("yyyy-MM-dd hh:mm:ss fff"), DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss fff"), this.internalSingleton.ConsumerIpCount, this._conQueue.Count)); } catch (Exception) { } finally { this.internalSingleton.ConsumerIpCount++; httpClient.Dispose(); httpClient = null; mdi = null; } } } } } }
private void SpiderProcess(IPMetaDataItem mdi) { if (mdi != null) { if (string.IsNullOrWhiteSpace(mdi.ipwithport) || mdi.ipwithport.Contains("{")) { return; } //string proxy = mdi.ipwithport; //RefreshIESettings(proxy); //IEProxy ie = new IEProxy(proxy); //browser.RequestWeb(mdi.requesturl); //if (GatherTradeData(mdi, null)) //{ // GatherCarData(mdi, null); // _storeQueue.Enqueue(mdi); // //if (internalSingleton.RequestIpCount > internalSingleton.RequestTotal && // (_conQueue.Count == 0)) //{ // return; //} if (!string.IsNullOrWhiteSpace(mdi.ipwithport)) { HttpClient httpClient = GetProxyHttpClient(mdi); if (ConfigUtls.proxy_rate_open > 0) { if (mdi.requesturls != null && mdi.requesturls.Length > 0) { try { foreach (var item in mdi.requesturls) { if (!string.IsNullOrWhiteSpace(item)) { string httpResult = ""; string tempRequest = HttpUtility.UrlDecode(item, System.Text.Encoding.UTF8); try { httpClient.GetAsync(tempRequest); httpResult = "dddd" + httpResult; this.internalSingleton.RequestCount++; //Console.WriteLine(string.Format("*执行总数totalCount:{0}/{1}当前执行数量 /获取资源数量{2}/消耗资源数量{5}/开始时间{3} 结束时间{4}", // this.internalSingleton.RequestTotal, // internalSingleton.RequestCount, // internalSingleton.RequestIpCount, // internalSingleton.BeginTime.ToString("yyyy-MM-dd hh:mm:ss fff"), // DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss fff"), // this.internalSingleton.ConsumerIpCount)); } catch (Exception e1) { //Console.WriteLine(string.Format("{0}\n{1}", mdi.ipwithport, e1.StackTrace.ToString())); } } //Thread.Sleep(20); Thread.Sleep(ConfigUtls.time_space); } string tzResult = ""; //if (DateTime.Now.Minute % 9 == 0) //{ // string tempRequest = HttpUtility.UrlDecode("http://wetopic.api.autohome.com.cn/api/test", System.Text.Encoding.UTF8); // var response = httpClient.GetAsync(tempRequest).Result; // if (response.IsSuccessStatusCode) // { // tzResult = response.ToString(); // } //} Console.WriteLine(string.Format("*执行总数totalCount:{0}/{1}当前执行数量 /获取资源数量{2}/消耗资源数量{5}/队列长度{7}/开始时间{3} 结束时间{4} Result:{6}", this.internalSingleton.RequestTotal, internalSingleton.RequestCount, internalSingleton.RequestIpCount, internalSingleton.BeginTime.ToString("yyyy-MM-dd hh:mm:ss fff"), DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss fff"), this.internalSingleton.ConsumerIpCount, tzResult, this._conQueue.Count)); } catch (Exception) { } finally { this.internalSingleton.ConsumerIpCount++; httpClient.Dispose(); httpClient = null; //mdi.Dispose(); mdi = null; } } } else { if (!string.IsNullOrWhiteSpace(mdi.requesturl)) { string httpResult = ""; string tempRequest = HttpUtility.UrlDecode(mdi.requesturl, System.Text.Encoding.UTF8); try { httpClient.GetAsync(tempRequest); httpResult = "dddd" + httpResult; this.internalSingleton.RequestCount++; Console.WriteLine(string.Format("*执行总数totalCount:{0}/{1}当前执行数量 /获取资源数量{2}/消耗资源数量{5}/队列长度{6}/开始时间{3} 结束时间{4}", this.internalSingleton.RequestTotal, internalSingleton.RequestCount, internalSingleton.RequestIpCount, internalSingleton.BeginTime.ToString("yyyy-MM-dd hh:mm:ss fff"), DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss fff"), this.internalSingleton.ConsumerIpCount, this._conQueue.Count)); } catch (Exception e2) { } finally { this.internalSingleton.ConsumerIpCount++; httpClient.Dispose(); httpClient = null; //mdi.Dispose(); mdi = null; } } } } } }