private void client_OpenReadCompleted(object sender, OpenReadCompletedEventArgs e) { Stream data = null; StreamReader reader = null; try { NDAWebClient client = (NDAWebClient)sender; string ipPort = ((WebProxy)client.Proxy).Address.Authority; reader = new StreamReader(e.Result, Encoding.GetEncoding(_PageEncoding)); string s = reader.ReadToEnd(); _CheckResults.Add(ipPort, s); } catch (Exception ex) { } finally { if (data != null) { data.Close(); data.Dispose(); } if (reader != null) { reader.Close(); reader.Dispose(); } } }
private Nullable <TimeSpan> CheckIpPortUsable(string ip, string port, int timeout, string testPageUrl, string checkText) { NDAWebClient client = null; try { DateTime dt1 = DateTime.Now; client = new NDAWebClient(); client.Id = ip + ":" + port; System.Net.ServicePointManager.DefaultConnectionLimit = 512; client.Timeout = timeout; WebProxy webProxy = new WebProxy(ip, int.Parse(port)); string ipPort = webProxy.Address.Authority; client.Proxy = webProxy; client.OpenReadCompleted += client_OpenReadCompleted; client.OpenReadAsync(new Uri(testPageUrl)); string s = null; int waitingTime = 0; while (s == null && waitingTime < timeout) { if (_CheckResults.ContainsKey(ipPort)) { s = _CheckResults[ipPort]; } else { waitingTime = waitingTime + 1000; Thread.Sleep(1000); } } if (s != null && s.Contains(checkText)) { DateTime dt2 = DateTime.Now; string msgInfo = "检测成功!"; //this.InvokeAppendLogText(errorInfo, LogLevelType.Error, true); this.RunPage.InvokeAppendLogText(msgInfo, LogLevelType.Normal, true); return(dt2 - dt1); } else { DateTime dt2 = DateTime.Now; string msgInfo = "检测超时!"; this.RunPage.InvokeAppendLogText(msgInfo, LogLevelType.Error, true); return(null); } } catch (Exception ex) { DateTime dt2 = DateTime.Now; string errorInfo = "访问失败, " + ip + ":" + port + ". " + ex.Message; //this.InvokeAppendLogText(errorInfo, LogLevelType.Error, true); this.RunPage.InvokeAppendLogText(errorInfo, LogLevelType.Error, true); return(null); } }
public override void CheckRequestCompleteFile(string webPageText, Dictionary <string, string> listRow) { if (!webPageText.Contains("redirect-message")) { throw new BlockedException("被封了"); } else { string pageUrl = listRow[SysConfig.DetailPageUrlFieldName]; HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument(); htmlDoc.LoadHtml(webPageText); HtmlNode linkNode = htmlDoc.DocumentNode.SelectSingleNode("//div[@id=\"redirect-message\"]/p/a"); string pdfUrl = linkNode.GetAttributeValue("href", ""); string sourceDir = this.RunPage.GetDetailSourceFileDir(); string pdfFilePath = this.RunPage.GetFilePath(pdfUrl, sourceDir); try { NDAWebClient wc = new NDAWebClient(); wc.CookieContainer = this._CC; //string cookie = this._CC.GetCookieHeader(new Uri(pageUrl)); byte[] bytes = wc.DownloadData(pdfUrl); FileStream fs = null; try { fs = new FileStream(pdfFilePath, FileMode.Create, FileAccess.Write); fs.Write(bytes, 0, bytes.Length); fs.Flush(); } catch (Exception ex) { this.RunPage.InvokeAppendLogText("保存pdf文件出错", LogLevelType.Error, true); throw ex; } finally { if (fs != null) { fs.Close(); fs.Dispose(); fs = null; } } } catch (Exception ex) { this.RunPage.InvokeAppendLogText("获取pdf文件出错", LogLevelType.Error, true); throw ex; } } }
public byte[] GetFileByRequest(string pageUrl, decimal intervalAfterLoaded, int timeout) { NDAWebClient client = null; try { client = new NDAWebClient(); client.Timeout = timeout; client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)"); byte[] data = client.DownloadData(pageUrl); return(data); } catch (Exception ex) { string errorInfo = "访问失败, PageUrl = " + pageUrl + ". " + ex.Message; //this.InvokeAppendLogText(errorInfo, LogLevelType.Error, true); throw new GrabRequestException(errorInfo); } }
private void client_OpenReadCompleted(object sender, OpenReadCompletedEventArgs e) { if (this.RunPage.Grabing) { Stream stream = null; StreamReader reader = null; try { NDAWebClient client = (NDAWebClient)sender; if (e.Error == null) { stream = (Stream)e.Result; reader = new StreamReader(stream, client.ResponseEncoding); string s = reader.ReadToEnd(); AddResponseData(client.Id, s); } else { AddResponseData(client.Id, e.Error); } } catch (Exception ex) { this.RunPage.InvokeAppendLogText("ReadToEnd获取字符串超时. " + ex.Message, LogLevelType.System, true); } finally { if (stream != null) { stream.Close(); stream.Dispose(); } if (reader != null) { reader.Close(); reader.Dispose(); } } } }
public string GetTextByRequest(string pageUrl, Dictionary <string, string> listRow, bool needProxy, decimal intervalAfterLoaded, int timeout, Encoding encoding, string cookie, string xRequestedWith, bool autoAbandonDisableProxy, Proj_DataAccessType dataAccessType, Proj_CompleteCheckList completeChecks, int intervalProxyRequest) { NDAWebClient client = null; try { DateTime dt1 = DateTime.Now; client = new NDAWebClient(); client.Id = pageUrl; client.ResponseEncoding = encoding; System.Net.ServicePointManager.DefaultConnectionLimit = 512; client.Timeout = timeout; if (needProxy) { client.ProxyServer = this.RunPage.CurrentProxyServers.BeginUse(intervalProxyRequest); } //client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)"); string userAgent = this.RunPage.CurrentUserAgents.GetOneUserAgent(); client.Headers.Add("user-agent", userAgent); if (!CommonUtil.IsNullOrBlank(cookie)) { client.Headers.Add("cookie", cookie); //client.Headers.Add("connection", "keep-alive"); } client.Headers.Add("x-requested-with", "XMLHttpRequest"); client.OpenReadCompleted += client_OpenReadCompleted; client.OpenReadAsync(new Uri(pageUrl)); int waitingTime = 0; object data = null; while (data == null && waitingTime < timeout) { data = GetResponseString(client.Id); if (data == null) { waitingTime = waitingTime + 3000; Thread.Sleep(3000); } } if (data != null) { RemoveResponseData(client.Id); if (data is Exception) { throw (Exception)data; } else { string s = null; if (data is string) { s = (string)data; } if (data is byte[]) { s = encoding.GetString((byte[])data); } this.CheckRequestCompleteFile(s, listRow); if (needProxy) { this.RunPage.CurrentProxyServers.Success(client.ProxyServer); } //再增加个等待,等待异步加载的数据 Thread.Sleep((int)intervalAfterLoaded); DateTime dt2 = DateTime.Now; double ts = (dt2 - dt1).TotalSeconds; return(s); } } else { throw new Exception("访问超时."); } } catch (NoneProxyException ex) { throw ex; } catch (Exception ex) { string errorInfo = ""; if (needProxy) { if (autoAbandonDisableProxy) { this.RunPage.CurrentProxyServers.Error(client.ProxyServer); if (client.ProxyServer.IsAbandon) { errorInfo = "放弃代理服务器:" + client.ProxyServer.IP + ":" + client.ProxyServer.Port.ToString() + ". "; } else { errorInfo = "代理服务器:" + client.ProxyServer.IP + ":" + client.ProxyServer.Port.ToString() + ". "; } } else { errorInfo = "代理服务器:" + client.ProxyServer.IP + ":" + client.ProxyServer.Port.ToString() + ". "; } } errorInfo = "获取网页失败.\r\n" + errorInfo + " " + pageUrl; throw new GrabRequestException(errorInfo, ex); } finally { if (needProxy) { this.RunPage.CurrentProxyServers.EndUse(client.ProxyServer); } } }
/// <summary> /// 发送请求前(通过httprequest获取页面) /// </summary> /// <param name="pageUrl"></param> /// <param name="listRow"></param> /// <param name="client"></param> public virtual void WebRequestHtml_BeforeSendRequest(string pageUrl, Dictionary <string, string> listRow, NDAWebClient client) { }
public override void WebRequestHtml_BeforeSendRequest(string pageUrl, Dictionary <string, string> listRow, NDAWebClient client) { this._CC = new CookieContainer(); client.CookieContainer = this._CC; }