private void client_OpenReadCompleted(object sender, OpenReadCompletedEventArgs e)
{
Stream data = null;
StreamReader reader = null;
try
{
NDAWebClient client = (NDAWebClient)sender;
string ipPort = ((WebProxy)client.Proxy).Address.Authority;
reader = new StreamReader(e.Result, Encoding.GetEncoding(_PageEncoding));
string s = reader.ReadToEnd();
_CheckResults.Add(ipPort, s);
}
catch (Exception ex)
{
}
finally
{
if (data != null)
{
data.Close();
data.Dispose();
}
if (reader != null)
{
reader.Close();
reader.Dispose();
}
}
}
private Nullable <TimeSpan> CheckIpPortUsable(string ip, string port, int timeout, string testPageUrl, string checkText)
{
NDAWebClient client = null;
try
{
DateTime dt1 = DateTime.Now;
client = new NDAWebClient();
client.Id = ip + ":" + port;
System.Net.ServicePointManager.DefaultConnectionLimit = 512;
client.Timeout = timeout;
WebProxy webProxy = new WebProxy(ip, int.Parse(port));
string ipPort = webProxy.Address.Authority;
client.Proxy = webProxy;
client.OpenReadCompleted += client_OpenReadCompleted;
client.OpenReadAsync(new Uri(testPageUrl));
string s = null;
int waitingTime = 0;
while (s == null && waitingTime < timeout)
{
if (_CheckResults.ContainsKey(ipPort))
{
s = _CheckResults[ipPort];
}
else
{
waitingTime = waitingTime + 1000;
Thread.Sleep(1000);
}
}
if (s != null && s.Contains(checkText))
{
DateTime dt2 = DateTime.Now;
string msgInfo = "检测成功!";
//this.InvokeAppendLogText(errorInfo, LogLevelType.Error, true);
this.RunPage.InvokeAppendLogText(msgInfo, LogLevelType.Normal, true);
return(dt2 - dt1);
}
else
{
DateTime dt2 = DateTime.Now;
string msgInfo = "检测超时!";
this.RunPage.InvokeAppendLogText(msgInfo, LogLevelType.Error, true);
return(null);
}
}
catch (Exception ex)
{
DateTime dt2 = DateTime.Now;
string errorInfo = "访问失败, " + ip + ":" + port + ". " + ex.Message;
//this.InvokeAppendLogText(errorInfo, LogLevelType.Error, true);
this.RunPage.InvokeAppendLogText(errorInfo, LogLevelType.Error, true);
return(null);
}
}
public override void CheckRequestCompleteFile(string webPageText, Dictionary <string, string> listRow)
{
if (!webPageText.Contains("redirect-message"))
{
throw new BlockedException("被封了");
}
else
{
string pageUrl = listRow[SysConfig.DetailPageUrlFieldName];
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(webPageText);
HtmlNode linkNode = htmlDoc.DocumentNode.SelectSingleNode("//div[@id=\"redirect-message\"]/p/a");
string pdfUrl = linkNode.GetAttributeValue("href", "");
string sourceDir = this.RunPage.GetDetailSourceFileDir();
string pdfFilePath = this.RunPage.GetFilePath(pdfUrl, sourceDir);
try
{
NDAWebClient wc = new NDAWebClient();
wc.CookieContainer = this._CC;
//string cookie = this._CC.GetCookieHeader(new Uri(pageUrl));
byte[] bytes = wc.DownloadData(pdfUrl);
FileStream fs = null;
try
{
fs = new FileStream(pdfFilePath, FileMode.Create, FileAccess.Write);
fs.Write(bytes, 0, bytes.Length);
fs.Flush();
}
catch (Exception ex)
{
this.RunPage.InvokeAppendLogText("保存pdf文件出错", LogLevelType.Error, true);
throw ex;
}
finally
{
if (fs != null)
{
fs.Close();
fs.Dispose();
fs = null;
}
}
}
catch (Exception ex)
{
this.RunPage.InvokeAppendLogText("获取pdf文件出错", LogLevelType.Error, true);
throw ex;
}
}
}
public byte[] GetFileByRequest(string pageUrl, decimal intervalAfterLoaded, int timeout)
{
NDAWebClient client = null;
try
{
client = new NDAWebClient();
client.Timeout = timeout;
client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
byte[] data = client.DownloadData(pageUrl);
return(data);
}
catch (Exception ex)
{
string errorInfo = "访问失败, PageUrl = " + pageUrl + ". " + ex.Message;
//this.InvokeAppendLogText(errorInfo, LogLevelType.Error, true);
throw new GrabRequestException(errorInfo);
}
}
private void client_OpenReadCompleted(object sender, OpenReadCompletedEventArgs e)
{
if (this.RunPage.Grabing)
{
Stream stream = null;
StreamReader reader = null;
try
{
NDAWebClient client = (NDAWebClient)sender;
if (e.Error == null)
{
stream = (Stream)e.Result;
reader = new StreamReader(stream, client.ResponseEncoding);
string s = reader.ReadToEnd();
AddResponseData(client.Id, s);
}
else
{
AddResponseData(client.Id, e.Error);
}
}
catch (Exception ex)
{
this.RunPage.InvokeAppendLogText("ReadToEnd获取字符串超时. " + ex.Message, LogLevelType.System, true);
}
finally
{
if (stream != null)
{
stream.Close();
stream.Dispose();
}
if (reader != null)
{
reader.Close();
reader.Dispose();
}
}
}
}
public string GetTextByRequest(string pageUrl, Dictionary <string, string> listRow, bool needProxy, decimal intervalAfterLoaded, int timeout, Encoding encoding, string cookie, string xRequestedWith, bool autoAbandonDisableProxy, Proj_DataAccessType dataAccessType, Proj_CompleteCheckList completeChecks, int intervalProxyRequest)
{
NDAWebClient client = null;
try
{
DateTime dt1 = DateTime.Now;
client = new NDAWebClient();
client.Id = pageUrl;
client.ResponseEncoding = encoding;
System.Net.ServicePointManager.DefaultConnectionLimit = 512;
client.Timeout = timeout;
if (needProxy)
{
client.ProxyServer = this.RunPage.CurrentProxyServers.BeginUse(intervalProxyRequest);
}
//client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
string userAgent = this.RunPage.CurrentUserAgents.GetOneUserAgent();
client.Headers.Add("user-agent", userAgent);
if (!CommonUtil.IsNullOrBlank(cookie))
{
client.Headers.Add("cookie", cookie);
//client.Headers.Add("connection", "keep-alive");
}
client.Headers.Add("x-requested-with", "XMLHttpRequest");
client.OpenReadCompleted += client_OpenReadCompleted;
client.OpenReadAsync(new Uri(pageUrl));
int waitingTime = 0;
object data = null;
while (data == null && waitingTime < timeout)
{
data = GetResponseString(client.Id);
if (data == null)
{
waitingTime = waitingTime + 3000;
Thread.Sleep(3000);
}
}
if (data != null)
{
RemoveResponseData(client.Id);
if (data is Exception)
{
throw (Exception)data;
}
else
{
string s = null;
if (data is string)
{
s = (string)data;
}
if (data is byte[])
{
s = encoding.GetString((byte[])data);
}
this.CheckRequestCompleteFile(s, listRow);
if (needProxy)
{
this.RunPage.CurrentProxyServers.Success(client.ProxyServer);
}
//再增加个等待，等待异步加载的数据
Thread.Sleep((int)intervalAfterLoaded);
DateTime dt2 = DateTime.Now;
double ts = (dt2 - dt1).TotalSeconds;
return(s);
}
}
else
{
throw new Exception("访问超时.");
}
}
catch (NoneProxyException ex)
{
throw ex;
}
catch (Exception ex)
{
string errorInfo = "";
if (needProxy)
{
if (autoAbandonDisableProxy)
{
this.RunPage.CurrentProxyServers.Error(client.ProxyServer);
if (client.ProxyServer.IsAbandon)
{
errorInfo = "放弃代理服务器:" + client.ProxyServer.IP + ":" + client.ProxyServer.Port.ToString() + ". ";
}
else
{
errorInfo = "代理服务器:" + client.ProxyServer.IP + ":" + client.ProxyServer.Port.ToString() + ". ";
}
}
else
{
errorInfo = "代理服务器:" + client.ProxyServer.IP + ":" + client.ProxyServer.Port.ToString() + ". ";
}
}
errorInfo = "获取网页失败.\r\n" + errorInfo + " " + pageUrl;
throw new GrabRequestException(errorInfo, ex);
}
finally
{
if (needProxy)
{
this.RunPage.CurrentProxyServers.EndUse(client.ProxyServer);
}
}
}
/// <summary>
/// 发送请求前（通过httprequest获取页面）
/// </summary>
/// <param name="pageUrl"></param>
/// <param name="listRow"></param>
/// <param name="client"></param>
public virtual void WebRequestHtml_BeforeSendRequest(string pageUrl, Dictionary <string, string> listRow, NDAWebClient client)
{
}
public override void WebRequestHtml_BeforeSendRequest(string pageUrl, Dictionary <string, string> listRow, NDAWebClient client)
{
this._CC = new CookieContainer();
client.CookieContainer = this._CC;
}
