/// <summary>
/// 为请求准备参数
/// </summary>
///<param name="item">参数列表</param>
private void SetRequest(HttpItem item)
{
// 验证证书
SetCer(item);
if (item.IPEndPoint != null)
{
_ipEndPoint = item.IPEndPoint;
//设置本地的出口ip和端口
_request.ServicePoint.BindIPEndPointDelegate = new BindIPEndPoint(BindIPEndPointCallback);
}
//设置Header参数
if (item.Header != null && item.Header.Count > 0)
{
foreach (string key in item.Header.AllKeys)
{
_request.Headers.Add(key, item.Header[key]);
}
}
// 设置代理
SetProxy(item);
if (item.ProtocolVersion != null)
{
//获取或设置用于请求的 HTTP 版本
_request.ProtocolVersion = item.ProtocolVersion;
}
//获取或设置一个 System.Boolean 值，该值确定是否使用 100-Continue 行为
_request.ServicePoint.Expect100Continue = item.Expect100Continue;
//请求方式Get或者Post
_request.Method = item.Method;
//请求超时时间
_request.Timeout = item.Timeout;
//是否建立永久链接
_request.KeepAlive = item.KeepAlive;
//写入数据超时时间
_request.ReadWriteTimeout = item.ReadWriteTimeout;
if (item.IfModifiedSince != null)
{
//获取或设置 If-Modified-Since HTTP 标头的值
_request.IfModifiedSince = Convert.ToDateTime(item.IfModifiedSince);
}
//Accept
_request.Accept = item.Accept;
//ContentType返回类型
_request.ContentType = item.ContentType;
//UserAgent客户端的访问类型，包括浏览器版本和操作系统信息
_request.UserAgent = item.UserAgent;
// 编码
_encoding = item.Encoding;
//设置安全凭证
_request.Credentials = item.ICredentials;
//设置Cookie
SetCookie(item);
//来源地址
_request.Referer = item.Referer;
//是否执行跳转功能
_request.AllowAutoRedirect = item.Allowautoredirect;
if (item.MaximumAutomaticRedirections > 0)
{
//获取或设置请求将跟随的重定向的最大数目
_request.MaximumAutomaticRedirections = item.MaximumAutomaticRedirections;
}
//设置Post数据
SetPostData(item);
//设置最大连接
if (item.Connectionlimit > 0)
{
//获取或设置此 System.Net.ServicePoint 对象上允许的最大连接数
_request.ServicePoint.ConnectionLimit = item.Connectionlimit;
}
}
public void Grab_ProxyIp(ConcurrentQueue <string> proxyIpQueue)
{
HashSet <string> proxyIp = new HashSet <string>();
HttpHelper http = new HttpHelper();
HttpItem para = new HttpItem();
para.Timeout = 1000 * 10;
para.Method = "GET";
int count = 0;
para.Url = "http://www.xicidaili.com/nn/1"; // 西刺
RetryFunc(() =>
{
HttpResult result = http.GetHtml(para);
if (result.StatusCode == System.Net.HttpStatusCode.OK)
{
string regex = @"<td>(\d+\.\d+\.\d+\.\d+)</td>\s+<td>(\d+)</td>";
Match mstr = Regex.Match(result.Html, regex);
while (mstr.Success && count < 20)
{
proxyIp.Add(mstr.Groups[1].Value + ":" + mstr.Groups[2].Value);
mstr = mstr.NextMatch();
count++;
}
return(true);
}
else
{
return(false);
}
}, 10);
count = 0;
para.Url = "http://ip84.com/dlgn"; // IP巴士
RetryFunc(() =>
{
HttpResult result = http.GetHtml(para);
if (result.StatusCode == System.Net.HttpStatusCode.OK)
{
string regex = @"<td>(\d+\.\d+\.\d+\.\d+)</td>\s+<td>(\d+)</td>";
Match mstr = Regex.Match(result.Html, regex);
while (mstr.Success && count < 10)
{
proxyIp.Add(mstr.Groups[1].Value + ":" + mstr.Groups[2].Value);
mstr = mstr.NextMatch();
count++;
}
return(true);
}
else
{
return(false);
}
}, 10);
count = 0;
para.Url = "http://www.ip3366.net/free/?stype=1"; // 云代理
RetryFunc(() =>
{
HttpResult result = http.GetHtml(para);
if (result.StatusCode == System.Net.HttpStatusCode.OK)
{
string regex = @"<td>(\d+\.\d+\.\d+\.\d+)</td>\s+<td>(\d+)</td>";
Match mstr = Regex.Match(result.Html, regex);
while (mstr.Success && count < 10)
{
proxyIp.Add(mstr.Groups[1].Value + ":" + mstr.Groups[2].Value);
mstr = mstr.NextMatch();
count++;
}
return(true);
}
else
{
return(false);
}
}, 10);
count = 0;
para.Url = "http://www.iphai.com/free/ng"; // IP海
RetryFunc(() =>
{
HttpResult result = http.GetHtml(para);
if (result.StatusCode == System.Net.HttpStatusCode.OK)
{
string regex = @"<td>\s+(\d+\.\d+\.\d+\.\d+)\s+</td>\s+<td>\s+(\d+)\s+</td>";
Match mstr = Regex.Match(result.Html, regex);
while (mstr.Success && count < 10)
{
proxyIp.Add(mstr.Groups[1].Value + ":" + mstr.Groups[2].Value);
mstr = mstr.NextMatch();
count++;
}
return(true);
}
else
{
return(false);
}
}, 10);
count = 0;
para.Url = "http://www.66ip.cn/nmtq.php?getnum=10&isp=0&anonymoustype=3&start=&ports=&export=&ipaddress=&area=1&proxytype=2&api=66ip"; // 66ip
RetryFunc(() =>
{
HttpResult result = http.GetHtml(para);
if (result.StatusCode == System.Net.HttpStatusCode.OK)
{
string regex = @"(\d+\.\d+\.\d+\.\d+):(\d+)<br/>";
Match mstr = Regex.Match(result.Html, regex);
while (mstr.Success && count < 10)
{
proxyIp.Add(mstr.Groups[1].Value + ":" + mstr.Groups[2].Value);
mstr = mstr.NextMatch();
count++;
}
return(true);
}
else
{
return(false);
}
}, 10);
foreach (var item in proxyIp)
{
proxyIpQueue.Enqueue(item);
}
}