/// <summary>
/// 获取西刺首页代理
/// </summary>
public static void Xici()
{
while (true)
{
try
{
string url = "http://www.xicidaili.com/";
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(IWebClient(url));
var table = doc.DocumentNode.SelectSingleNode("//table[@id='ip_list']");
var tdList = table.SelectNodes("//tr").ToList();
for (int i = 0; i < tdList.Count; i++)
{
try
{
var td = tdList[i].SelectNodes("td").ToList();
if (td.Count != 8)
{
continue;
}
var ip = td[1].InnerText;
int port = Convert.ToInt32(td[2].InnerText);
ProxyViewModel proxy = new ProxyViewModel()
{
Id = string.Format("{0}:{1}", ip, port), ProxyIP = ip, ProxyPort = port, CreateTime = DateTime.Now, State = 0
};
if (QueueOperation(proxy, IQueueType.Exsist).First().Key > 0)
{
continue;
}
ProxyVerification(proxy, "西刺");
}
catch (Exception e)
{ }
}
}
catch (Exception e)
{ }
Thread.Sleep(TimeSpan.FromMinutes(1));
}
}
/// <summary>
/// 验证将要从接口出去的代理IP
/// </summary>
/// <param name="msg"></param>
/// <returns></returns>
public static bool DbVerIp(ProxyViewModel proxy)
{
bool success = false;
Stopwatch sw = new Stopwatch();
sw.Start();
try
{
HttpWebRequest Req;
HttpWebResponse Resp;
WebProxy proxyObject = new WebProxy(proxy.ProxyIP, proxy.ProxyPort); // port为端口号 整数型
Req = WebRequest.Create("https://www.baidu.com") as HttpWebRequest;
Req.Proxy = proxyObject; //设置代理
Req.Timeout = 3000; //超时
Resp = (HttpWebResponse)Req.GetResponse();
Encoding bin = Encoding.GetEncoding("UTF-8");
using (StreamReader sr = new StreamReader(Resp.GetResponseStream(), bin))
{
string str = sr.ReadToEnd();
if (str.Contains("百度"))
{
Resp.Close();
// 更新验证时间
proxy.CreateTime = DateTime.Now;
// 更新验证状态
proxy.State = 1;
// 记录验证状态
success = true;
// 验证通过，归队
QueueOperation(proxy, IQueueType.EnQueue);
}
}
}
catch (Exception e)
{
System.Console.WriteLine("{0}> 接口验证异常{1}", DateTime.Now.ToString("s"), e.Message);
}
sw.Stop();
System.Console.WriteLine("{0}> 接口验证{1} {2} 耗时{3}s", DateTime.Now.ToString("s"), proxy.Id, success, sw.ElapsedMilliseconds / 1000.00);
return(success);
}
/// <summary>
/// 流年
/// </summary>
public static void Liunian()
{
while (true)
{
try
{
string LiunianUrl = "http://www.89ip.cn/";
var html = IWebClient(LiunianUrl);
if (!string.IsNullOrEmpty(html))
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(html);
var table = doc.DocumentNode.SelectSingleNode("//table[@class='layui-table']");
var trlist = table.SelectNodes("//tr").ToList();
for (int i = 1; i < trlist.Count; i++)
{
try
{
var iplist = trlist[i].SelectNodes("td").ToList();
var ip = iplist[0].InnerText.Trim();
var port = iplist[1].InnerText.Trim();
ProxyViewModel proxy = new ProxyViewModel()
{
Id = string.Format("{0}:{1}", ip, port), ProxyIP = ip, ProxyPort = Convert.ToInt32(port), CreateTime = DateTime.Now, State = 0
};
//判断Ip是否已经存在
if (QueueOperation(proxy, IQueueType.Exsist).First().Key > 0)
{
continue;
}
#region 启用多线程去验证
IList <Task> itasks = new List <Task>();
CancellationTokenSource isoure = new CancellationTokenSource();
CancellationToken itoken = isoure.Token;
itasks.Add(new Task(() =>
{
try
{
ProxyViewModel taskProxy = new ProxyViewModel()
{
Id = proxy.Id,
CreateTime = proxy.CreateTime,
ProxyIP = proxy.ProxyIP,
ProxyPort = proxy.ProxyPort,
State = proxy.State
};
ProxyVerification(taskProxy, "流年");
}
catch (Exception ex)
{ }
}, itoken));
itasks[0].Start();
Task.WaitAll(itasks.ToArray(), (4 * 1000), itoken);
#endregion
}
catch (Exception ex)
{ }
}
}
}
catch (Exception e)
{ }
}
}
/// <summary>
/// 队列操作
/// </summary>
/// <returns></returns>
public static Dictionary <int, ProxyViewModel> QueueOperation(ProxyViewModel proxy, IQueueType type)
{
var result = new Dictionary <int, ProxyViewModel>();
lock (_lock)
{
try
{
switch (type)
{
// 归队
case IQueueType.EnQueue:
_ProxyQueue.Enqueue(proxy);
result.Add(1, proxy);
break;
// 出队
case IQueueType.DeQueue:
if (_ProxyQueue.Any())
{
result.Add(1, _ProxyQueue.Dequeue());
}
else
{
result.Add(0, null);
}
break;
// 统计个数
case IQueueType.CountQueue:
result.Add(_ProxyQueue.Count(), null);
break;
// 检测队列是否包含该元素
case IQueueType.Exsist:
var exsistList = _ProxyQueue.Any() ? _ProxyQueue.Where(p => p.Id == proxy.Id).ToList() : new List <ProxyViewModel>();
result.Add(exsistList.Count(), exsistList.Any() ? proxy : null);
break;
// 删除队列指定元素
case IQueueType.Del:
var total = _ProxyQueue.Count();
for (int i = 0; i < total; i++)
{
if (_ProxyQueue.Any())
{
var delProxy = _ProxyQueue.Dequeue();
if (delProxy.Id != proxy.Id)
{
_ProxyQueue.Enqueue(delProxy);
}
else
{
result.Add(1, proxy);
}
}
}
break;
}
}
catch (Exception e)
{
System.Console.WriteLine("{0}> 队列操作异常：{1}，{2}", DateTime.Now.ToString("s"), type.ToString(), e.Message);
}
}
return(result);
}
/// <summary>
/// 验证list集合里面的代理IP
/// </summary>
/// <param name="msg"></param>
public static void ProxyVerification(object msg, string name)
{
if (null == msg)
{
return;
}
ProxyViewModel proxy = (ProxyViewModel)msg;
try
{
using (WebClient web = new WebClient())
{
try
{
HttpWebRequest Req;
HttpWebResponse Resp;
WebProxy proxyObject = new WebProxy(proxy.ProxyIP, proxy.ProxyPort);
Req = WebRequest.Create("https://www.baidu.com") as HttpWebRequest;
Req.Proxy = proxyObject; //设置代理
Req.Timeout = 3000; //超时
Resp = (HttpWebResponse)Req.GetResponse();
Encoding bin = Encoding.GetEncoding("UTF-8");
using (StreamReader sr = new StreamReader(Resp.GetResponseStream(), bin))
{
string str = sr.ReadToEnd();
if (str.Contains("百度"))
{
Resp.Close();
// 更新验证时间
proxy.CreateTime = DateTime.Now;
// 更新验证状态
proxy.State = 1;
// 验证通过，归队
QueueOperation(proxy, IQueueType.EnQueue);
ip_agency_data info = new ip_agency_data()
{
ip_address = proxy.ProxyIP,
ip_port = proxy.ProxyPort.ToString(),
ip_createtime = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")),
ip_sourcename = name
};
//插入到数据库
Program.Add_IPAddress(info);
System.Console.WriteLine("{0}> [{2}]自动验证成功{1}", DateTime.Now.ToString("s"), proxy.Id, name);
}
else
{
System.Console.WriteLine("{0}> [{2}]自动验证失败{1}", DateTime.Now.ToString("s"), proxy.Id, name);
}
}
}
catch (Exception ex)
{ }
}
}
catch (Exception e)
{
System.Console.WriteLine("{0}> [{3}]自动验证异常{1} {2}", DateTime.Now.ToString("s"), proxy.Id, e.Message, name);
}
}