// 下载快代理 public void Downkuaidaili(object DATA) { try { string url = "http://www.kuaidaili.com/proxylist/"; for (int i = 1; i < 4; i++) { string html = HttpHelper.DownloadHtml(url + i.ToString() + "/", null); string xpath = "//tbody/tr"; HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(html); HtmlNode node = doc.DocumentNode; HtmlNodeCollection collection = node.SelectNodes(xpath); foreach (var item in collection) { var proxy = new IpProxy(); proxy.Address = item.FirstChild.InnerHtml; xpath = "td[2]"; proxy.Port = int.Parse(item.SelectSingleNode(xpath).InnerHtml); Task.Run(() => { PoolManageService.Add(proxy); }); } } } catch (Exception e) { LogUtils.ErrorLog(new Exception("下载快代理IP池出现故障", e)); } }
internal static void Add(IpProxy proxy) { proxy.FullAddress = string.Format("{0}:{1}", proxy.Address, proxy.Port); var count = IpProxyManage.Instance.Count(b => b.FullAddress == proxy.FullAddress); if (count == 0) { IpProxyManage.Instance.Add(proxy); } }
// 下载66免费代理 public void Downloadproxy66(object DATA) { try { List <string> list = new List <string>() { "http://www.66ip.cn/areaindex_35/index.html", "http://www.66ip.cn/areaindex_35/2.html", "http://www.66ip.cn/areaindex_35/3.html" }; foreach (var utlitem in list) { string url = utlitem; string html = HttpHelper.DownloadHtml(url, null); if (string.IsNullOrEmpty(html)) { LogUtils.ErrorLog(new Exception("代理地址:" + url + " 访问失败")); break; } HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(html); HtmlNode node = doc.DocumentNode; string xpathstring = "//table[@bordercolor='#6699ff']/tr"; HtmlNodeCollection collection = node.SelectNodes(xpathstring); foreach (var item in collection) { var proxy = new IpProxy(); var xpath = "td[1]"; proxy.Address = item.SelectSingleNode(xpath).InnerHtml; if (proxy.Address.Contains("ip")) { continue; } xpath = "td[2]"; proxy.Port = int.Parse(item.SelectSingleNode(xpath).InnerHtml); Task.Run(() => { PoolManageService.Add(proxy); }); } } } catch (Exception e) { LogUtils.ErrorLog(new Exception("下载66免费代理IP池出现故障", e)); } }
// 下载逼格代理 public void DownloadproxyBiGe(object DATA) { try { List <string> list = new List <string>() { "http://www.bigdaili.com/dailiip/1/{0}.html", "http://www.bigdaili.com/dailiip/2/{0}.html", "http://www.bigdaili.com/dailiip/3/{0}.html", "http://www.bigdaili.com/dailiip/4/{0}.html" }; foreach (var utlitem in list) { for (int i = 1; i < 5; i++) { string url = String.Format(utlitem, i); string html = HttpHelper.DownloadHtml(url, null); if (string.IsNullOrEmpty(html)) { LogUtils.ErrorLog(new Exception("代理地址:" + url + " 访问失败")); continue; } HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(html); HtmlNode node = doc.DocumentNode; string xpathstring = "//tbody/tr"; HtmlNodeCollection collection = node.SelectNodes(xpathstring); foreach (var item in collection) { var proxy = new IpProxy(); var xpath = "td[1]"; proxy.Address = item.SelectSingleNode(xpath).InnerHtml; xpath = "td[2]"; proxy.Port = int.Parse(item.SelectSingleNode(xpath).InnerHtml); Task.Run(() => { PoolManageService.Add(proxy); }); } } } } catch (Exception e) { LogUtils.ErrorLog(new Exception("下载逼格代理IP池出现故障", e)); } }
public static bool Test(IpProxy proxy, out string error) { error = ""; try { //var ms = HttpHelper.DownloadHtml("http://news.163.com/", proxy.FullAddress); var ip = IPAddress.Parse(proxy.Address); Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); sock.SendTimeout = 500; sock.Connect(ip, proxy.Port); sock.Close(); proxy.CheckResult = true; } catch (Exception ero) { error = ero.Message; proxy.CheckResult = false; } proxy.CheckTime = DateTime.Now; IpProxyManage.Instance.Update(proxy); return(proxy.CheckResult); }
// 下载proxy360 public void Downloadproxy360(object DATA) { try { string url = "http://www.proxy360.cn/default.aspx"; string html = HttpHelper.DownloadHtml(url, null); if (string.IsNullOrEmpty(html)) { LogUtils.ErrorLog(new Exception("代理地址:" + url + " 访问失败")); return; } HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(html); string xpathstring = "//div[@class='proxylistitem']"; HtmlNode node = doc.DocumentNode; HtmlNodeCollection collection = node.SelectNodes(xpathstring); foreach (var item in collection) { var proxy = new IpProxy(); var childnode = item.ChildNodes[1]; xpathstring = "span[1]"; proxy.Address = childnode.SelectSingleNode(xpathstring).InnerHtml.Trim(); xpathstring = "span[2]"; proxy.Port = int.Parse(childnode.SelectSingleNode(xpathstring).InnerHtml); Task.Run(() => { PoolManageService.Add(proxy); }); } } catch (Exception e) { LogUtils.ErrorLog(new Exception("下载proxy360IP池出现故障", e)); } }
// 下载西刺代理的html页面 public void Downloadxicidaili(object DATA) { try { List <string> list = new List <string>() { "http://www.xicidaili.com/nt/", "http://www.xicidaili.com/nn/", "http://www.xicidaili.com/wn/", "http://www.xicidaili.com/wt/" }; foreach (var utlitem in list) { for (int i = 1; i < 5; i++) { string url = utlitem + i.ToString(); //var ipProxy = PoolManageService.GetProxy(); //if (string.IsNullOrEmpty(ipProxy)) //{ // LogUtils.ErrorLog(new Exception("Ip代理池暂无可用代理IP")); // return; //} //var ip = ipProxy; //WebProxy webproxy; //if (ipProxy.Contains(":")) //{ // ip = ipProxy.Split(new[] { ':' })[0]; // var port = int.Parse(ipProxy.Split(new[] { ':' })[1]); // webproxy = new WebProxy(ip, port); //} //else //{ // webproxy = new WebProxy(ip); //} string html = HttpHelper.DownloadHtml(url, null); if (string.IsNullOrEmpty(html)) { LogUtils.ErrorLog(new Exception("代理地址:" + url + " 访问失败")); continue; } HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(html); HtmlNode node = doc.DocumentNode; string xpathstring = "//tr[@class='odd']"; HtmlNodeCollection collection = node.SelectNodes(xpathstring); foreach (var item in collection) { var proxy = new IpProxy(); string xpath = "td[2]"; proxy.Address = item.SelectSingleNode(xpath).InnerHtml; xpath = "td[3]"; proxy.Port = int.Parse(item.SelectSingleNode(xpath).InnerHtml); Task.Run(() => { PoolManageService.Add(proxy); }); } } } } catch (Exception e) { LogUtils.ErrorLog(new Exception("下载西刺代理IP池出现故障", e)); } }