public static CreateGetHttpResponse ( string url, int timeout, string userAgent, |
||
url | string | 请求的URL |
timeout | int | 请求的超时时间 |
userAgent | string | 请求的客户端浏览器信息,可以为空 |
cookies | 随同HTTP请求发送的Cookie信息,如果不需要身份验证可以为空 | |
return |
/// <summary> /// 爬取http://www.ishadowsocks.com/的免费账号 /// </summary> /// <returns></returns> private List <Server> crawIShadowsocks() { List <Server> servers = new List <Server>(); try { HttpWebResponse response = HttpWebResponseUtility.CreateGetHttpResponse(ishadowsocks_url, null, null, null); Stream dataStream = response.GetResponseStream(); StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")); string responseFromServer = reader.ReadToEnd(); dataStream.Close(); reader.Close(); response.Close(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(responseFromServer); HtmlNode node = doc.DocumentNode; HtmlNodeCollection datas = node.SelectNodes("//div[@class='col-sm-4 text-center']"); int count = 0; foreach (var data in datas) { MessageBox.Show(data.InnerText); if (count >= 3) { break; } count++; string[] strs = data.InnerText.Split('\n'); MessageBox.Show(strs[1].Trim().Split(':')[1]); Server server = new Server(); server.remarks = CRAWLER_REMARKS; server.server = strs[1].Trim().Split(':')[1]; server.server_port = int.Parse(strs[2].Trim().Split(':')[1]); server.password = strs[3].Trim().Split(':')[1]; server.method = strs[4].Trim().Split(':')[1]; if (server.password.Length > 0) { servers.Add(server); } } return(servers); } catch (Exception) { return(null); } }
/// <summary> /// 爬取https://www.ezlink.hk/的免费账号 /// </summary> /// <returns></returns> private Server crawlEzlink() { Server server = new Server(); server.remarks = CRAWLER_REMARKS; try { HttpWebResponse response = HttpWebResponseUtility.CreateGetHttpResponse(ezlink_url, null, null, null); Stream dataStream = response.GetResponseStream(); StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")); string responseFromServer = reader.ReadToEnd(); dataStream.Close(); reader.Close(); response.Close(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(responseFromServer); HtmlNode node = doc.DocumentNode; HtmlNodeCollection datas = node.SelectNodes("//tr/td"); int count = 0; foreach (var data in datas) { switch (count) { case 13: server.server = data.InnerText; break; case 15: server.server_port = int.Parse(data.InnerText); break; case 17: server.password = data.InnerText; break; case 19: server.method = data.InnerText; break; } count++; } return(server); } catch (Exception) { return(null); } }
/// <summary> /// 爬取http://www.hishadowsocks.com/的免费账号 /// </summary> /// <returns></returns> private Server crawlHiShadowsocks() { Server server = new Server(); server.remarks = CRAWLER_REMARKS; try { HttpWebResponse response = HttpWebResponseUtility.CreateGetHttpResponse(hishadowsocks_url, null, null, null); Stream dataStream = response.GetResponseStream(); StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8")); string responseFromServer = reader.ReadToEnd(); dataStream.Close(); reader.Close(); response.Close(); HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(responseFromServer); HtmlNode node = doc.DocumentNode; HtmlNodeCollection datas = node.SelectNodes("//div[@class='col-md-6']/p"); int count = 0; foreach (var data in datas) { string[] texts = data.InnerText.Split(':'); switch (count) { case 1: server.server = texts[1]; break; case 2: server.server_port = int.Parse(texts[1]); break; case 3: server.password = texts[1]; break; case 4: server.method = texts[1]; break; } count++; } return(server); } catch (Exception) { return(null); } }