|
public static CreateGetHttpResponse ( string url, int timeout, string userAgent, |
||
| url | string | 请求的URL |
| timeout | int | 请求的超时时间 |
| userAgent | string | 请求的客户端浏览器信息，可以为空 |
| cookies | 随同HTTP请求发送的Cookie信息，如果不需要身份验证可以为空 | |
| return | ||
/// <summary>
/// 爬取http://www.ishadowsocks.com/的免费账号
/// </summary>
/// <returns></returns>
private List <Server> crawIShadowsocks()
{
List <Server> servers = new List <Server>();
try
{
HttpWebResponse response = HttpWebResponseUtility.CreateGetHttpResponse(ishadowsocks_url, null, null, null);
Stream dataStream = response.GetResponseStream();
StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8"));
string responseFromServer = reader.ReadToEnd();
dataStream.Close();
reader.Close();
response.Close();
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(responseFromServer);
HtmlNode node = doc.DocumentNode;
HtmlNodeCollection datas = node.SelectNodes("//div[@class='col-sm-4 text-center']");
int count = 0;
foreach (var data in datas)
{
MessageBox.Show(data.InnerText);
if (count >= 3)
{
break;
}
count++;
string[] strs = data.InnerText.Split('\n');
MessageBox.Show(strs[1].Trim().Split(':')[1]);
Server server = new Server();
server.remarks = CRAWLER_REMARKS;
server.server = strs[1].Trim().Split(':')[1];
server.server_port = int.Parse(strs[2].Trim().Split(':')[1]);
server.password = strs[3].Trim().Split(':')[1];
server.method = strs[4].Trim().Split(':')[1];
if (server.password.Length > 0)
{
servers.Add(server);
}
}
return(servers);
}
catch (Exception)
{
return(null);
}
}
/// <summary>
/// 爬取https://www.ezlink.hk/的免费账号
/// </summary>
/// <returns></returns>
private Server crawlEzlink()
{
Server server = new Server();
server.remarks = CRAWLER_REMARKS;
try
{
HttpWebResponse response = HttpWebResponseUtility.CreateGetHttpResponse(ezlink_url, null, null, null);
Stream dataStream = response.GetResponseStream();
StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8"));
string responseFromServer = reader.ReadToEnd();
dataStream.Close();
reader.Close();
response.Close();
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(responseFromServer);
HtmlNode node = doc.DocumentNode;
HtmlNodeCollection datas = node.SelectNodes("//tr/td");
int count = 0;
foreach (var data in datas)
{
switch (count)
{
case 13: server.server = data.InnerText; break;
case 15: server.server_port = int.Parse(data.InnerText); break;
case 17: server.password = data.InnerText; break;
case 19: server.method = data.InnerText; break;
}
count++;
}
return(server);
}
catch (Exception)
{
return(null);
}
}
/// <summary>
/// 爬取http://www.hishadowsocks.com/的免费账号
/// </summary>
/// <returns></returns>
private Server crawlHiShadowsocks()
{
Server server = new Server();
server.remarks = CRAWLER_REMARKS;
try
{
HttpWebResponse response = HttpWebResponseUtility.CreateGetHttpResponse(hishadowsocks_url, null, null, null);
Stream dataStream = response.GetResponseStream();
StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding("utf-8"));
string responseFromServer = reader.ReadToEnd();
dataStream.Close();
reader.Close();
response.Close();
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(responseFromServer);
HtmlNode node = doc.DocumentNode;
HtmlNodeCollection datas = node.SelectNodes("//div[@class='col-md-6']/p");
int count = 0;
foreach (var data in datas)
{
string[] texts = data.InnerText.Split(':');
switch (count)
{
case 1: server.server = texts[1]; break;
case 2: server.server_port = int.Parse(texts[1]); break;
case 3: server.password = texts[1]; break;
case 4: server.method = texts[1]; break;
}
count++;
}
return(server);
}
catch (Exception)
{
return(null);
}
}