public void Start()
{
foreach (Website website in websites)
{
var thread = new DownloadThread(website.Url, app);
thread.Completed += CheckCompleted;
thread.Start();
threads.Add(thread);
}
}
public static void GetHtmlString(DownloadThread downloader)
{
string result = "";
if (string.IsNullOrEmpty(downloader.Url))
{
return;
}
var sw = new Stopwatch();
sw.Start();
var r = new Random();
string url = downloader.url;
var webRequest = (HttpWebRequest)WebRequest.Create(new Uri(url).ToString());
try
{
downloader.Description = "设置完成，开始访问网页...";
webRequest.Method = "Get";
webRequest.Accept = "*/*";
webRequest.UserAgent = "Mozilla/4.0 (MSIE 6.0; Windows NT " + r.Next(99999) + ")";
webRequest.Headers["Accept-Language"] = "zh-cn";
webRequest.ContentType = "application/x-www-form-urlencoded";
webRequest.Referer = "http://www.loamen.com/404.htm?name=" + Application.ProductName + "&url=" + url;
webRequest.Timeout = downloader.timeOut * 1000;
webRequest.ReadWriteTimeout = downloader.timeOut * 1000;
webRequest.ServicePoint.ConnectionLimit = 100;
WebResponse response = webRequest.GetResponse();
result = GetHtmlString(response, Encoding.Default);
response.Close();
string line = result;
var regex = new Regex(Download.RegexProxy);
MatchCollection matches = regex.Matches(line);
int count = 0;
for (int i = 0; i < matches.Count; i++)
{
string ip = matches[i].Groups["Proxy"].Value;
string port = matches[i].Groups["Port"].Value;
var proxy = new ProxyServer();
proxy.proxy = matches[i].Groups["Proxy"].Value;
proxy.port = Convert.ToInt32(matches[i].Groups["Port"].Value);
proxy.type = "HTTP";
proxy.status = -1;
var regexProxy =
new Regex(
@"(?<Proxy>(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9]))");
if (regexProxy.Matches(proxy.proxy).Count != 1 || proxy.port == 0)
{
return;
}
lock (downloader.app.ProxyList)
{
if (downloader.app.ProxyList.Count(p => p.proxy == proxy.proxy && p.port == proxy.port) == 0)
{
downloader.app.ProxyList.Add(proxy);
}
count++;
}
}
sw.Stop();
downloader.Description = url + "采集完毕，耗时：" + sw.ElapsedMilliseconds + "毫秒，共" + count + "条";
}
catch (Exception ex)
{
sw.Stop();
downloader.Description = url + "出错啦：" + ex.Message + sw.ElapsedMilliseconds;
}
finally
{
webRequest.Abort();
}
}
private static void Downloading(DownloadThread downloader)
{
GetHtmlString(downloader); //httpWebRequest方式
}