コード例 #1
0
ファイル: Downloader.cs プロジェクト: zhangjingpu/ProxyHero
 public void Start()
 {
     foreach (Website website in websites)
     {
         var thread = new DownloadThread(website.Url, app);
         thread.Completed += CheckCompleted;
         thread.Start();
         threads.Add(thread);
     }
 }
コード例 #2
0
        public static void GetHtmlString(DownloadThread downloader)
        {
            string result = "";

            if (string.IsNullOrEmpty(downloader.Url))
            {
                return;
            }
            var sw = new Stopwatch();

            sw.Start();


            var    r   = new Random();
            string url = downloader.url;

            var webRequest = (HttpWebRequest)WebRequest.Create(new Uri(url).ToString());

            try
            {
                downloader.Description = "设置完成,开始访问网页...";

                webRequest.Method    = "Get";
                webRequest.Accept    = "*/*";
                webRequest.UserAgent = "Mozilla/4.0 (MSIE 6.0; Windows NT " + r.Next(99999) + ")";
                webRequest.Headers["Accept-Language"] = "zh-cn";
                webRequest.ContentType                  = "application/x-www-form-urlencoded";
                webRequest.Referer                      = "http://www.loamen.com/404.htm?name=" + Application.ProductName + "&url=" + url;
                webRequest.Timeout                      = downloader.timeOut * 1000;
                webRequest.ReadWriteTimeout             = downloader.timeOut * 1000;
                webRequest.ServicePoint.ConnectionLimit = 100;

                WebResponse response = webRequest.GetResponse();
                result = GetHtmlString(response, Encoding.Default);
                response.Close();
                string          line    = result;
                var             regex   = new Regex(Download.RegexProxy);
                MatchCollection matches = regex.Matches(line);

                int count = 0;
                for (int i = 0; i < matches.Count; i++)
                {
                    string ip   = matches[i].Groups["Proxy"].Value;
                    string port = matches[i].Groups["Port"].Value;

                    var proxy = new ProxyServer();

                    proxy.proxy  = matches[i].Groups["Proxy"].Value;
                    proxy.port   = Convert.ToInt32(matches[i].Groups["Port"].Value);
                    proxy.type   = "HTTP";
                    proxy.status = -1;

                    var regexProxy =
                        new Regex(
                            @"(?<Proxy>(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9]))");
                    if (regexProxy.Matches(proxy.proxy).Count != 1 || proxy.port == 0)
                    {
                        return;
                    }
                    lock (downloader.app.ProxyList)
                    {
                        if (downloader.app.ProxyList.Count(p => p.proxy == proxy.proxy && p.port == proxy.port) == 0)
                        {
                            downloader.app.ProxyList.Add(proxy);
                        }
                        count++;
                    }
                }
                sw.Stop();
                downloader.Description = url + "采集完毕,耗时:" + sw.ElapsedMilliseconds + "毫秒,共" + count + "条";
            }
            catch (Exception ex)
            {
                sw.Stop();
                downloader.Description = url + "出错啦:" + ex.Message + sw.ElapsedMilliseconds;
            }
            finally
            {
                webRequest.Abort();
            }
        }
コード例 #3
0
 private static void Downloading(DownloadThread downloader)
 {
     GetHtmlString(downloader); //httpWebRequest方式
 }