Example #1
0
 /// <summary>
 /// 获取西刺首页代理
 /// </summary>
 public static void Xici()
 {
     while (true)
     {
         try
         {
             string       url = "http://www.xicidaili.com/";
             HtmlDocument doc = new HtmlDocument();
             doc.LoadHtml(IWebClient(url));
             var table  = doc.DocumentNode.SelectSingleNode("//table[@id='ip_list']");
             var tdList = table.SelectNodes("//tr").ToList();
             for (int i = 0; i < tdList.Count; i++)
             {
                 try
                 {
                     var td = tdList[i].SelectNodes("td").ToList();
                     if (td.Count != 8)
                     {
                         continue;
                     }
                     var            ip    = td[1].InnerText;
                     int            port  = Convert.ToInt32(td[2].InnerText);
                     ProxyViewModel proxy = new ProxyViewModel()
                     {
                         Id = string.Format("{0}:{1}", ip, port), ProxyIP = ip, ProxyPort = port, CreateTime = DateTime.Now, State = 0
                     };
                     if (QueueOperation(proxy, IQueueType.Exsist).First().Key > 0)
                     {
                         continue;
                     }
                     ProxyVerification(proxy, "西刺");
                 }
                 catch (Exception e)
                 { }
             }
         }
         catch (Exception e)
         { }
         Thread.Sleep(TimeSpan.FromMinutes(1));
     }
 }
Example #2
0
        /// <summary>
        /// 验证将要从接口出去的代理IP
        /// </summary>
        /// <param name="msg"></param>
        /// <returns></returns>
        public static bool DbVerIp(ProxyViewModel proxy)
        {
            bool      success = false;
            Stopwatch sw      = new Stopwatch();

            sw.Start();
            try
            {
                HttpWebRequest  Req;
                HttpWebResponse Resp;
                WebProxy        proxyObject = new WebProxy(proxy.ProxyIP, proxy.ProxyPort); // port为端口号 整数型
                Req         = WebRequest.Create("https://www.baidu.com") as HttpWebRequest;
                Req.Proxy   = proxyObject;                                                  //设置代理
                Req.Timeout = 3000;                                                         //超时
                Resp        = (HttpWebResponse)Req.GetResponse();
                Encoding bin = Encoding.GetEncoding("UTF-8");
                using (StreamReader sr = new StreamReader(Resp.GetResponseStream(), bin))
                {
                    string str = sr.ReadToEnd();
                    if (str.Contains("百度"))
                    {
                        Resp.Close();
                        // 更新验证时间
                        proxy.CreateTime = DateTime.Now;
                        // 更新验证状态
                        proxy.State = 1;
                        // 记录验证状态
                        success = true;
                        // 验证通过,归队
                        QueueOperation(proxy, IQueueType.EnQueue);
                    }
                }
            }
            catch (Exception e)
            {
                System.Console.WriteLine("{0}> 接口验证异常{1}", DateTime.Now.ToString("s"), e.Message);
            }
            sw.Stop();
            System.Console.WriteLine("{0}> 接口验证{1} {2} 耗时{3}s", DateTime.Now.ToString("s"), proxy.Id, success, sw.ElapsedMilliseconds / 1000.00);
            return(success);
        }
Example #3
0
 /// <summary>
 /// 流年
 /// </summary>
 public static void Liunian()
 {
     while (true)
     {
         try
         {
             string LiunianUrl = "http://www.89ip.cn/";
             var    html       = IWebClient(LiunianUrl);
             if (!string.IsNullOrEmpty(html))
             {
                 HtmlDocument doc = new HtmlDocument();
                 doc.LoadHtml(html);
                 var table  = doc.DocumentNode.SelectSingleNode("//table[@class='layui-table']");
                 var trlist = table.SelectNodes("//tr").ToList();
                 for (int i = 1; i < trlist.Count; i++)
                 {
                     try
                     {
                         var            iplist = trlist[i].SelectNodes("td").ToList();
                         var            ip     = iplist[0].InnerText.Trim();
                         var            port   = iplist[1].InnerText.Trim();
                         ProxyViewModel proxy  = new ProxyViewModel()
                         {
                             Id = string.Format("{0}:{1}", ip, port), ProxyIP = ip, ProxyPort = Convert.ToInt32(port), CreateTime = DateTime.Now, State = 0
                         };
                         //判断Ip是否已经存在
                         if (QueueOperation(proxy, IQueueType.Exsist).First().Key > 0)
                         {
                             continue;
                         }
                         #region 启用多线程去验证
                         IList <Task>            itasks = new List <Task>();
                         CancellationTokenSource isoure = new CancellationTokenSource();
                         CancellationToken       itoken = isoure.Token;
                         itasks.Add(new Task(() =>
                         {
                             try
                             {
                                 ProxyViewModel taskProxy = new ProxyViewModel()
                                 {
                                     Id         = proxy.Id,
                                     CreateTime = proxy.CreateTime,
                                     ProxyIP    = proxy.ProxyIP,
                                     ProxyPort  = proxy.ProxyPort,
                                     State      = proxy.State
                                 };
                                 ProxyVerification(taskProxy, "流年");
                             }
                             catch (Exception ex)
                             { }
                         }, itoken));
                         itasks[0].Start();
                         Task.WaitAll(itasks.ToArray(), (4 * 1000), itoken);
                         #endregion
                     }
                     catch (Exception ex)
                     { }
                 }
             }
         }
         catch (Exception e)
         { }
     }
 }
Example #4
0
        /// <summary>
        /// 队列操作
        /// </summary>
        /// <returns></returns>
        public static Dictionary <int, ProxyViewModel> QueueOperation(ProxyViewModel proxy, IQueueType type)
        {
            var result = new Dictionary <int, ProxyViewModel>();

            lock (_lock)
            {
                try
                {
                    switch (type)
                    {
                    // 归队
                    case IQueueType.EnQueue:
                        _ProxyQueue.Enqueue(proxy);
                        result.Add(1, proxy);
                        break;

                    // 出队
                    case IQueueType.DeQueue:
                        if (_ProxyQueue.Any())
                        {
                            result.Add(1, _ProxyQueue.Dequeue());
                        }
                        else
                        {
                            result.Add(0, null);
                        }
                        break;

                    // 统计个数
                    case IQueueType.CountQueue:
                        result.Add(_ProxyQueue.Count(), null);
                        break;

                    // 检测队列是否包含该元素
                    case IQueueType.Exsist:
                        var exsistList = _ProxyQueue.Any() ? _ProxyQueue.Where(p => p.Id == proxy.Id).ToList() : new List <ProxyViewModel>();
                        result.Add(exsistList.Count(), exsistList.Any() ? proxy : null);
                        break;

                    // 删除队列指定元素
                    case IQueueType.Del:
                        var total = _ProxyQueue.Count();
                        for (int i = 0; i < total; i++)
                        {
                            if (_ProxyQueue.Any())
                            {
                                var delProxy = _ProxyQueue.Dequeue();
                                if (delProxy.Id != proxy.Id)
                                {
                                    _ProxyQueue.Enqueue(delProxy);
                                }
                                else
                                {
                                    result.Add(1, proxy);
                                }
                            }
                        }
                        break;
                    }
                }
                catch (Exception e)
                {
                    System.Console.WriteLine("{0}> 队列操作异常:{1},{2}", DateTime.Now.ToString("s"), type.ToString(), e.Message);
                }
            }
            return(result);
        }
Example #5
0
        /// <summary>
        /// 验证list集合里面的代理IP
        /// </summary>
        /// <param name="msg"></param>
        public static void ProxyVerification(object msg, string name)
        {
            if (null == msg)
            {
                return;
            }
            ProxyViewModel proxy = (ProxyViewModel)msg;

            try
            {
                using (WebClient web = new WebClient())
                {
                    try
                    {
                        HttpWebRequest  Req;
                        HttpWebResponse Resp;
                        WebProxy        proxyObject = new WebProxy(proxy.ProxyIP, proxy.ProxyPort);
                        Req         = WebRequest.Create("https://www.baidu.com") as HttpWebRequest;
                        Req.Proxy   = proxyObject; //设置代理
                        Req.Timeout = 3000;        //超时
                        Resp        = (HttpWebResponse)Req.GetResponse();
                        Encoding bin = Encoding.GetEncoding("UTF-8");
                        using (StreamReader sr = new StreamReader(Resp.GetResponseStream(), bin))
                        {
                            string str = sr.ReadToEnd();
                            if (str.Contains("百度"))
                            {
                                Resp.Close();
                                // 更新验证时间
                                proxy.CreateTime = DateTime.Now;
                                // 更新验证状态
                                proxy.State = 1;
                                // 验证通过,归队
                                QueueOperation(proxy, IQueueType.EnQueue);
                                ip_agency_data info = new ip_agency_data()
                                {
                                    ip_address    = proxy.ProxyIP,
                                    ip_port       = proxy.ProxyPort.ToString(),
                                    ip_createtime = Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")),
                                    ip_sourcename = name
                                };
                                //插入到数据库
                                Program.Add_IPAddress(info);

                                System.Console.WriteLine("{0}> [{2}]自动验证成功{1}", DateTime.Now.ToString("s"), proxy.Id, name);
                            }
                            else
                            {
                                System.Console.WriteLine("{0}> [{2}]自动验证失败{1}", DateTime.Now.ToString("s"), proxy.Id, name);
                            }
                        }
                    }
                    catch (Exception ex)
                    { }
                }
            }
            catch (Exception e)
            {
                System.Console.WriteLine("{0}> [{3}]自动验证异常{1} {2}", DateTime.Now.ToString("s"), proxy.Id, e.Message, name);
            }
        }