public static void Main(string[] args)
        {
            ProxyPool proxyPool = new ProxyPool(GetProxyList());

            proxyPool.SetReuseInterval(10000);
            proxyPool.SaveProxyList();

            while (true)
            {
                List <HttpHost> httphostList = new List <HttpHost>();
                Console.Read();
                int i = 0;
                while (proxyPool.GetIdleNum() > 2)
                {
                    HttpHost httphost = proxyPool.GetProxy();
                    httphostList.Add(httphost);
                    // proxyPool.proxyPool.use(httphost);
                    //Logger.Info("borrow object>>>>" + i + ">>>>" + httphostList[i]);
                    i++;
                }
                Console.WriteLine(proxyPool.AllProxyStatus());
                Console.Read();
                for (i = 0; i < httphostList.Count; i++)
                {
                    proxyPool.ReturnProxy(httphostList[i], 200);
                    //Logger.Info("return object>>>>" + i + ">>>>" + httphostList[i]);
                }
                Console.WriteLine(proxyPool.AllProxyStatus());
                Console.Read();
            }
            // ReSharper disable once FunctionNeverReturns
        }
        public static bool ValidateProxy(HttpHost p)
        {
            if (_localAddr == null)
            {
                //Logger.Error("cannot get local ip");
                return(false);
            }
            bool   isReachable = false;
            Socket socket      = null;

            try
            {
                socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IPv4);

                socket.Bind(new IPEndPoint(_localAddr, 0));
                IPAddress address = IPAddress.Parse(p.Host);
                socket.SendTimeout    = 3000;
                socket.ReceiveTimeout = 3000;
                socket.Connect(address, p.Port);

                //Logger.Info("SUCCESS - connection established! Local: " + _localAddr + " remote: " + p);
                isReachable = true;
            }
            catch (IOException)
            {
                //Logger.Warn("FAILRE - CAN not connect! Local: " + _localAddr + " remote: " + p);
            }
            finally
            {
                try
                {
#if !NET_CORE
                    socket?.Close();
#else
                    socket?.Dispose();
#endif
                }
                catch (IOException e)
                {
                    //Logger.Warn("Error occurred while closing socket of validating proxy", e);
                }
            }
            return(isReachable);
        }
Exemple #3
0
        public static bool ValidateProxy(HttpHost p)
        {
            if (_localAddr == null)
            {
                //Logger.Error("cannot get local ip");
                return false;
            }
            bool isReachable = false;
            Socket socket = null;
            try
            {
                socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IPv4);

                socket.Bind(new IPEndPoint(_localAddr, 0));
                IPAddress address = IPAddress.Parse(p.Host);
                socket.SendTimeout = 3000;
                socket.ReceiveTimeout = 3000;
                socket.Connect(address, p.Port);

                //Logger.Info("SUCCESS - connection established! Local: " + _localAddr + " remote: " + p);
                isReachable = true;
            }
            catch (IOException)
            {
                //Logger.Warn("FAILRE - CAN not connect! Local: " + _localAddr + " remote: " + p);
            }
            finally
            {
                    try
                    {
            #if !NET_CORE
                        socket?.Close();
            #else
                        socket?.Dispose();
            #endif
                    }
                    catch (IOException e)
                    {
                        //Logger.Warn("Error occurred while closing socket of validating proxy", e);
                    }
            }
            return isReachable;
        }
Exemple #4
0
        public Proxy(HttpHost httpHost)
        {
            _httpHost = httpHost;

            _canReuseTime = DateTimeUtils.GetCurrentTimeStamp() + _reuseTimeInterval * 100;
        }
        public void ReturnProxy(HttpHost host, int statusCode)
        {
            Proxy p = _allProxy[host.Host];

            if (p == null)
            {
                return;
            }
            switch (statusCode)
            {
            case Proxy.Success:
                p.SetReuseTimeInterval(_reuseInterval);
                p.SetFailedNum(0);
                p.SetFailedErrorType(new List <int>());
                p.RecordResponse();
                p.SuccessNumIncrement(1);
                break;

            case Proxy.Error403:
                p.Fail(Proxy.Error403);
                p.SetReuseTimeInterval(_reuseInterval * p.FailedNum);
                //Logger.Info(host + " >>>> reuseTimeInterval is >>>> " + p.GetReuseTimeInterval() / 1000.0);
                break;

            case Proxy.ErrorBanned:
                p.Fail(Proxy.ErrorBanned);
                p.SetReuseTimeInterval(10 * 60 * 1000 * p.FailedNum);
                //Logger.Warn("this proxy is banned >>>> " + p.GetHttpHost());
                //Logger.Info(host + " >>>> reuseTimeInterval is >>>> " + p.GetReuseTimeInterval() / 1000.0);
                break;

            case Proxy.Error404:
                p.Fail(Proxy.Error404);
                p.SetReuseTimeInterval(_reuseInterval * p.FailedNum);
                break;

            default:
                p.Fail(statusCode);
                break;
            }
            if (p.FailedNum > 20)
            {
                // allProxy.remove(host.getAddress().getHostAddress());
                p.SetReuseTimeInterval(_reviveTime);
                //Logger.Error($"Remove proxy {host}: p.GetFailedType(). Remain proxy: { _proxyQueue.Count}");
                return;
            }
            if (p.FailedNum % 5 == 0)
            {
                if (!ProxyUtil.ValidateProxy(host))
                {
                    // allProxy.remove(host.getAddress().getHostAddress());
                    p.SetReuseTimeInterval(_reviveTime);
                    //Logger.Error("remove proxy >>>> " + host + ">>>>" + p.GetFailedType() + " >>>> remain proxy >>>> " + _proxyQueue.Count);
                    return;
                }
            }
            try
            {
                _proxyQueue.Enqueue(p);
            }
            catch (Exception e)
            {
                //Logger.Warn("proxyQueue return proxy error", e);
            }
        }
Exemple #6
0
		public void ReturnHttpProxyToPool(HttpHost proxy, int statusCode)
		{
			_httpProxyPool.ReturnProxy(proxy, statusCode);
		}
Exemple #7
0
        public Proxy(HttpHost httpHost)
        {
            _httpHost = httpHost;

            _canReuseTime = DateTimeUtils.GetCurrentTimeStamp() + _reuseTimeInterval * 100;
        }
Exemple #8
0
        public Proxy(HttpHost httpHost)
        {
            _httpHost = httpHost;

            _canReuseTime = DateTime.UtcNow.CurrentTimeMillis() + _reuseTimeInterval * 100;
        }
Exemple #9
0
        public Proxy(HttpHost httpHost)
        {
            _httpHost = httpHost;

            _canReuseTime = DateTime.UtcNow.CurrentTimeMillis() + _reuseTimeInterval * 100;
        }
Exemple #10
0
 public void ReturnProxy(HttpHost host, int statusCode)
 {
     Proxy p = _allProxy[host.Host];
     if (p == null)
     {
         return;
     }
     switch (statusCode)
     {
         case Proxy.Success:
             p.SetReuseTimeInterval(_reuseInterval);
             p.SetFailedNum(0);
             p.SetFailedErrorType(new List<int>());
             p.RecordResponse();
             p.SuccessNumIncrement(1);
             break;
         case Proxy.Error403:
             p.Fail(Proxy.Error403);
             p.SetReuseTimeInterval(_reuseInterval * p.FailedNum);
             //Logger.Info(host + " >>>> reuseTimeInterval is >>>> " + p.GetReuseTimeInterval() / 1000.0);
             break;
         case Proxy.ErrorBanned:
             p.Fail(Proxy.ErrorBanned);
             p.SetReuseTimeInterval(10 * 60 * 1000 * p.FailedNum);
             //Logger.Warn("this proxy is banned >>>> " + p.GetHttpHost());
             //Logger.Info(host + " >>>> reuseTimeInterval is >>>> " + p.GetReuseTimeInterval() / 1000.0);
             break;
         case Proxy.Error404:
             p.Fail(Proxy.Error404);
             p.SetReuseTimeInterval(_reuseInterval * p.FailedNum);
             break;
         default:
             p.Fail(statusCode);
             break;
     }
     if (p.FailedNum > 20)
     {
         // allProxy.remove(host.getAddress().getHostAddress());
         p.SetReuseTimeInterval(_reviveTime);
         //Logger.Error($"Remove proxy {host}: p.GetFailedType(). Remain proxy: { _proxyQueue.Count}");
         return;
     }
     if (p.FailedNum % 5 == 0)
     {
         if (!ProxyUtil.ValidateProxy(host))
         {
             // allProxy.remove(host.getAddress().getHostAddress());
             p.SetReuseTimeInterval(_reviveTime);
             //Logger.Error("remove proxy >>>> " + host + ">>>>" + p.GetFailedType() + " >>>> remain proxy >>>> " + _proxyQueue.Count);
             return;
         }
     }
     try
     {
         _proxyQueue.Enqueue(p);
     }
     catch (Exception e)
     {
         //Logger.Warn("proxyQueue return proxy error", e);
     }
 }