public static void Main(string[] args) { ProxyPool proxyPool = new ProxyPool(GetProxyList()); proxyPool.SetReuseInterval(10000); proxyPool.SaveProxyList(); while (true) { List <HttpHost> httphostList = new List <HttpHost>(); Console.Read(); int i = 0; while (proxyPool.GetIdleNum() > 2) { HttpHost httphost = proxyPool.GetProxy(); httphostList.Add(httphost); // proxyPool.proxyPool.use(httphost); //Logger.Info("borrow object>>>>" + i + ">>>>" + httphostList[i]); i++; } Console.WriteLine(proxyPool.AllProxyStatus()); Console.Read(); for (i = 0; i < httphostList.Count; i++) { proxyPool.ReturnProxy(httphostList[i], 200); //Logger.Info("return object>>>>" + i + ">>>>" + httphostList[i]); } Console.WriteLine(proxyPool.AllProxyStatus()); Console.Read(); } // ReSharper disable once FunctionNeverReturns }
public static bool ValidateProxy(HttpHost p) { if (_localAddr == null) { //Logger.Error("cannot get local ip"); return(false); } bool isReachable = false; Socket socket = null; try { socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IPv4); socket.Bind(new IPEndPoint(_localAddr, 0)); IPAddress address = IPAddress.Parse(p.Host); socket.SendTimeout = 3000; socket.ReceiveTimeout = 3000; socket.Connect(address, p.Port); //Logger.Info("SUCCESS - connection established! Local: " + _localAddr + " remote: " + p); isReachable = true; } catch (IOException) { //Logger.Warn("FAILRE - CAN not connect! Local: " + _localAddr + " remote: " + p); } finally { try { #if !NET_CORE socket?.Close(); #else socket?.Dispose(); #endif } catch (IOException e) { //Logger.Warn("Error occurred while closing socket of validating proxy", e); } } return(isReachable); }
public static bool ValidateProxy(HttpHost p) { if (_localAddr == null) { //Logger.Error("cannot get local ip"); return false; } bool isReachable = false; Socket socket = null; try { socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IPv4); socket.Bind(new IPEndPoint(_localAddr, 0)); IPAddress address = IPAddress.Parse(p.Host); socket.SendTimeout = 3000; socket.ReceiveTimeout = 3000; socket.Connect(address, p.Port); //Logger.Info("SUCCESS - connection established! Local: " + _localAddr + " remote: " + p); isReachable = true; } catch (IOException) { //Logger.Warn("FAILRE - CAN not connect! Local: " + _localAddr + " remote: " + p); } finally { try { #if !NET_CORE socket?.Close(); #else socket?.Dispose(); #endif } catch (IOException e) { //Logger.Warn("Error occurred while closing socket of validating proxy", e); } } return isReachable; }
public Proxy(HttpHost httpHost) { _httpHost = httpHost; _canReuseTime = DateTimeUtils.GetCurrentTimeStamp() + _reuseTimeInterval * 100; }
public void ReturnProxy(HttpHost host, int statusCode) { Proxy p = _allProxy[host.Host]; if (p == null) { return; } switch (statusCode) { case Proxy.Success: p.SetReuseTimeInterval(_reuseInterval); p.SetFailedNum(0); p.SetFailedErrorType(new List <int>()); p.RecordResponse(); p.SuccessNumIncrement(1); break; case Proxy.Error403: p.Fail(Proxy.Error403); p.SetReuseTimeInterval(_reuseInterval * p.FailedNum); //Logger.Info(host + " >>>> reuseTimeInterval is >>>> " + p.GetReuseTimeInterval() / 1000.0); break; case Proxy.ErrorBanned: p.Fail(Proxy.ErrorBanned); p.SetReuseTimeInterval(10 * 60 * 1000 * p.FailedNum); //Logger.Warn("this proxy is banned >>>> " + p.GetHttpHost()); //Logger.Info(host + " >>>> reuseTimeInterval is >>>> " + p.GetReuseTimeInterval() / 1000.0); break; case Proxy.Error404: p.Fail(Proxy.Error404); p.SetReuseTimeInterval(_reuseInterval * p.FailedNum); break; default: p.Fail(statusCode); break; } if (p.FailedNum > 20) { // allProxy.remove(host.getAddress().getHostAddress()); p.SetReuseTimeInterval(_reviveTime); //Logger.Error($"Remove proxy {host}: p.GetFailedType(). Remain proxy: { _proxyQueue.Count}"); return; } if (p.FailedNum % 5 == 0) { if (!ProxyUtil.ValidateProxy(host)) { // allProxy.remove(host.getAddress().getHostAddress()); p.SetReuseTimeInterval(_reviveTime); //Logger.Error("remove proxy >>>> " + host + ">>>>" + p.GetFailedType() + " >>>> remain proxy >>>> " + _proxyQueue.Count); return; } } try { _proxyQueue.Enqueue(p); } catch (Exception e) { //Logger.Warn("proxyQueue return proxy error", e); } }
public void ReturnHttpProxyToPool(HttpHost proxy, int statusCode) { _httpProxyPool.ReturnProxy(proxy, statusCode); }
public Proxy(HttpHost httpHost) { _httpHost = httpHost; _canReuseTime = DateTime.UtcNow.CurrentTimeMillis() + _reuseTimeInterval * 100; }
public void ReturnProxy(HttpHost host, int statusCode) { Proxy p = _allProxy[host.Host]; if (p == null) { return; } switch (statusCode) { case Proxy.Success: p.SetReuseTimeInterval(_reuseInterval); p.SetFailedNum(0); p.SetFailedErrorType(new List<int>()); p.RecordResponse(); p.SuccessNumIncrement(1); break; case Proxy.Error403: p.Fail(Proxy.Error403); p.SetReuseTimeInterval(_reuseInterval * p.FailedNum); //Logger.Info(host + " >>>> reuseTimeInterval is >>>> " + p.GetReuseTimeInterval() / 1000.0); break; case Proxy.ErrorBanned: p.Fail(Proxy.ErrorBanned); p.SetReuseTimeInterval(10 * 60 * 1000 * p.FailedNum); //Logger.Warn("this proxy is banned >>>> " + p.GetHttpHost()); //Logger.Info(host + " >>>> reuseTimeInterval is >>>> " + p.GetReuseTimeInterval() / 1000.0); break; case Proxy.Error404: p.Fail(Proxy.Error404); p.SetReuseTimeInterval(_reuseInterval * p.FailedNum); break; default: p.Fail(statusCode); break; } if (p.FailedNum > 20) { // allProxy.remove(host.getAddress().getHostAddress()); p.SetReuseTimeInterval(_reviveTime); //Logger.Error($"Remove proxy {host}: p.GetFailedType(). Remain proxy: { _proxyQueue.Count}"); return; } if (p.FailedNum % 5 == 0) { if (!ProxyUtil.ValidateProxy(host)) { // allProxy.remove(host.getAddress().getHostAddress()); p.SetReuseTimeInterval(_reviveTime); //Logger.Error("remove proxy >>>> " + host + ">>>>" + p.GetFailedType() + " >>>> remain proxy >>>> " + _proxyQueue.Count); return; } } try { _proxyQueue.Enqueue(p); } catch (Exception e) { //Logger.Warn("proxyQueue return proxy error", e); } }