public static void Main(string[] args)
{
ProxyPool proxyPool = new ProxyPool(GetProxyList());
proxyPool.SetReuseInterval(10000);
proxyPool.SaveProxyList();
while (true)
{
List <HttpHost> httphostList = new List <HttpHost>();
Console.Read();
int i = 0;
while (proxyPool.GetIdleNum() > 2)
{
HttpHost httphost = proxyPool.GetProxy();
httphostList.Add(httphost);
// proxyPool.proxyPool.use(httphost);
//Logger.Info("borrow object>>>>" + i + ">>>>" + httphostList[i]);
i++;
}
Console.WriteLine(proxyPool.AllProxyStatus());
Console.Read();
for (i = 0; i < httphostList.Count; i++)
{
proxyPool.ReturnProxy(httphostList[i], 200);
//Logger.Info("return object>>>>" + i + ">>>>" + httphostList[i]);
}
Console.WriteLine(proxyPool.AllProxyStatus());
Console.Read();
}
// ReSharper disable once FunctionNeverReturns
}
public static bool ValidateProxy(HttpHost p)
{
if (_localAddr == null)
{
//Logger.Error("cannot get local ip");
return(false);
}
bool isReachable = false;
Socket socket = null;
try
{
socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IPv4);
socket.Bind(new IPEndPoint(_localAddr, 0));
IPAddress address = IPAddress.Parse(p.Host);
socket.SendTimeout = 3000;
socket.ReceiveTimeout = 3000;
socket.Connect(address, p.Port);
//Logger.Info("SUCCESS - connection established! Local: " + _localAddr + " remote: " + p);
isReachable = true;
}
catch (IOException)
{
//Logger.Warn("FAILRE - CAN not connect! Local: " + _localAddr + " remote: " + p);
}
finally
{
try
{
#if !NET_CORE
socket?.Close();
#else
socket?.Dispose();
#endif
}
catch (IOException e)
{
//Logger.Warn("Error occurred while closing socket of validating proxy", e);
}
}
return(isReachable);
}
public static bool ValidateProxy(HttpHost p)
{
if (_localAddr == null)
{
//Logger.Error("cannot get local ip");
return false;
}
bool isReachable = false;
Socket socket = null;
try
{
socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IPv4);
socket.Bind(new IPEndPoint(_localAddr, 0));
IPAddress address = IPAddress.Parse(p.Host);
socket.SendTimeout = 3000;
socket.ReceiveTimeout = 3000;
socket.Connect(address, p.Port);
//Logger.Info("SUCCESS - connection established! Local: " + _localAddr + " remote: " + p);
isReachable = true;
}
catch (IOException)
{
//Logger.Warn("FAILRE - CAN not connect! Local: " + _localAddr + " remote: " + p);
}
finally
{
try
{
#if !NET_CORE
socket?.Close();
#else
socket?.Dispose();
#endif
}
catch (IOException e)
{
//Logger.Warn("Error occurred while closing socket of validating proxy", e);
}
}
return isReachable;
}
public Proxy(HttpHost httpHost)
{
_httpHost = httpHost;
_canReuseTime = DateTimeUtils.GetCurrentTimeStamp() + _reuseTimeInterval * 100;
}
public void ReturnProxy(HttpHost host, int statusCode)
{
Proxy p = _allProxy[host.Host];
if (p == null)
{
return;
}
switch (statusCode)
{
case Proxy.Success:
p.SetReuseTimeInterval(_reuseInterval);
p.SetFailedNum(0);
p.SetFailedErrorType(new List <int>());
p.RecordResponse();
p.SuccessNumIncrement(1);
break;
case Proxy.Error403:
p.Fail(Proxy.Error403);
p.SetReuseTimeInterval(_reuseInterval * p.FailedNum);
//Logger.Info(host + " >>>> reuseTimeInterval is >>>> " + p.GetReuseTimeInterval() / 1000.0);
break;
case Proxy.ErrorBanned:
p.Fail(Proxy.ErrorBanned);
p.SetReuseTimeInterval(10 * 60 * 1000 * p.FailedNum);
//Logger.Warn("this proxy is banned >>>> " + p.GetHttpHost());
//Logger.Info(host + " >>>> reuseTimeInterval is >>>> " + p.GetReuseTimeInterval() / 1000.0);
break;
case Proxy.Error404:
p.Fail(Proxy.Error404);
p.SetReuseTimeInterval(_reuseInterval * p.FailedNum);
break;
default:
p.Fail(statusCode);
break;
}
if (p.FailedNum > 20)
{
// allProxy.remove(host.getAddress().getHostAddress());
p.SetReuseTimeInterval(_reviveTime);
//Logger.Error($"Remove proxy {host}: p.GetFailedType(). Remain proxy: { _proxyQueue.Count}");
return;
}
if (p.FailedNum % 5 == 0)
{
if (!ProxyUtil.ValidateProxy(host))
{
// allProxy.remove(host.getAddress().getHostAddress());
p.SetReuseTimeInterval(_reviveTime);
//Logger.Error("remove proxy >>>> " + host + ">>>>" + p.GetFailedType() + " >>>> remain proxy >>>> " + _proxyQueue.Count);
return;
}
}
try
{
_proxyQueue.Enqueue(p);
}
catch (Exception e)
{
//Logger.Warn("proxyQueue return proxy error", e);
}
}
public void ReturnHttpProxyToPool(HttpHost proxy, int statusCode)
{
_httpProxyPool.ReturnProxy(proxy, statusCode);
}
public Proxy(HttpHost httpHost)
{
_httpHost = httpHost;
_canReuseTime = DateTimeUtils.GetCurrentTimeStamp() + _reuseTimeInterval * 100;
}
public Proxy(HttpHost httpHost)
{
_httpHost = httpHost;
_canReuseTime = DateTime.UtcNow.CurrentTimeMillis() + _reuseTimeInterval * 100;
}
public Proxy(HttpHost httpHost)
{
_httpHost = httpHost;
_canReuseTime = DateTime.UtcNow.CurrentTimeMillis() + _reuseTimeInterval * 100;
}
public void ReturnProxy(HttpHost host, int statusCode)
{
Proxy p = _allProxy[host.Host];
if (p == null)
{
return;
}
switch (statusCode)
{
case Proxy.Success:
p.SetReuseTimeInterval(_reuseInterval);
p.SetFailedNum(0);
p.SetFailedErrorType(new List<int>());
p.RecordResponse();
p.SuccessNumIncrement(1);
break;
case Proxy.Error403:
p.Fail(Proxy.Error403);
p.SetReuseTimeInterval(_reuseInterval * p.FailedNum);
//Logger.Info(host + " >>>> reuseTimeInterval is >>>> " + p.GetReuseTimeInterval() / 1000.0);
break;
case Proxy.ErrorBanned:
p.Fail(Proxy.ErrorBanned);
p.SetReuseTimeInterval(10 * 60 * 1000 * p.FailedNum);
//Logger.Warn("this proxy is banned >>>> " + p.GetHttpHost());
//Logger.Info(host + " >>>> reuseTimeInterval is >>>> " + p.GetReuseTimeInterval() / 1000.0);
break;
case Proxy.Error404:
p.Fail(Proxy.Error404);
p.SetReuseTimeInterval(_reuseInterval * p.FailedNum);
break;
default:
p.Fail(statusCode);
break;
}
if (p.FailedNum > 20)
{
// allProxy.remove(host.getAddress().getHostAddress());
p.SetReuseTimeInterval(_reviveTime);
//Logger.Error($"Remove proxy {host}: p.GetFailedType(). Remain proxy: { _proxyQueue.Count}");
return;
}
if (p.FailedNum % 5 == 0)
{
if (!ProxyUtil.ValidateProxy(host))
{
// allProxy.remove(host.getAddress().getHostAddress());
p.SetReuseTimeInterval(_reviveTime);
//Logger.Error("remove proxy >>>> " + host + ">>>>" + p.GetFailedType() + " >>>> remain proxy >>>> " + _proxyQueue.Count);
return;
}
}
try
{
_proxyQueue.Enqueue(p);
}
catch (Exception e)
{
//Logger.Warn("proxyQueue return proxy error", e);
}
}