private void RefreshProxies()
{
var result = _supplier.GetProxies();
Parallel.ForEach(result, new ParallelOptions
{
MaxDegreeOfParallelism = 4
}, proxy =>
{
var key = proxy.Key;
var value = proxy.Value;
if (_allProxy.ContainsKey(key))
{
return;
}
if (ProxyUtil.ValidateProxy(proxy.Value.HttpHost.Uri.Host, proxy.Value.HttpHost.Uri.Port))
{
value.SetFailedNum(0);
value.SetReuseTime(_reuseInterval);
lock (this)
{
_proxyQueue.Add(value);
}
_allProxy.GetOrAdd(key, value);
}
});
}
private void UpdateProxy()
{
var result = _supplier.GetProxies();
Parallel.ForEach(result, new ParallelOptions
{
MaxDegreeOfParallelism = 10
}, proxy =>
{
var key = proxy.Key;
var value = proxy.Value;
if (_allProxy.ContainsKey(key))
{
return;
}
if (ProxyUtil.ValidateProxy(proxy.Value.HttpHost.Uri.Host, proxy.Value.HttpHost.Uri.Port))
{
LogCenter.Log(null, $"Detect one usefull proxy: {key}", LogLevel.Debug);
value.SetFailedNum(0);
value.SetReuseTime(_reuseInterval);
_proxyQueue.Add(value);
_allProxy.GetOrAdd(key, value);
}
});
}
public void ReturnProxy(UseSpecifiedUriWebProxy host, HttpStatusCode statusCode)
{
if (host == null)
{
return;
}
var key = $"{host.Uri.Host}:{host.Uri.Port}";
if (!_allProxy.ContainsKey(key))
{
return;
}
Proxy p = _allProxy[key];
switch (statusCode)
{
case HttpStatusCode.OK:
p.SetFailedNum(0);
p.SetReuseTime(_reuseInterval);
p.RecordResponse();
break;
case HttpStatusCode.Forbidden:
p.Fail();
p.SetReuseTime(_reuseInterval * p.FailedNum);
break;
case HttpStatusCode.NotFound:
p.Fail();
p.SetReuseTime(_reuseInterval * p.FailedNum);
break;
default:
p.Fail();
p.SetReuseTime(_reuseInterval * p.FailedNum);
break;
}
if (p.FailedNum > 20)
{
return;
}
if (p.FailedNum % 3 == 0 && !ProxyUtil.ValidateProxy(p.HttpHost.Uri.Host, p.HttpHost.Uri.Port))
{
return;
}
lock (this)
{
_proxyQueue.Add(p);
}
}
