private void RefreshProxies()
{
if (ProxyValidator != null)
{
var result = _supplier.GetProxies();
Parallel.ForEach(result, new ParallelOptions
{
MaxDegreeOfParallelism = 4
}, proxy =>
{
var key = proxy.Key;
var value = proxy.Value;
if (_allProxy.ContainsKey(key))
{
return;
}
if (ProxyValidator.IsAvailable(proxy.Value.WebProxy))
{
value.SetFailedNum(0);
value.SetReuseTime(_reuseInterval);
lock (this)
{
_proxyQueue.Add(value);
}
_allProxy.GetOrAdd(key, value);
}
});
}
}
private void RefreshProxies()
{
var result = _supplier.GetProxies();
Parallel.ForEach(result, new ParallelOptions
{
MaxDegreeOfParallelism = 4
}, proxy =>
{
var key = proxy.Key;
var value = proxy.Value;
if (_allProxy.ContainsKey(key))
{
return;
}
if (ProxyUtil.ValidateProxy(proxy.Value.HttpHost.Uri.Host, proxy.Value.HttpHost.Uri.Port))
{
value.SetFailedNum(0);
value.SetReuseTime(_reuseInterval);
lock (this)
{
_proxyQueue.Add(value);
}
_allProxy.GetOrAdd(key, value);
}
});
}
private void UpdateProxy()
{
var result = _supplier.GetProxies();
Parallel.ForEach(result, new ParallelOptions
{
MaxDegreeOfParallelism = 10
}, proxy =>
{
var key = proxy.Key;
var value = proxy.Value;
if (_allProxy.ContainsKey(key))
{
return;
}
if (ProxyUtil.ValidateProxy(proxy.Value.HttpHost.Uri.Host, proxy.Value.HttpHost.Uri.Port))
{
LogCenter.Log(null, $"Detect one usefull proxy: {key}", LogLevel.Debug);
value.SetFailedNum(0);
value.SetReuseTime(_reuseInterval);
_proxyQueue.Add(value);
_allProxy.GetOrAdd(key, value);
}
});
}
private void RefreshProxies(object threadState)
{
if (ProxyValidator == null)
{
return;
}
ThreadCommonPool threadCommonPool = new ThreadCommonPool(4);
while (!_isDispose)
{
if (_proxyQueue.Count < 50)
{
var proxies = _supplier.GetProxies().GetAwaiter().GetResult();
foreach (var proxy in proxies)
{
threadCommonPool.QueueUserWork(item =>
{
var px = (Proxy)item.Value;
var proxyKey = $"{ px.WebProxy.Address.Host}:{ px.WebProxy.Address.Port}";
bool isExisting = false;
lock (_proxyQueueLocker)
{
isExisting = _proxies.ContainsKey(proxyKey);
}
if (!isExisting)
{
if (ProxyValidator.IsAvailable(item.Value.WebProxy))
{
item.Value.SetFailedNum(0);
item.Value.SetReuseTime(_reuseInterval);
lock (_proxyQueueLocker)
{
if (!_proxies.ContainsKey(proxyKey))
{
_proxyQueue.Add(item.Value);
_proxies.Add(proxyKey, item.Value);
}
}
}
}
}, proxy);
}
}
Thread.Sleep(30000);
}
}
private void RefreshProxies(object threadState)
{
if (ProxyValidator == null)
{
return;
}
var threadCommonPool = new LimitedConcurrencyThreadPool(4);
while (!_isDispose)
{
if (_proxyQueue.Count < 50)
{
var proxies = _supplier.GetProxies().GetAwaiter().GetResult();
foreach (var proxy in proxies)
{
threadCommonPool.QueueUserWork(async item =>
{
if (!_proxies.ContainsKey(item.Key))
{
if (await ProxyValidator.IsAvailable(item.Value.WebProxy))
{
_logger.LogInformation("");
item.Value.SetFailedNum(0);
item.Value.SetReuseTime(_reuseInterval);
lock (_proxyQueueLocker)
{
_proxyQueue.Add(item.Value);
}
_proxies.Add(item.Key, item.Value);
_logger.LogInformation($"Acquired available proxy {proxy.Value.WebProxy.Address}");
}
}
}, proxy);
}
}
Thread.Sleep(30000);
}
}
private void RefreshProxies(object threadState)
{
if (ProxyValidator == null)
{
return;
}
ThreadCommonPool threadCommonPool = new ThreadCommonPool(4);
while (!_isDispose)
{
if (_proxyQueue.Count < 50)
{
var proxies = _supplier.GetProxies();
foreach (var proxy in proxies)
{
threadCommonPool.QueueUserWork(item =>
{
if (!_proxies.ContainsKey(item.Key))
{
if (ProxyValidator.IsAvailable(item.Value.WebProxy))
{
item.Value.SetFailedNum(0);
item.Value.SetReuseTime(_reuseInterval);
lock (_proxyQueueLocker)
{
_proxyQueue.Add(item.Value);
}
_proxies.Add(item.Key, item.Value);
}
}
}, proxy);
}
}
Thread.Sleep(30000);
}
}
