public NodeClient()
{
_options = new NodeOptions();
_options.Broker = ConfigurationManager.AppSettings["broker"].Trim();
_options.Group = ConfigurationManager.AppSettings["group"].Trim();
_options.Heartbeat = int.Parse(ConfigurationManager.AppSettings["heartbeat"].Trim()) * 1000;
if (int.TryParse(ConfigurationManager.AppSettings["processCount"]?.Trim(), out int processCount))
{
_options.ProcessCount = processCount;
}
_options.RetryDownload = int.Parse(ConfigurationManager.AppSettings["retryDownload"].Trim());
_options.Token = ConfigurationManager.AppSettings["token"]?.Trim();
if (int.TryParse(ConfigurationManager.AppSettings["rertyPush"]?.Trim(), out int rertyPush))
{
_options.RertyPush = rertyPush;
}
_heartbeatUrl = $"{_options.Broker}api/node/heartbeat";
_pushBlockinputUrl = $"{_options.Broker}api/node/blockinput";
_brokerSite = new Site {
Accept = "application/json, text/plain, */*", ContentType = ContentType.Json, EncodingName = "UTF-8", Headers = new Dictionary <string, string> {
{ "User-Agent", "DOTNETSPIDER.NODE" }, { "Token", _options.Token }, { "Accept-Encoding", "gzip" }, { "Content-Type", "application/json" }
}
};
ThreadPool.SetMinThreads(256, 256);
_threadPool = new ThreadCommonPool(_options.ProcessCount.Value);
}
private void RefreshProxies(object threadState)
{
if (ProxyValidator == null)
{
return;
}
ThreadCommonPool threadCommonPool = new ThreadCommonPool(4);
while (!_isDispose)
{
if (_proxyQueue.Count < 50)
{
var proxies = _supplier.GetProxies().GetAwaiter().GetResult();
foreach (var proxy in proxies)
{
threadCommonPool.QueueUserWork(item =>
{
var px = (Proxy)item.Value;
var proxyKey = $"{ px.WebProxy.Address.Host}:{ px.WebProxy.Address.Port}";
bool isExisting = false;
lock (_proxyQueueLocker)
{
isExisting = _proxies.ContainsKey(proxyKey);
}
if (!isExisting)
{
if (ProxyValidator.IsAvailable(item.Value.WebProxy))
{
item.Value.SetFailedNum(0);
item.Value.SetReuseTime(_reuseInterval);
lock (_proxyQueueLocker)
{
if (!_proxies.ContainsKey(proxyKey))
{
_proxyQueue.Add(item.Value);
_proxies.Add(proxyKey, item.Value);
}
}
}
}
}, proxy);
}
}
Thread.Sleep(30000);
}
}
