public NodeClient() { _options = new NodeOptions(); _options.Broker = ConfigurationManager.AppSettings["broker"].Trim(); _options.Group = ConfigurationManager.AppSettings["group"].Trim(); _options.Heartbeat = int.Parse(ConfigurationManager.AppSettings["heartbeat"].Trim()) * 1000; if (int.TryParse(ConfigurationManager.AppSettings["processCount"]?.Trim(), out int processCount)) { _options.ProcessCount = processCount; } _options.RetryDownload = int.Parse(ConfigurationManager.AppSettings["retryDownload"].Trim()); _options.Token = ConfigurationManager.AppSettings["token"]?.Trim(); if (int.TryParse(ConfigurationManager.AppSettings["rertyPush"]?.Trim(), out int rertyPush)) { _options.RertyPush = rertyPush; } _heartbeatUrl = $"{_options.Broker}api/node/heartbeat"; _pushBlockinputUrl = $"{_options.Broker}api/node/blockinput"; _brokerSite = new Site { Accept = "application/json, text/plain, */*", ContentType = ContentType.Json, EncodingName = "UTF-8", Headers = new Dictionary <string, string> { { "User-Agent", "DOTNETSPIDER.NODE" }, { "Token", _options.Token }, { "Accept-Encoding", "gzip" }, { "Content-Type", "application/json" } } }; ThreadPool.SetMinThreads(256, 256); _threadPool = new ThreadCommonPool(_options.ProcessCount.Value); }
private void RefreshProxies(object threadState) { if (ProxyValidator == null) { return; } ThreadCommonPool threadCommonPool = new ThreadCommonPool(4); while (!_isDispose) { if (_proxyQueue.Count < 50) { var proxies = _supplier.GetProxies().GetAwaiter().GetResult(); foreach (var proxy in proxies) { threadCommonPool.QueueUserWork(item => { var px = (Proxy)item.Value; var proxyKey = $"{ px.WebProxy.Address.Host}:{ px.WebProxy.Address.Port}"; bool isExisting = false; lock (_proxyQueueLocker) { isExisting = _proxies.ContainsKey(proxyKey); } if (!isExisting) { if (ProxyValidator.IsAvailable(item.Value.WebProxy)) { item.Value.SetFailedNum(0); item.Value.SetReuseTime(_reuseInterval); lock (_proxyQueueLocker) { if (!_proxies.ContainsKey(proxyKey)) { _proxyQueue.Add(item.Value); _proxies.Add(proxyKey, item.Value); } } } } }, proxy); } } Thread.Sleep(30000); } }