public bool ValidProxy(ProxyDto proxy) { var httpClientHandler = new HttpClientHandler { Proxy = new WebProxy(string.Format("http://{0}:{1}", proxy.IP, proxy.Port), false), UseProxy = true }; var proxyclient = new HttpClient(httpClientHandler); proxyclient.Timeout = new TimeSpan(0, 0, 40); proxyclient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"); var testurl = "https://www.baidu.com"; try { //var html = proxyclient.GetStringAsync(testurl).Result; var task = proxyclient.GetStringAsync(testurl); var html = task.Result; if (string.IsNullOrEmpty(html)) { return(false); } } catch (Exception e) { log.WarnFormat("Valid Proxy Faied. Exception {0}", e.ToString()); return(false); } return(true); }
public async Task Check(ProxyDto dto) { var proxyStatus = await _socket.ConnectAsync(dto.IP, dto.Port, 5000); if (proxyStatus.Connected) { await IncreaseScoreAsync(dto, 1); await _randomPool.AddAsync(dto.IP, dto.Port); } else { if (dto.Score <= 1) { await DeleteProxyAsync(dto); await _randomPool.RemoveAsync(dto.IP, dto.Port); } else { await IncreaseScoreAsync(dto, -1); await _randomPool.AddAsync(dto.IP, dto.Port); } } }
private bool CheckProxyExist(List <ProxyDto> list, ProxyDto dto) { foreach (var item in list) { if (string.Equals(item.IP, dto.IP) && string.Equals(item.Port, dto.Port)) { return(true); } } return(false); }
async Task IncreaseScoreAsync(ProxyDto proxy, int value) { var newScore = proxy.Score + value; if (newScore > MAX_SCORE) { return; } await _dbContext.Database.ExecuteSqlInterpolatedAsync($"UPDATE t_proxys SET score={newScore}, updated_time={DateTime.Now} WHERE id={proxy.Id}"); }
async Task DoWork(ProxyDto dto) { using var scope = _serviceProvider.CreateScope(); try { var proxyCheck = scope.ServiceProvider.GetService <IProxyCheck>(); await proxyCheck.Check(dto); } catch (Exception ex) { _logger.LogError(ex, ex.Message); } }
public List <ProxyDto> GetContent(string url, string ipxpath, string portxpath) { var proxyList = new List <ProxyDto>(); var html = client.GetStringAsync(url).Result; HtmlDocument hd = new HtmlDocument(); hd.LoadHtml(html); HtmlNode root = hd.DocumentNode; HtmlNodeCollection ipNodes = root.SelectNodes(ipxpath); HtmlNodeCollection portNodes = root.SelectNodes(portxpath); for (int j = 0; j < ipNodes.Count; j++) { var proxy = new ProxyDto(); proxy.IP = ipNodes[j].InnerText; proxy.Port = portNodes[j].InnerText; proxyList.Add(proxy); } return(proxyList); }
public async Task AddAsync(ProxyDto dto) { await _context.Proxys.AddAsync(new Proxy { AnonymousDegree = dto.AnonymousDegree, CreatedTime = dto.CreatedTime, IP = dto.IP, Port = dto.Port, Score = dto.Score, UpdatedTime = dto.UpdatedTime }); try { await _context.SaveChangesAsync(); } catch (System.Exception) { throw; } }
public async Task EnqueueTaskItemAsync(ProxyDto dto) { var conn = _connectionMultiplexerFactory.CreateConnectionMultiplexer(); var db = conn.GetDatabase(); await db.ListRightPushAsync(REDIS_CHECK_KEY, JsonConvert.SerializeObject(dto)); }
async Task DeleteProxyAsync(ProxyDto proxy) { await _dbContext.Database.ExecuteSqlInterpolatedAsync($"DELETE FROM t_proxys WHERE id={proxy.Id}"); }