public bool ValidProxy(ProxyDto proxy)
{
var httpClientHandler = new HttpClientHandler
{
Proxy = new WebProxy(string.Format("http://{0}:{1}", proxy.IP, proxy.Port), false),
UseProxy = true
};
var proxyclient = new HttpClient(httpClientHandler);
proxyclient.Timeout = new TimeSpan(0, 0, 40);
proxyclient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36");
var testurl = "https://www.baidu.com";
try
{
//var html = proxyclient.GetStringAsync(testurl).Result;
var task = proxyclient.GetStringAsync(testurl);
var html = task.Result;
if (string.IsNullOrEmpty(html))
{
return(false);
}
}
catch (Exception e)
{
log.WarnFormat("Valid Proxy Faied. Exception {0}", e.ToString());
return(false);
}
return(true);
}
public async Task Check(ProxyDto dto)
{
var proxyStatus = await _socket.ConnectAsync(dto.IP, dto.Port, 5000);
if (proxyStatus.Connected)
{
await IncreaseScoreAsync(dto, 1);
await _randomPool.AddAsync(dto.IP, dto.Port);
}
else
{
if (dto.Score <= 1)
{
await DeleteProxyAsync(dto);
await _randomPool.RemoveAsync(dto.IP, dto.Port);
}
else
{
await IncreaseScoreAsync(dto, -1);
await _randomPool.AddAsync(dto.IP, dto.Port);
}
}
}
private bool CheckProxyExist(List <ProxyDto> list, ProxyDto dto)
{
foreach (var item in list)
{
if (string.Equals(item.IP, dto.IP) && string.Equals(item.Port, dto.Port))
{
return(true);
}
}
return(false);
}
async Task IncreaseScoreAsync(ProxyDto proxy, int value)
{
var newScore = proxy.Score + value;
if (newScore > MAX_SCORE)
{
return;
}
await _dbContext.Database.ExecuteSqlInterpolatedAsync($"UPDATE t_proxys SET score={newScore}, updated_time={DateTime.Now} WHERE id={proxy.Id}");
}
async Task DoWork(ProxyDto dto)
{
using var scope = _serviceProvider.CreateScope();
try
{
var proxyCheck = scope.ServiceProvider.GetService <IProxyCheck>();
await proxyCheck.Check(dto);
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
}
}
public List <ProxyDto> GetContent(string url, string ipxpath, string portxpath)
{
var proxyList = new List <ProxyDto>();
var html = client.GetStringAsync(url).Result;
HtmlDocument hd = new HtmlDocument();
hd.LoadHtml(html);
HtmlNode root = hd.DocumentNode;
HtmlNodeCollection ipNodes = root.SelectNodes(ipxpath);
HtmlNodeCollection portNodes = root.SelectNodes(portxpath);
for (int j = 0; j < ipNodes.Count; j++)
{
var proxy = new ProxyDto();
proxy.IP = ipNodes[j].InnerText;
proxy.Port = portNodes[j].InnerText;
proxyList.Add(proxy);
}
return(proxyList);
}
public async Task AddAsync(ProxyDto dto)
{
await _context.Proxys.AddAsync(new Proxy
{
AnonymousDegree = dto.AnonymousDegree,
CreatedTime = dto.CreatedTime,
IP = dto.IP,
Port = dto.Port,
Score = dto.Score,
UpdatedTime = dto.UpdatedTime
});
try
{
await _context.SaveChangesAsync();
}
catch (System.Exception)
{
throw;
}
}
public async Task EnqueueTaskItemAsync(ProxyDto dto)
{
var conn = _connectionMultiplexerFactory.CreateConnectionMultiplexer();
var db = conn.GetDatabase();
await db.ListRightPushAsync(REDIS_CHECK_KEY, JsonConvert.SerializeObject(dto));
}
async Task DeleteProxyAsync(ProxyDto proxy)
{
await _dbContext.Database.ExecuteSqlInterpolatedAsync($"DELETE FROM t_proxys WHERE id={proxy.Id}");
}
