public async Task <HtmlDocument> LoadHtmlDocumentAsync(string url)
{
using (_webClient = new ExtendedWebClient())
{
_webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36");
_webClient.Headers.Add("Accept-Language", "en-US,en;q=0.9,ru;q=0.8");
_webClient.Timeout = _settingsService.GetAllowedTimeoutForProxy();
var htmlString = await _webClient.DownloadStringTaskAsync(url);
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(htmlString);
return(htmlDoc);
}
}
public async Task <HtmlDocument> LoadHtmlDocumentAsync(string url)
{
var proxyUrl = _proxyService.GetRandomProxyUrlFromRedis();
_webClient.Proxy = new WebProxy(proxyUrl);
_webClient.Timeout = 10000;
_webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36");
_webClient.Headers.Add("Accept-Language", "en-US,en;q=0.9,ru;q=0.8");
_webClient.Encoding = Encoding.GetEncoding(1251);
_logger.LogInformation($"Make request to {url} , using proxy: {proxyUrl}");
string htmlString = await _webClient.DownloadStringTaskAsync(url);
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(htmlString);
return(htmlDoc);
}
public async Task <(string proxyUrl, bool isWorking)> IsWorking(string proxyUrl)
{
using (_webClient = new ExtendedWebClient())
{
try
{
_webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36");
_webClient.Headers.Add("Accept-Language", "en-US,en;q=0.9,ru;q=0.8");
_webClient.Proxy = new WebProxy(proxyUrl);
_webClient.Timeout = _settings.GetAllowedTimeoutForProxy();
foreach (var url in _settings.GetUrlsForCheck())
{
await _webClient.DownloadStringTaskAsync(url);
}
return(proxyUrl, isWorking : true);
}
catch (Exception ex)
{
return(proxyUrl, isWorking : false);
}
}
}
