protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var proxyHealthCheckersTaskList = new List <Task <(string proxyUrl, bool isWorking)> >();
_proxyService.WriteToRedisExistingProxiesFromFile();
while (!stoppingToken.IsCancellationRequested)
{
try
{
var parsedProxies = new List <string>();
foreach (var proxyParseProvider in _proxyParseProviders)
{
parsedProxies.AddRange(await proxyParseProvider.GetProxiesAsync());
}
_logger.LogInformation($"Total amount of parsed proxies {parsedProxies.Count}");
for (int i = 0; i < parsedProxies.Count; i++)
{
proxyHealthCheckersTaskList.Add(_proxyHealthChecker.IsWorking(parsedProxies[i]));
}
await Task.WhenAll(proxyHealthCheckersTaskList);
var resultOfCheck = proxyHealthCheckersTaskList.Select(x => x.Result).ToList();
for (int i = 0; i < resultOfCheck.Count; i++)
{
var proxy = resultOfCheck[i].proxyUrl;
var isWorking = resultOfCheck[i].isWorking;
var isAlreadyExists = _proxyService.IsAlreadyExists(proxy);
if (!isAlreadyExists && isWorking)
{
_proxyService.SaveProxyToRedis(proxy);
_proxyService.SaveProxyToFile(proxy);
_logger.LogInformation($"Added new proxy {proxy}");
}
if ((i % 50) == 0)
{
_logger.LogInformation($"Processed {i}/{parsedProxies.Count}");
}
}
await Task.Delay(_settings.GetDelayBetweenGrabbing());
}
catch (Exception ex)
{
_logger.LogError(ex.Message + "\n" + ex.StackTrace);
}
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var proxyHealthCheckersTaskList = new List <Task <(string proxyUrl, bool isWorking)> >();
while (!stoppingToken.IsCancellationRequested)
{
try
{
await Task.Delay(_settings.GetHealthCheckFrequencyInMilliseconds());
_logger.LogInformation($"Started work to check proxy healthy");
var existsProxies = _proxyService.GetProxiesFromRedis().ToList();
for (int i = 0; i < existsProxies.Count; i++)
{
proxyHealthCheckersTaskList.Add(_proxyHealthChecker.IsWorking(existsProxies[i]));
}
await Task.WhenAll(proxyHealthCheckersTaskList);
var resultOfCheck = proxyHealthCheckersTaskList.Select(x => x.Result).ToList();
for (int i = 0; i < resultOfCheck.Count; i++)
{
var proxyUrl = resultOfCheck[i].proxyUrl;
var isWorking = resultOfCheck[i].isWorking;
var isCandidateToDelete = _candidatesToRemove.Any(x => x.Key == proxyUrl);
if (isWorking && isCandidateToDelete)
{
_candidatesToRemove[proxyUrl] = 0;
}
if (!isWorking)
{
if (!isCandidateToDelete)
{
_candidatesToRemove.Add(proxyUrl, 1);
}
else
{
var currentAttemptCount = _candidatesToRemove[proxyUrl];
currentAttemptCount += 1;
if (currentAttemptCount >= _settings.GetFailedHealthCheckAttemptCount())
{
_proxyService.RemoveFromFile(proxyUrl);
_proxyService.RemoveFromRedis(proxyUrl);
_logger.LogInformation($"Removed {proxyUrl}, it has failed checking {_settings.GetFailedHealthCheckAttemptCount()} times");
}
}
}
}
}
catch (Exception ex)
{
_logger.LogError(ex.Message + "\n" + ex.StackTrace);
}
}
}