public override void Run()
{
try
{
Log.Debug("Beginning Request Loop");
foreach (Request request in this.master.RequestQueue.GetConsumingEnumerable(this.cancellationToken))
{
Log.Debug("Processing Request");
if (!this.master.InternetConnectivity)
{
Log.Debug("No internet connectivity. Aborting request.");
master.EnqueueRequest(request); // Toss it back on the queue.
Thread.Sleep(10000);
continue;
}
try
{
ProxyCheck(request);
Response response = null;
if (request.DriverType == Request.DriverTypes.RawHTTP)
{
response = DoRawHTTPWebRequest(request);
}
else if (request.DriverType == Request.DriverTypes.HeadlessChrome)
{
response = DoHeadlessChromeWebRequest(request);
}
else
{
throw new NotImplementedException();
}
this.contiguousFailureCount = 0;
proxy?.RegisterSuccess();
this.Status = "Request executed successfully. Sleeping a bit before the next request";
// TODO make this count down and check for cancelationtoken.
Thread.Sleep(SleepBetweenRequests);
master.EnqueueResponse(response);
}
catch (ProxyRepositoryFailureException ex)
{
this.Status = "Unable to execute the request. No available proxies.";
// TODO make this count down and check for cancelationtoken.
Thread.Sleep(SleepBetweenRequests);
master.EnqueueResponse(new Response(request, exception: ex, proxy: proxy));
}
catch (DriverServiceNotFoundException)
{
this.Status = "A driver used for scraping (probably chromedriver) is not installed.";
throw;
}
catch (Exception ex)
{
request.FailureCount++;
proxy?.RegisterFailure();
this.RegisterFailure();
this.Status = "Error: " + ex.Message;
// TODO check to see if the URL was bad. If it is, don't register failure on this thread, just on the request.
if (request.FailureCount < Request.MAX_FAILURES)
{
master.EnqueueRequest(request); // Toss it back on the queue.
}
else
{
Response response = new Response(request, exception: ex, proxy: proxy as ProxyModel.Proxy);
master.EnqueueResponse(response);
}
}
}
master.EnqueueResponse(this);
}
catch (OperationCanceledException) { }
finally
{
this.Status = "Running Ended.";
}
}
