public ScraperBase(ProxyServices proxyService = null, bool enableRandomSleepTime = true, int httpRequestMaxRetryCount = 5, int sleepTimeBetweenHTTPRequestsLowRange = 250, int sleepTimeBetweenHTTPRequestsHighRange = 1500)
{
ProxyService = proxyService;
HTTPRequestMaxRetryCount = httpRequestMaxRetryCount;
SleepTimeBetweenHTTPRequestsLowRange = sleepTimeBetweenHTTPRequestsLowRange;
SleepTimeBetweenHTTPRequestsHighRange = sleepTimeBetweenHTTPRequestsHighRange;
EnableRandomSleepTime = enableRandomSleepTime;
//User-Agent: https://techblog.willshouse.com/2012/01/03/most-common-user-agents/
//HttpWebClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;");
//wc.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip, deflate"); //This gets added above in the HttpClientHandler constructor
//_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", _randomUserAgentString);
//_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Charset", "ISO-8859-1");
}
public IpdbScraper(ProxyServices proxyService, bool enableRandomSleepTime = true, int httpRequestMaxRetryCount = 5, int sleepTimeBetweenHTTPRequestsLowRange = 250, int sleepTimeBetweenHTTPRequestsHighRange = 1500)
: base(proxyService, enableRandomSleepTime, httpRequestMaxRetryCount, sleepTimeBetweenHTTPRequestsLowRange, sleepTimeBetweenHTTPRequestsHighRange)
{
}