public Crawler(WorkerPool workerPool, ICrawlRequestFilter crawlRequestFilter, ILinkExtractor linkExtractor, ICrawlerObserver observer,
IReadOnlyDictionary <string, string> customHttpHeaders = null, TimeSpan?requestTimeout = null, int maxRetries = 0, string userAgent = null)
{
if (this.maxRetries < 0)
{
throw new ArgumentOutOfRangeException(nameof(maxRetries), "Max retries must be non-negative");
}
this.crawlRequestFilter = crawlRequestFilter;
this.linkExtractor = linkExtractor;
this.observer = observer;
this.maxRetries = maxRetries;
this.userAgent = userAgent;
this.customHttpHeaders = customHttpHeaders ?? new Dictionary <string, string>();
this.workerPool = workerPool;
if (requestTimeout != null)
{
this.httpClient.Timeout = requestTimeout.Value;
}
}
public Worker(WorkerPool pool)
{
this.pool = pool;
}