public Crawler(WorkerPool workerPool, ICrawlRequestFilter crawlRequestFilter, ILinkExtractor linkExtractor, ICrawlerObserver observer, IReadOnlyDictionary <string, string> customHttpHeaders = null, TimeSpan?requestTimeout = null, int maxRetries = 0, string userAgent = null) { if (this.maxRetries < 0) { throw new ArgumentOutOfRangeException(nameof(maxRetries), "Max retries must be non-negative"); } this.crawlRequestFilter = crawlRequestFilter; this.linkExtractor = linkExtractor; this.observer = observer; this.maxRetries = maxRetries; this.userAgent = userAgent; this.customHttpHeaders = customHttpHeaders ?? new Dictionary <string, string>(); this.workerPool = workerPool; if (requestTimeout != null) { this.httpClient.Timeout = requestTimeout.Value; } }
public Worker(WorkerPool pool) { this.pool = pool; }