public HttpClientWebCrawler(CrawlConfiguration Config) : base(Config) { this.Client = new HttpClient(new HttpClientHandler { UseProxy = false }, false); this.Client.Timeout = TimeSpan.FromSeconds(this.Config.CrawlTimeoutSeconds); this.Client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Safari/537.36"); }
public PuppeteerWebCrawler(CrawlConfiguration Config) : base(Config) { this.Client.DefaultWaitForTimeout = this.Config.CrawlTimeoutSeconds * SECOND; }
public WebCrawler(CrawlConfiguration Config) { this.Config = Config; }