public HttpClientWebCrawler(CrawlConfiguration Config) : base(Config)
{
this.Client = new HttpClient(new HttpClientHandler
{
UseProxy = false
}, false);
this.Client.Timeout = TimeSpan.FromSeconds(this.Config.CrawlTimeoutSeconds);
this.Client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Safari/537.36");
}
public PuppeteerWebCrawler(CrawlConfiguration Config) : base(Config)
{
this.Client.DefaultWaitForTimeout = this.Config.CrawlTimeoutSeconds * SECOND;
}
public WebCrawler(CrawlConfiguration Config)
{
this.Config = Config;
}