public GeneralSiteCrawler(SiteParameter siteParameter) { IItemReader itemReader = new RegexItemReader(siteParameter); IHtmlReader htmlReader = new HttpClientReader(); this.pageReader = new SequentialPageReader(siteParameter, htmlReader, itemReader); this.pageParser = new RegexPageParser(siteParameter, htmlReader); }
public GeneralSiteCrawler(SiteParameter siteParameter) { this.dataService = new DbDataService(CrawlerDbHelper.GetContext()); IItemReader itemReader = new RegexItemReader(siteParameter); IHtmlReader htmlReader = new HttpClientReader(); this.pageReader = new SequentialPageReader(siteParameter, htmlReader, itemReader); this.pageParser = new RegexPageParser(siteParameter, htmlReader); this.pageParser.SetErrorHandler((url, exception) => this.dataService.AddLog(new CrawlerLog { Url = url, LogTime = DateTime.Now, Message = exception.Message })); }