public GeneralSiteCrawler(SiteParameter siteParameter)
{
IItemReader itemReader = new RegexItemReader(siteParameter);
IHtmlReader htmlReader = new HttpClientReader();
this.pageReader = new SequentialPageReader(siteParameter, htmlReader, itemReader);
this.pageParser = new RegexPageParser(siteParameter, htmlReader);
}
public GeneralSiteCrawler(SiteParameter siteParameter)
{
this.dataService = new DbDataService(CrawlerDbHelper.GetContext());
IItemReader itemReader = new RegexItemReader(siteParameter);
IHtmlReader htmlReader = new HttpClientReader();
this.pageReader = new SequentialPageReader(siteParameter, htmlReader, itemReader);
this.pageParser = new RegexPageParser(siteParameter, htmlReader);
this.pageParser.SetErrorHandler((url, exception) =>
this.dataService.AddLog(new CrawlerLog
{
Url = url,
LogTime = DateTime.Now,
Message = exception.Message
}));
}
