public Crawler(CrawlRequest request, ILogger log)
{
_id = request.Id;
_host = request.Host;
var internalDomains = new List <string>
{
_host,
_githubDomain
};
_processor = new CloudflareCgiProcesser()
.Next(new LegacyProcessor()
.Next(new ImageProcessor(new ClientWrapper(log))
.Next(new ContentProcessor(new ClientWrapper(log))
.Next(new KnownPageProcessor()
.Next(new EmailProcessor()
.Next(new ExternalPageProcessor(internalDomains)
.Next(new PodcastRoadmapProcessor(new ClientWrapper(log)))
.Next(new PageProcessor(_gistDomain, new ClientWrapper(log), null)
.Next(new PageProcessor(_githubDomain, new ClientWrapper(log), null)
.Next(new PageProcessor(_host, new ClientWrapper(log), new ContentLinksExtractor(_host))
.Next(new UnknownProcessor()))))))))));
}
public IProcessUrl Next(IProcessUrl nextProcessor)
{
_next = nextProcessor;
return(this);
}
