/// <summary>
/// 页面下载
/// </summary>
private void ProcessRequest(Request request, IDownLoader downLoader)
{
var page = downLoader.DownLoader(request, this);
PageProcessor.Process(page);
Scheduler.AddFinishRequest(request);
SpiderListening.ForEach(item => item.AfterSuccess(request));
if (page.IsSave)
{
Pipelines.ForEach(item => item.Process(page.PageResult));
}
GetPageUrl(page).ForEach(item => Scheduler.AddWaitRequest(new Request(item)));
}
public RankTrackerController(IDownLoader downLoader, IParser parser)
{
this.downloader = downLoader;
this.parser = parser;
}
public Spider(Site site, IPageProcessor pageProcessor, IScheduler scheduler, IDownLoader downLoader)
: this(site, pageProcessor, scheduler)
{
DownLoader = downLoader;
}
