protected override EntitySpider GetEntitySpider()
{
EntitySpider context = new EntitySpider(new Site());
context.SetSite(new Site());
context.SetThreadNum(2);
context.ThreadNum = 1;
context.RetryWhenResultIsEmpty = false;
context.Deep = 100;
context.EmptySleepTime = 5000;
context.SetEmptySleepTime(5000);
context.ExitWhenComplete = true;
context.SetCachedSize(1);
context.SetDownloader(new HttpClientDownloader());
context.SetScheduler(new QueueDuplicateRemovedScheduler());
context.SkipWhenResultIsEmpty = true;
context.SpawnUrl = true;
context.SetIdentity("qidian_" + DateTime.Now.ToString("yyyy_MM_dd_HHmmss"));
context.AddEntityPipeline(new CollectEntityPipeline());
context.AddStartUrl("http://www.cas.cn/kx/kpwz/index.shtml");
context.AddEntityType(typeof(ArticleSummary));
return(context);
}
