protected override EntitySpider GetEntitySpider() { EntitySpider context = new EntitySpider(new Site()); context.SetSite(new Site()); context.SetThreadNum(2); context.ThreadNum = 1; context.RetryWhenResultIsEmpty = false; context.Deep = 100; context.EmptySleepTime = 5000; context.SetEmptySleepTime(5000); context.ExitWhenComplete = true; context.CachedSize = 1; context.SetDownloader(new HttpClientDownloader()); context.SetScheduler(new QueueDuplicateRemovedScheduler()); context.SkipWhenResultIsEmpty = true; context.SpawnUrl = true; context.AddPipeline(new CollectEntityPipeline()); context.AddStartUrl("http://www.cas.cn/kx/kpwz/index.shtml"); context.AddEntityType(typeof(ArticleSummary)); Name = "qidian"; Batch = DateTime.Now.ToString("yyyy_MM_dd_HHmmss"); return(context); }