public static void Run() { Site site = new Site(); site.AddStartUrl("http://www.36kr.com/"); Core.Spider thread = OoSpider.Create(site, new CollectorPageModelToDbPipeline(), typeof(Kr36NewsModel)).SetThreadNum(20); thread.Start(); SpiderMonitor spiderMonitor = SpiderMonitor.Instance; spiderMonitor.Register(thread); }
//[ExtractBy(Value = "//div[@class='BlogStat']/regex('\\d+-\\d+-\\d+\\s+\\d+:\\d+')")] //public DateTime Date { get; set; } public static void Run() { Site site = new Site { UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36" }; site.AddStartUrl("http://my.oschina.net/flashsword/blog"); site.SleepTime = 0; site.RetryTimes = 3; OoSpider.Create(site, new CollectorPageModelToDbPipeline(),typeof(OschinaBlog)).SetThreadNum(1).Run(); }
public static void Run() { Site site = new Site {Encoding = Encoding.UTF8}; site.AddStartUrl("http://www.oschina.net/question/1995445_2136783"); OoSpider.Create(site, typeof(OschinaAnswer)).Run(); }
public static void Run() { var site = new Site(); site.AddStartUrl("http://flashsword20.iteye.com/blog"); OoSpider.Create(site, typeof(IteyeBlog)).Run(); }