public static void Run()
{
Site site = new Site();
site.AddStartUrl("http://www.36kr.com/");
Core.Spider thread = OoSpider.Create(site, new CollectorPageModelToDbPipeline(), typeof(Kr36NewsModel)).SetThreadNum(20);
thread.Start();
SpiderMonitor spiderMonitor = SpiderMonitor.Instance;
spiderMonitor.Register(thread);
}
//[ExtractBy(Value = "//div[@class='BlogStat']/regex('\\d+-\\d+-\\d+\\s+\\d+:\\d+')")]
//public DateTime Date { get; set; }
public static void Run()
{
Site site = new Site
{
UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36"
};
site.AddStartUrl("http://my.oschina.net/flashsword/blog");
site.SleepTime = 0;
site.RetryTimes = 3;
OoSpider.Create(site, new CollectorPageModelToDbPipeline(),typeof(OschinaBlog)).SetThreadNum(1).Run();
}
public static void Run()
{
Site site = new Site {Encoding = Encoding.UTF8};
site.AddStartUrl("http://www.oschina.net/question/1995445_2136783");
OoSpider.Create(site, typeof(OschinaAnswer)).Run();
}
public static void Run()
{
var site = new Site();
site.AddStartUrl("http://flashsword20.iteye.com/blog");
OoSpider.Create(site, typeof(IteyeBlog)).Run();
}
