public static void Run() { OoSpider.Create(new Site(), typeof(News163)).AddUrl("http://news.163.com/13/0802/05/958I1E330001124J_2.html") .AddPipeline(new MultiPagePipeline()) .AddPipeline(new ConsolePipeline()) .SetScheduler(new RedisScheduler("localhost", "")).Run(); }
public static void Run() { var site = new Site(); site.AddStartUrl("http://flashsword20.iteye.com/blog"); OoSpider.Create(site, typeof(IteyeBlog)).Run(); }
public static void Run() { Site site = new Site { Encoding = Encoding.UTF8 }; site.AddStartUrl("http://www.oschina.net/question/1995445_2136783"); OoSpider.Create(site, typeof(OschinaAnswer)).Run(); }
public static void Run() { Site site = new Site(); site.AddStartUrl("http://www.36kr.com/"); Core.Spider thread = OoSpider.Create(site, new CollectorPageModelToDbPipeline(), typeof(Kr36NewsModel)).SetThreadNum(20); thread.Start(); SpiderMonitor spiderMonitor = SpiderMonitor.Instance; spiderMonitor.Register(thread); }
public static void RunTask() { OoSpider ooSpider = OoSpider.Create("ganji_posts_" + DateTime.Now.Date.ToString("yyyy-MM-dd"), new Site { SleepTime = 1000, Encoding = Encoding.UTF8 }, new CollectorPageModelToDbPipeline(), typeof(Ganji)); ooSpider.SetEmptySleepTime(15000); ooSpider.SetThreadNum(10); ooSpider.SetScheduler(new RedisScheduler("localhost", "")); ooSpider.AddUrl("http://sh.ganji.com/zpdianhuaxiaoshou/o1/"); ooSpider.Run(); }
public static void RunTask() { OoSpider ooSpider = OoSpider.Create("aiqiyi_movies_" + DateTime.Now.ToLocalTime().ToString(), new Site { SleepTime = 1000, Encoding = Encoding.UTF8 }, new CollectorPageModelToDbPipeline(), typeof(walter)); ooSpider.SetEmptySleepTime(15000); ooSpider.SetThreadNum(1); ooSpider.SetScheduler(new RedisScheduler("localhost", null)); ooSpider.AddUrl("http://top.iqiyi.com/dianshiju.html#"); ooSpider.Run(); }
//[ExtractBy(Value = "//div[@class='BlogStat']/regex('\\d+-\\d+-\\d+\\s+\\d+:\\d+')")] //public DateTime Date { get; set; } public static void Run() { Site site = new Site { UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36" }; site.AddStartUrl("http://my.oschina.net/flashsword/blog"); site.SleepTime = 0; site.RetryTimes = 3; OoSpider.Create(site, new CollectorPageModelToDbPipeline(), typeof(OschinaBlog)).SetThreadNum(1).Run(); }
public static void RunTask() { OoSpider ooSpider = OoSpider.Create(new Site { SleepTime = 10000 }, new CollectorPageModelToDbPipeline(), typeof(Ganji)); ooSpider.SetThreadNum(1); Request request = new Request("http://mobds.ganji.com/datashare/", null); request.Method = "POST"; ooSpider.AddRequest(request); ooSpider.Run(); }
public static void Run() { OoSpider.Create(new Site(), new ConsolePageModelPipeline(), typeof(QqMeishi)).AddUrl("http://meishi.qq.com/beijing/c/all").SetThreadNum(4).Run(); }
public void TestOschinaBlog() { //results will be saved to "/data/webmagic/" in json format OoSpider.Create(new Site(), new JsonFilePageModelPipeline("/data/webmagic/"), typeof(OschinaBlog)).AddUrl("http://my.oschina.net/flashsword/blog").Run(); }