public static void Run()
{
OoSpider.Create(new Site(), typeof(News163)).AddUrl("http://news.163.com/13/0802/05/958I1E330001124J_2.html")
.AddPipeline(new MultiPagePipeline())
.AddPipeline(new ConsolePipeline())
.SetScheduler(new RedisScheduler("localhost", "")).Run();
}
public static void Run()
{
var site = new Site();
site.AddStartUrl("http://flashsword20.iteye.com/blog");
OoSpider.Create(site, typeof(IteyeBlog)).Run();
}
public static void Run()
{
Site site = new Site {
Encoding = Encoding.UTF8
};
site.AddStartUrl("http://www.oschina.net/question/1995445_2136783");
OoSpider.Create(site, typeof(OschinaAnswer)).Run();
}
public static void Run()
{
Site site = new Site();
site.AddStartUrl("http://www.36kr.com/");
Core.Spider thread = OoSpider.Create(site, new CollectorPageModelToDbPipeline(), typeof(Kr36NewsModel)).SetThreadNum(20);
thread.Start();
SpiderMonitor spiderMonitor = SpiderMonitor.Instance;
spiderMonitor.Register(thread);
}
public static void RunTask()
{
OoSpider ooSpider = OoSpider.Create("ganji_posts_" + DateTime.Now.Date.ToString("yyyy-MM-dd"), new Site {
SleepTime = 1000, Encoding = Encoding.UTF8
}, new CollectorPageModelToDbPipeline(), typeof(Ganji));
ooSpider.SetEmptySleepTime(15000);
ooSpider.SetThreadNum(10);
ooSpider.SetScheduler(new RedisScheduler("localhost", ""));
ooSpider.AddUrl("http://sh.ganji.com/zpdianhuaxiaoshou/o1/");
ooSpider.Run();
}
public static void RunTask()
{
OoSpider ooSpider = OoSpider.Create("aiqiyi_movies_" + DateTime.Now.ToLocalTime().ToString(),
new Site {
SleepTime = 1000, Encoding = Encoding.UTF8
}, new CollectorPageModelToDbPipeline(), typeof(walter));
ooSpider.SetEmptySleepTime(15000);
ooSpider.SetThreadNum(1);
ooSpider.SetScheduler(new RedisScheduler("localhost", null));
ooSpider.AddUrl("http://top.iqiyi.com/dianshiju.html#");
ooSpider.Run();
}
//[ExtractBy(Value = "//div[@class='BlogStat']/regex('\\d+-\\d+-\\d+\\s+\\d+:\\d+')")]
//public DateTime Date { get; set; }
public static void Run()
{
Site site = new Site
{
UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36"
};
site.AddStartUrl("http://my.oschina.net/flashsword/blog");
site.SleepTime = 0;
site.RetryTimes = 3;
OoSpider.Create(site, new CollectorPageModelToDbPipeline(), typeof(OschinaBlog)).SetThreadNum(1).Run();
}
public static void RunTask()
{
OoSpider ooSpider = OoSpider.Create(new Site
{
SleepTime = 10000
}, new CollectorPageModelToDbPipeline(), typeof(Ganji));
ooSpider.SetThreadNum(1);
Request request = new Request("http://mobds.ganji.com/datashare/", null);
request.Method = "POST";
ooSpider.AddRequest(request);
ooSpider.Run();
}
public static void Run()
{
OoSpider.Create(new Site(), new ConsolePageModelPipeline(), typeof(QqMeishi)).AddUrl("http://meishi.qq.com/beijing/c/all").SetThreadNum(4).Run();
}
public void TestOschinaBlog()
{
//results will be saved to "/data/webmagic/" in json format
OoSpider.Create(new Site(), new JsonFilePageModelPipeline("/data/webmagic/"), typeof(OschinaBlog)).AddUrl("http://my.oschina.net/flashsword/blog").Run();
}
