public static void RunTask()
{
OoSpider ooSpider = OoSpider.Create("ganji_posts_" + DateTime.Now.Date.ToString("yyyy-MM-dd"), new Site {
SleepTime = 1000, Encoding = Encoding.UTF8
}, new CollectorPageModelToDbPipeline(), typeof(Ganji));
ooSpider.SetEmptySleepTime(15000);
ooSpider.SetThreadNum(10);
ooSpider.SetScheduler(new RedisScheduler("localhost", ""));
ooSpider.AddUrl("http://sh.ganji.com/zpdianhuaxiaoshou/o1/");
ooSpider.Run();
}
public static void RunTask()
{
OoSpider ooSpider = OoSpider.Create("aiqiyi_movies_" + DateTime.Now.ToLocalTime().ToString(),
new Site {
SleepTime = 1000, Encoding = Encoding.UTF8
}, new CollectorPageModelToDbPipeline(), typeof(walter));
ooSpider.SetEmptySleepTime(15000);
ooSpider.SetThreadNum(1);
ooSpider.SetScheduler(new RedisScheduler("localhost", null));
ooSpider.AddUrl("http://top.iqiyi.com/dianshiju.html#");
ooSpider.Run();
}
public static void RunTask()
{
OoSpider ooSpider = OoSpider.Create(new Site
{
SleepTime = 10000
}, new CollectorPageModelToDbPipeline(), typeof(Ganji));
ooSpider.SetThreadNum(1);
Request request = new Request("http://mobds.ganji.com/datashare/", null);
request.Method = "POST";
ooSpider.AddRequest(request);
ooSpider.Run();
}
