protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.Site = new Site { MaxSleepTime = 1, MinSleepTime = 1 }; context.SetTaskGroup("cnblogs homepage"); context.SetSpiderName("cnblogs homepage " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddStartUrl("http://news.cnblogs.com/n/page/1/"); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='taobao';Data Source= 86research.imwork.net;User ID=root;Password=1qazZAQ!;Port=4306" }); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Expression = "//*[@id='pager']", Type = ExtractType.XPath } }); context.ThreadNum = 5; context.AddEntityType(typeof(Cnblogs)); return(context); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetThreadNum(2); context.SetSpiderName("JD_sku_store_test_" + DateTime.Now.ToString("yyyy_MM_dd_HHmmss")); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Type = ExtractType.XPath, Expression = "//span[@class=\"p-num\"]" }, Patterns = new List <string> { @"&page=[0-9]+&" } }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='test';Data Source=mysqlserver;User ID=root;Password=1qazZAQ!;Port=4306" }); context.AddStartUrl("http://list.jd.com/list.html?cat=9987,653,655&page=2&JL=6_0_0&ms=5#J_main", new Dictionary <string, object> { { "name", "手机" }, { "cat3", "655" } }); context.AddEntityType(typeof(Product)); return(context); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetSpiderName("JD sku/store test " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Type = ExtractType.XPath, Expression = "//span[@class=\"p-num\"]" }, Patterns = new List <string> { @"&page=[0-9]+&" } }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='mysql';Data Source=192.168.199.211;User ID=root;Password=1qazZAQ!;Port=3306" }); context.AddStartUrl("http://list.jd.com/list.html?cat=9987,653,655&page=2&JL=6_0_0&ms=5#J_main", new Dictionary <string, object> { { "name", "手机" }, { "cat3", "655" } }); context.AddEntityType(typeof(Product)); context.SetDownloader(new WebDriverDownloader { Browser = Extension.Downloader.WebDriver.Browser.Chrome }); return(context); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetTaskGroup("JD sku/store test"); context.SetSpiderName("JD sku/store test " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Type = ExtractType.XPath, Expression = "//span[@class=\"p-num\"]" }, Patterns = new List <string> { @"&page=[0-9]+&" } }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='test';Data Source=ooodata.com;User ID=root;Password=1qazZAQ!123456;Port=4306" }); context.SetScheduler(new RedisScheduler { Host = "ooodata.com", Password = "******", Port = 6379 }); context.AddStartUrl("http://list.jd.com/list.html?cat=9987,653,655&page=2&JL=6_0_0&ms=5#J_main", new Dictionary <string, object> { { "name", "手机" }, { "cat3", "655" } }); context.AddEntityType(typeof(Product)); return(context); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetTaskGroup("JD sku/store test"); context.SetSpiderName("JD sku/store test " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Type = ExtractType.XPath, Expression = "//span[@class=\"p-num\"]" }, Patterns = new List <string> { @"&page=[0-9]+&" } }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='test';Data Source=ooodata.com;User ID=root;Password=1qazZAQ!123456;Port=4306" }); context.SetScheduler(new RedisScheduler { Host = "ooodata.com", Password = "******", Port = 6379 }); context.AddStartUrl("http://list.jd.com/list.html?cat=9987,653,655&page=1", new Dictionary <string, object> { { "name", "手机" }, { "cat3", "655" } }); context.AddPrepareStartUrls(new ConfigurableDbPrepareStartUrls() { Source = DataSource.MySql, ConnectString = "Database='test';Data Source= ooodata.com;User ID=root;Password=1qazZAQ!123456;Port=4306", TableName = "jd.category", Columns = new List <BaseDbPrepareStartUrls.Column> { new BaseDbPrepareStartUrls.Column { Name = "url", Formatters = new List <Formatter> { new ReplaceFormatter { OldValue = ".html", NewValue = "" } } } }, FormateStrings = new List <string> { "{0}&page=1&JL=6_0_0" } }); context.AddEntityType(typeof(Product)); return(context); }
protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetSpiderName("JD sku/store test " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Type = ExtractType.XPath, Expression = "//span[@class=\"p-num\"]" }, Patterns = new List<string> { @"&page=[0-9]+&" } }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='test';Data Source=;User ID=root;Password=;Port=4306" }); context.AddStartUrl("http://list.jd.com/list.html?cat=9987,653,655&page=2&JL=6_0_0&ms=5#J_main", new Dictionary<string, object> { { "name", "手机" }, { "cat3", "655" } }); context.AddEntityType(typeof(Product)); return context; }