protected override SpiderContext GetSpiderContext() { SpiderContext context = new SpiderContext(); context.SetTaskGroup("JD sku/store test"); context.SetSpiderName("JD sku/store test " + DateTime.Now.ToString("yyyy-MM-dd HHmmss")); context.AddTargetUrlExtractor(new Extension.Configuration.TargetUrlExtractor { Region = new Extension.Configuration.Selector { Type = ExtractType.XPath, Expression = "//span[@class=\"p-num\"]" }, Patterns = new List <string> { @"&page=[0-9]+&" } }); context.AddPipeline(new MysqlPipeline { ConnectString = "Database='test';Data Source=ooodata.com;User ID=root;Password=1qazZAQ!123456;Port=4306" }); context.SetScheduler(new RedisScheduler { Host = "ooodata.com", Password = "******", Port = 6379 }); context.AddStartUrl("http://list.jd.com/list.html?cat=9987,653,655&page=1", new Dictionary <string, object> { { "name", "手机" }, { "cat3", "655" } }); context.AddPrepareStartUrls(new ConfigurableDbPrepareStartUrls() { Source = DataSource.MySql, ConnectString = "Database='test';Data Source= ooodata.com;User ID=root;Password=1qazZAQ!123456;Port=4306", TableName = "jd.category", Columns = new List <BaseDbPrepareStartUrls.Column> { new BaseDbPrepareStartUrls.Column { Name = "url", Formatters = new List <Formatter> { new ReplaceFormatter { OldValue = ".html", NewValue = "" } } } }, FormateStrings = new List <string> { "{0}&page=1&JL=6_0_0" } }); context.AddEntityType(typeof(Product)); return(context); }