protected override void Run() { BidWebsiteSpider sp = new BidWebsiteSpider(); var config = new BidSourceConfig { ListUrl = "http://www.chinabidding.com/zbzx.jhtml?method=outlineOne&type=biddingProjectGG&channelId=205" , DetailUrlPattern = @"http://www.chinabidding.com/zbzx-detail-\d+.html" , TitleXpath = "/html/body/div/div[2]/div[2]/div[1]/div/h2" , ContentXpath = "/html/body/div/div[2]/div[2]/div[1]/div/div[2]" , PubishDateXpath = "/html/body/div/div[2]/div[2]/div[1]/div/div[1]" , PubishDatePattern = @"(\d{4}\.\d{2}\.\d{2})" }; //BidSourceConfigBLL configService = new BidSourceConfigBLL(); //configService.Insert(config); var bidList = sp.DownLoadBids(config); BidBLL bidService = new BidBLL(); foreach (var entity in bidList) { bidService.Insert(entity); } }
static void Main(string[] args) { IBidSourceConfigBLL configService = new BidSourceConfigBLL(); int pageCount, recordCount; int pageIndex = 1; do { var configList = configService.GetList(null, pageIndex, 20, out pageCount, out recordCount); foreach (var config in configList) { if (config != null && !string.IsNullOrWhiteSpace(config.ListUrl)) { //ThreadPool.QueueUserWorkItem(d => //{ // BidWebsiteSpider sp = new BidWebsiteSpider(); // var bidList = sp.DownLoadBids((BidSourceConfig)d); // BidBLL bidService = new BidBLL(); // foreach (var entity in bidList) // { // bidService.Insert(entity); // } //}, config); BidWebsiteSpider sp = new BidWebsiteSpider(); var bidList = sp.DownLoadBids(config); BidBLL bidService = new BidBLL(); foreach (var entity in bidList) { bidService.Insert(entity); } //bidService.CreateLuceneIndex(bidList); } } }while (pageIndex > pageCount); Console.ReadKey(); }