/// <summary> /// 抓取 /// </summary> private static void Crawl() { DBInit dbInit = new DBInit(); CategoryRepository categoryRepository = new CategoryRepository(); Console.WriteLine("请输入Y/N进行类别表初始化确认! Y 删除Category表然后重新创建,然后抓取类型数据,N(或者其他)跳过"); string input = Console.ReadLine(); if (input.Equals("Y", StringComparison.OrdinalIgnoreCase)) { dbInit.InitCategoryTable(); // List<Category> categoryList = CategorySearch.Crawler("http://www.jd.com/allSort.aspx"); List <Category> categoryList = CategorySearch.Crawler("https://www.alibaba.com/?spm=a2700.icbuShop.scGlobalHeaderSmall.1.684142aeD461tx"); categoryRepository.Save(categoryList); Console.WriteLine("类型数据初始化完成,共抓取类别{0}个", categoryList.Count); } else { Console.WriteLine("你选择不初始化类别数据"); } Console.WriteLine("*****************^_^**********************"); Console.WriteLine("请输入Y/N进行商品数据初始化确认! Y 删除全部商品表表然后重新创建,然后抓取商品数据,N(或者其他)跳过"); input = Console.ReadLine(); if (input.Equals("Y", StringComparison.OrdinalIgnoreCase)) { dbInit.InitCommodityTable(); CrawlerCommodity(); } Console.WriteLine("*****************^_^**********************"); //CleanAll(); }
/// <summary> /// 抓取 /// </summary> public static void Handler(Action <List <Category> > action = null) { // Console.WriteLine("请输入Y/N进行类别表初始化确认! Y 删除Category表然后重新创建,然后抓取类型数据,N(或者其他)跳过"); //string input = Console.ReadLine(); //if (input.Equals("Y", StringComparison.OrdinalIgnoreCase)) { DBInit.InitCategoryTable(sourcename); CrawlerCategory(action); } //else //{ // Console.WriteLine("你选择不初始化类别数据"); //} //Console.WriteLine("*****************^_^**********************"); // Console.WriteLine("请输入Y/N进行商品数据初始化确认! Y 删除全部商品表表然后重新创建,然后抓取商品数据,N(或者其他)跳过"); //input = Console.ReadLine(); //if (input.Equals("Y", StringComparison.OrdinalIgnoreCase)) { DBInit.InitCommodityTable(sourcename); CrawlerCommodity(); } Console.WriteLine("*****************^_^**********************"); }
static void Main(string[] args) { try { //http://search.jd.com/Search?keyword=%E5%85%85%E6%B0%94%E5%A8%83%E5%A8%83&enc=utf-8&suggest=3.his.0.0&wq=&pvid=60124febe81f4f4288a1d482b687f994 // string html = HttpHelper.DownloadHtml(@"http://search.jd.com/Search?keyword=%E5%85%85%E6%B0%94%E5%A8%83%E5%A8%83&enc=utf-8&suggest=3.his.0.0&wq=&pvid=60124febe81f4f4288a1d482b687f994", Encoding.UTF8); //测试抓取商品列表 // string testCategory = "{\"Id\":73,\"Code\":\"02f01s01T\",\"ParentCode\":\"02f01s\",\"Name\":\"烟机/灶具\",\"Url\":\"http://list.jd.com/737-794-1300.html\",\"Level\":3}"; // new CommoditySearch(JsonConvert.DeserializeObject<Category>(testCategory)).Crawler(); DBInit dbInit = new DBInit(); CategoryRepository categoryRepository = new CategoryRepository(); Console.WriteLine("请输入Y/N进行类别表初始化确认! Y 删除Category表然后重新创建,然后抓取类型数据,N(或者其他)跳过"); string input = Console.ReadLine(); if (input.Equals("Y", StringComparison.OrdinalIgnoreCase)) { dbInit.InitCategoryTable(); List <Category> categoryList = CategorySearch.Crawler("http://www.jd.com/allSort.aspx"); categoryRepository.Save(categoryList); Console.WriteLine("类型数据初始化完成,共抓取类别{0}个", categoryList.Count); } else { Console.WriteLine("你选择不初始化类别数据"); } Console.WriteLine("*****************^_^**********************"); Console.WriteLine("请输入Y/N进行商品数据初始化确认! Y 删除全部商品表表然后重新创建,然后抓取商品数据,N(或者其他)跳过"); input = Console.ReadLine(); if (input.Equals("Y", StringComparison.OrdinalIgnoreCase)) { dbInit.InitCommodityTable(); CrawlerCommodity(); } Console.WriteLine("*****************^_^**********************"); //CleanAll(); Console.ReadLine(); } catch (Exception ex) { logger.Error("异常啦,", ex); Console.WriteLine("异常啦,{ 0} ", ex); Console.WriteLine("*****************木有成功**********************"); Console.ReadLine(); } }
static void Main(string[] args) { try { //测试DownloadHtml string html = HttpHelper.DownloadHtml(@"http://www.mafengwo.cn/yj/10065/1-0-1.html", Encoding.UTF8); //测试获取分类页 string html1 = HttpHelper.DownloadCategory("http://list.jd.com/list.html?cat=737%2C794%2C798&go=0"); //测试抓取商品列表 string testCategory = "{\"Id\":73,\"Code\":\"02f01s01T\",\"ParentCode\":\"02f01s\",\"Name\":\"烟机/灶具\",\"Url\":\"http://list.jd.com/737-794-1300.html\",\"Level\":3}"; new CommoditySearch(JsonConvert.DeserializeObject <Category>(testCategory)).Crawler(); DBInit dbInit = new DBInit(); CategoryRepository categoryRepository = new CategoryRepository(); TravelNodesListRepository Travel = new TravelNodesListRepository(); Console.WriteLine("请输入Y/N进行类别表初始化确认! Y 删除TravelNodes表然后重新创建,然后抓取类型数据,N(或者其他)跳过"); string input = Console.ReadLine(); if (input.Equals("Y", StringComparison.OrdinalIgnoreCase)) { //dbInit.InitCategoryTable(); //List<Category> categoryList = CategorySearch.Crawler("http://www.jd.com/allSort.aspx"); //categoryRepository.Save(categoryList); //Console.WriteLine("类型数据初始化完成,共抓取类别{0}个", categoryList.Count); dbInit.InitTravelNotesListTable(); List <TravelNotesList> TravelList = TravelNodesSearch.Crawler("http://www.mafengwo.cn/yj/10065/1-0-1.html"); Travel.Save(TravelList); Console.WriteLine("类型数据初始化完成,共抓取类别{0}个", TravelList.Count); } else { Console.WriteLine("你选择不初始化类别数据"); } Console.WriteLine("*****************^_^**********************"); Console.WriteLine("请输入Y/N进行商品数据初始化确认! Y 删除全部商品表表然后重新创建,然后抓取商品数据,N(或者其他)跳过"); input = Console.ReadLine(); if (input.Equals("Y", StringComparison.OrdinalIgnoreCase)) { dbInit.InitCommodityTable(); CrawlerCommodity(); } Console.WriteLine("*****************^_^**********************"); //CleanAll(); Console.ReadLine(); } catch (Exception ex) { logger.Error("异常啦,", ex); Console.WriteLine("*****************木有成功**********************"); Console.ReadLine(); } }