public static async void Crawl() { var context = SQLiteContextFactory.GetContext(); (await CrawlNewList().ConfigureAwait(true)).ForEach(obj => { var query = context.Query <ListArticle>().Where(a => a.Source == obj.Source && a.Title == obj.Title); if (query.Count() > 0) { logger.Debug($"内容已存在:{obj}"); return; } obj = context.Insert(obj); logger.Debug($"列表:{obj}"); Console.WriteLine($"列表:{obj}"); }); (await CrawlHotList().ConfigureAwait(true)).ForEach(obj => { var query = context.Query <ListArticle>().Where(a => a.Source == obj.Source && a.Title == obj.Title); if (query.Count() > 0) { logger.Debug($"内容已存在:{obj}"); return; } obj = context.Insert(obj); logger.Debug($"列表:{obj}"); Console.WriteLine($"列表:{obj}"); }); }
public SpiderModule() : base("/spider") { Get("ithome/hot_list", async _ => { var data = await ITHome.CrawlHotList().ConfigureAwait(false); var response = (Response)JsonConvert.SerializeObject(data); response.ContentType = "application/json"; return(response); }); Get("ithome/new_list", async _ => { var data = await ITHome.CrawlNewList().ConfigureAwait(false); var response = (Response)JsonConvert.SerializeObject(data); response.ContentType = "application/json"; return(response); }); Get("ithome/crawl", _ => { ITHome.Crawl(); return("已经开始任务"); }); Get("ithome/article/{id}", async param => { var id = param["id"]; var context = SQLiteContextFactory.GetContext(); ListArticle listArticle = context.QueryByKey <ListArticle>(id); var article = await ITHome.CrawlArticle(listArticle.Link); return(JsonConvert.SerializeObject(article)); }); Get("cnblog/list/{page_count}", async param => { var pageCount = param["page_count"]; var data = await CnBlog.CrawlList(pageCount); return(JsonConvert.SerializeObject(data)); }); }