public static async void Crawl()
{
var context = SQLiteContextFactory.GetContext();
(await CrawlNewList().ConfigureAwait(true)).ForEach(obj =>
{
var query = context.Query <ListArticle>().Where(a => a.Source == obj.Source && a.Title == obj.Title);
if (query.Count() > 0)
{
logger.Debug($"内容已存在：{obj}");
return;
}
obj = context.Insert(obj);
logger.Debug($"列表：{obj}");
Console.WriteLine($"列表：{obj}");
});
(await CrawlHotList().ConfigureAwait(true)).ForEach(obj =>
{
var query = context.Query <ListArticle>().Where(a => a.Source == obj.Source && a.Title == obj.Title);
if (query.Count() > 0)
{
logger.Debug($"内容已存在：{obj}");
return;
}
obj = context.Insert(obj);
logger.Debug($"列表：{obj}");
Console.WriteLine($"列表：{obj}");
});
}
public SpiderModule() : base("/spider")
{
Get("ithome/hot_list", async _ =>
{
var data = await ITHome.CrawlHotList().ConfigureAwait(false);
var response = (Response)JsonConvert.SerializeObject(data);
response.ContentType = "application/json";
return(response);
});
Get("ithome/new_list", async _ =>
{
var data = await ITHome.CrawlNewList().ConfigureAwait(false);
var response = (Response)JsonConvert.SerializeObject(data);
response.ContentType = "application/json";
return(response);
});
Get("ithome/crawl", _ =>
{
ITHome.Crawl();
return("已经开始任务");
});
Get("ithome/article/{id}", async param =>
{
var id = param["id"];
var context = SQLiteContextFactory.GetContext();
ListArticle listArticle = context.QueryByKey <ListArticle>(id);
var article = await ITHome.CrawlArticle(listArticle.Link);
return(JsonConvert.SerializeObject(article));
});
Get("cnblog/list/{page_count}", async param =>
{
var pageCount = param["page_count"];
var data = await CnBlog.CrawlList(pageCount);
return(JsonConvert.SerializeObject(data));
});
}