Example #1
0
        public static async void Crawl()
        {
            var context = SQLiteContextFactory.GetContext();

            (await CrawlNewList().ConfigureAwait(true)).ForEach(obj =>
            {
                var query = context.Query <ListArticle>().Where(a => a.Source == obj.Source && a.Title == obj.Title);
                if (query.Count() > 0)
                {
                    logger.Debug($"内容已存在:{obj}");
                    return;
                }
                obj = context.Insert(obj);
                logger.Debug($"列表:{obj}");
                Console.WriteLine($"列表:{obj}");
            });

            (await CrawlHotList().ConfigureAwait(true)).ForEach(obj =>
            {
                var query = context.Query <ListArticle>().Where(a => a.Source == obj.Source && a.Title == obj.Title);
                if (query.Count() > 0)
                {
                    logger.Debug($"内容已存在:{obj}");
                    return;
                }
                obj = context.Insert(obj);
                logger.Debug($"列表:{obj}");
                Console.WriteLine($"列表:{obj}");
            });
        }
        public SpiderModule() : base("/spider")
        {
            Get("ithome/hot_list", async _ =>
            {
                var data             = await ITHome.CrawlHotList().ConfigureAwait(false);
                var response         = (Response)JsonConvert.SerializeObject(data);
                response.ContentType = "application/json";
                return(response);
            });

            Get("ithome/new_list", async _ =>
            {
                var data             = await ITHome.CrawlNewList().ConfigureAwait(false);
                var response         = (Response)JsonConvert.SerializeObject(data);
                response.ContentType = "application/json";
                return(response);
            });

            Get("ithome/crawl", _ =>
            {
                ITHome.Crawl();
                return("已经开始任务");
            });

            Get("ithome/article/{id}", async param =>
            {
                var id                  = param["id"];
                var context             = SQLiteContextFactory.GetContext();
                ListArticle listArticle = context.QueryByKey <ListArticle>(id);
                var article             = await ITHome.CrawlArticle(listArticle.Link);
                return(JsonConvert.SerializeObject(article));
            });


            Get("cnblog/list/{page_count}", async param =>
            {
                var pageCount = param["page_count"];
                var data      = await CnBlog.CrawlList(pageCount);
                return(JsonConvert.SerializeObject(data));
            });
        }