示例#1
0
        /// <summary>
        /// 分页查询
        /// </summary>
        /// <param name="page"></param>
        /// <param name="limit"></param>
        /// <param name="queryModel"></param>
        /// <param name="orderFild"></param>
        /// <param name="isAsc"></param>
        /// <returns></returns>
        public override Page <Qq_Article> Page(long page, long limit, Qq_Article queryModel, string orderFild, string orderType = "ASC")
        {
            var sql   = new StringBuilder("select * from t_qq_Articles where 1=1 ");
            var param = new List <SqlParameter>();

            if (!string.IsNullOrEmpty(queryModel.Title))
            {
                sql.Append(" and Title like @Title ");
                param.Add(new SqlParameter("@Title", queryModel.Title));
            }
            if (!string.IsNullOrEmpty(queryModel.Author))
            {
                sql.Append(" and Author like @Author ");
                param.Add(new SqlParameter("@Author", queryModel.Author));
            }
            if (!string.IsNullOrEmpty(queryModel.HeadContent))
            {
                sql.Append(" and HeadContent like @HeadContent ");
                param.Add(new SqlParameter("@HeadContent", queryModel.HeadContent));
            }
            //排序
            if (!string.IsNullOrEmpty(orderFild))
            {
                sql.Append(string.Format(" order by {0} {1} ", orderFild, orderType));
            }
            return(GiftDB.GetInstance().Page <Qq_Article>(page, limit, sql.ToString(), param.ToArray()));
        }
示例#2
0
        public void SatrtSpider()
        {
            List <Request> resList = new List <Request>();
            var            site    = new DotnetSpider.Core.Site()
            {
                EncodingName = "UTF-8"
            };

            site.CycleRetryTimes = 1;
            //循环获取29页句子迷原创句子
            var db   = GiftDB.GetInstance();
            var list = db.Query <ArticleDetail>("SELECT * FROM t_gift_ArticleDetails");

            foreach (var item in list)
            {
                //var url = $"http://www.liwushuo.com/posts/{item.PID}/content";
                var url = $"http://www.liwushuo.com/posts/{item.PID}";

                site.AddStartUrl(url);
            }
            var spider = Spider.Create(site, new GiftPageProcessor())
                         .AddStartRequests(resList.ToArray())
                         .AddPipeline(new GiftPipeliner());

            spider.ThreadNum = 1;
            spider.Run();
        }
示例#3
0
        public override void Process(IEnumerable<ResultItems> resultItems, ISpider spider)
        {
            var list = new List<ArticleDetail>();
            foreach (var item in resultItems)
            {
                list = (List<ArticleDetail>)item.Results["key"];

            }
            if (list.Count == 0)
            {
                Console.WriteLine("****************************************可能已经结束 ****************************************");
            }
            foreach (var item in list)
            {
                var db = GiftDB.GetInstance();
                item.PID = item.id.ToString();
                item.id = 0;
                db.Insert(item);
            }
        }
示例#4
0
        /// <summary>
        /// 数据处理 (保存到sql)
        /// </summary>
        /// <param name="resultItems"></param>
        /// <param name="spider"></param>
        public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider)
        {
            var list = new List <Article>();

            foreach (var item in resultItems)
            {
                var model = new Article
                {
                    Type        = "1",
                    Remark      = "礼物说",
                    CreateTime  = DateTime.Now,
                    PID         = item.GetResultItem("pid"),
                    ContentHtml = item.GetResultItem("content"),
                    Title       = item.GetResultItem("title"),
                    Image       = item.GetResultItem("image")
                };
                list.Add(model);
            }

            foreach (var item in list)
            {
                GiftDB.GetInstance().Insert(item);
            }
        }