/// <summary>
/// 分页查询
/// </summary>
/// <param name="page"></param>
/// <param name="limit"></param>
/// <param name="queryModel"></param>
/// <param name="orderFild"></param>
/// <param name="isAsc"></param>
/// <returns></returns>
public override Page <Qq_Article> Page(long page, long limit, Qq_Article queryModel, string orderFild, string orderType = "ASC")
{
var sql = new StringBuilder("select * from t_qq_Articles where 1=1 ");
var param = new List <SqlParameter>();
if (!string.IsNullOrEmpty(queryModel.Title))
{
sql.Append(" and Title like @Title ");
param.Add(new SqlParameter("@Title", queryModel.Title));
}
if (!string.IsNullOrEmpty(queryModel.Author))
{
sql.Append(" and Author like @Author ");
param.Add(new SqlParameter("@Author", queryModel.Author));
}
if (!string.IsNullOrEmpty(queryModel.HeadContent))
{
sql.Append(" and HeadContent like @HeadContent ");
param.Add(new SqlParameter("@HeadContent", queryModel.HeadContent));
}
//排序
if (!string.IsNullOrEmpty(orderFild))
{
sql.Append(string.Format(" order by {0} {1} ", orderFild, orderType));
}
return(GiftDB.GetInstance().Page <Qq_Article>(page, limit, sql.ToString(), param.ToArray()));
}
public void SatrtSpider()
{
List <Request> resList = new List <Request>();
var site = new DotnetSpider.Core.Site()
{
EncodingName = "UTF-8"
};
site.CycleRetryTimes = 1;
//循环获取29页句子迷原创句子
var db = GiftDB.GetInstance();
var list = db.Query <ArticleDetail>("SELECT * FROM t_gift_ArticleDetails");
foreach (var item in list)
{
//var url = $"http://www.liwushuo.com/posts/{item.PID}/content";
var url = $"http://www.liwushuo.com/posts/{item.PID}";
site.AddStartUrl(url);
}
var spider = Spider.Create(site, new GiftPageProcessor())
.AddStartRequests(resList.ToArray())
.AddPipeline(new GiftPipeliner());
spider.ThreadNum = 1;
spider.Run();
}
public override void Process(IEnumerable<ResultItems> resultItems, ISpider spider)
{
var list = new List<ArticleDetail>();
foreach (var item in resultItems)
{
list = (List<ArticleDetail>)item.Results["key"];
}
if (list.Count == 0)
{
Console.WriteLine("****************************************可能已经结束 ****************************************");
}
foreach (var item in list)
{
var db = GiftDB.GetInstance();
item.PID = item.id.ToString();
item.id = 0;
db.Insert(item);
}
}
/// <summary>
/// 数据处理 (保存到sql)
/// </summary>
/// <param name="resultItems"></param>
/// <param name="spider"></param>
public override void Process(IEnumerable <ResultItems> resultItems, ISpider spider)
{
var list = new List <Article>();
foreach (var item in resultItems)
{
var model = new Article
{
Type = "1",
Remark = "礼物说",
CreateTime = DateTime.Now,
PID = item.GetResultItem("pid"),
ContentHtml = item.GetResultItem("content"),
Title = item.GetResultItem("title"),
Image = item.GetResultItem("image")
};
list.Add(model);
}
foreach (var item in list)
{
GiftDB.GetInstance().Insert(item);
}
}