public void PageProcess(PagePathogen pagePathogen)
{
var selector = new XPathSelector(pagePathogen.PageSource);
//小说
Novel novel = new Novel();
//文章信息
novel.Articles = new List <Article>();
var nameEle = selector.SelectSingleNode("//*[@id='info']/h1");
if (nameEle != null)
{
//小说名称
novel.Name = nameEle.InnerText;
}
var authorEle = selector.SelectSingleNode("//*[@id='info']/p[1]");
if (authorEle != null)
{
string pStr = authorEle.InnerText;
//作者
novel.Author = pStr.Split('：')[1];
}
//获取对应文章信息
GetArticles(selector, novel, pagePathogen.Url, "//*[@id='list']/dl/dd/a");
//传递抓取数据信息
pagePathogen.AddResult("novel", novel);
}
/// <summary>
/// 页面解析
/// </summary>
/// <param name="pagePathogen"></param>
public void PageProcess(PagePathogen pagePathogen)
{
try
{
//添加请求地址
pagePathogen.AddResult("requestUrl", pagePathogen.Url);
var selector = new XPathSelector(pagePathogen.PageSource);
var node = selector.SelectSingleNode("//*[@id='content']");
if (node != null)
{
pagePathogen.AddResult("article", node.InnerHtml);
}
else
{
//记录爬取日志
_loggerService.WriteLog(new Log()
{
DateTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"),
Msg = pagePathogen.Url + "---未解析到数据！",
ClassName = "",
ActionName = "",
Duration = 0,
LogLevel = (int)LCore.Logger.LogLevel.Warn
});
}
}
catch (Exception e)
{
//记录错误信息
_loggerService.WriteLog(new Log()
{
DateTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"),
LogLevel = (int)LCore.Logger.LogLevel.Error,
ClassName = this.GetType().FullName,
ActionName = e.TargetSite.Name,
Msg = pagePathogen.Url + "---" + e.Message + "---" + e.StackTrace
});
}
}
public override void HtmlParser(OnCompleteEventArgs e)
{
lock (_lock)
{
try
{
var stopWatch = new Stopwatch();
stopWatch.Start();
var article = Current;
if (!string.IsNullOrEmpty(e.Page))
{
var selector = new XPathSelector(e.Page);
var node = selector.SelectSingleNode("//*[@id='content']");
if (node != null)
{
//获取小说内容
string content = node.InnerHtml;
article.Content = content;
article.IsCrawlerContent = true;
//更新信息
_novelService.UpdateArticel(article);
//是否启动邮件发送
if (article.Novel != null && article.Novel.IsOpenEmail)
{
//发送邮件
EmailHelper.SendEmail(article.Title, content, new List <string>()
{
"*****@*****.**"
});
}
stopWatch.Stop();
//记录爬取日志
_loggerService.WriteLog(new Log()
{
DateTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"),
Msg = e.Uri + "请求消耗:" + e.Duration + "---" + "数据解析消耗:" + stopWatch.ElapsedMilliseconds,
ClassName = "",
ActionName = "",
Duration = e.Duration + stopWatch.ElapsedMilliseconds,
LogLevel = (int)LCore.Logger.LogLevel.Info
});
}
else
{
//记录爬取日志
_loggerService.WriteLog(new Log()
{
DateTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"),
Msg = e.Uri + "---未解析到数据！",
ClassName = "",
ActionName = "",
Duration = e.Duration + stopWatch.ElapsedMilliseconds,
LogLevel = (int)LCore.Logger.LogLevel.Warn
});
}
}
}
catch (Exception exception)
{
//记录错误信息
_loggerService.WriteLog(new Log()
{
DateTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"),
LogLevel = (int)LCore.Logger.LogLevel.Error,
ClassName = this.GetType().FullName,
ActionName = exception.TargetSite.Name,
Msg = e.Uri + "---" + exception.StackTrace
});
}
}
}
/// <summary>
/// html解析
/// </summary>
/// <param name="e"></param>
public override void HtmlParser(OnCompleteEventArgs e)
{
try
{
lock (_lock)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
var selector = new XPathSelector(e.Page);
//小说
Novel novel = new Novel();
//文章信息
novel.Articles = new List <Article>();
#region 小说信息
var nameEle = selector.SelectSingleNode("//*[@id='info']/h1");
if (nameEle != null)
{
novel.Name = nameEle.InnerText;
}
var authorEle = selector.SelectSingleNode("//*[@id='info']/p[1]");
if (authorEle != null)
{
string pStr = authorEle.InnerText;
novel.Author = pStr.Split('：')[1];
}
#endregion 小说信息
var oldNovel = _novelService.GetSingleNovel(new NovelSearchInput()
{
Name = novel.Name
});
if (oldNovel == null)
{
GetArticles(selector, novel, e.Uri, "//*[@id='list']/dl/dd/a");
if (novel.Articles.Count > 0)
{
//设为已爬取
novel.IsCrawlerArticle = true;
}
//添加小说
_novelService.AddNovel(novel);
}
else
{
var laestArticle = _novelService.GetLaestArticle();
oldNovel.Articles = new List <Article>();
//获取最新章节
GetArticles(selector, oldNovel, e.Uri, "//*[@id='list']/dl/dd/a[number(translate(@href,'.html',''))>" + laestArticle.Seq + "]");
//更新新章节
_novelService.AddArticles(oldNovel.Articles.ToList());
}
stopWatch.Stop();
//记录爬取日志
_loggerService.WriteLog(new Log()
{
DateTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"),
Msg = e.Uri + "请求消耗:" + e.Duration + "---" + "数据解析消耗:" + stopWatch.ElapsedMilliseconds,
ClassName = "",
ActionName = "",
Duration = e.Duration + stopWatch.ElapsedMilliseconds,
LogLevel = (int)LCore.Logger.LogLevel.Info
});
}
}
catch (Exception exception)
{
//记录错误信息
_loggerService.WriteLog(new Log()
{
DateTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"),
LogLevel = (int)LCore.Logger.LogLevel.Error,
ClassName = this.GetType().Name,
ActionName = exception.TargetSite.Name,
Msg = e.Uri + "---" + exception.Message
});
}
}