public DtoNews NewsGathering(string newsUrl)
{
try
{
var title = "";
var content = "";
var pubTime = "";
var from = "";
var author = "";
var strNewContent = HttpHelper.GetContentByMobileAgent(newsUrl, Encoding.UTF8);
content = XpathHelper.GetInnerHtmlByXPath(strNewContent, "//div[@id='artical_real']", "").Trim();
pubTime = XpathHelper.GetInnerHtmlByXPath(strNewContent, "//div[@id='artical_sth']/p/span[1]", "");
pubTime = StrHelper.FormatHtml(pubTime).Trim();
from = XpathHelper.GetInnerHtmlByXPath(strNewContent, "//div[@id='artical_sth']/p/span[3]/span", "");
from = StrHelper.FormatHtml(from).Trim();
author = XpathHelper.GetInnerHtmlByXPath(strNewContent, "//div[@id='artical_sth']/p/span[4]/span", "");
author = StrHelper.FormatHtml(author).Trim();
if (string.IsNullOrWhiteSpace(from))
{
from = "凤凰佛教";
}
content = DealContent(content);
var news = new DtoNews
{
Contents = content,
Title = title,
PubTime = StrHelper.ToDateTime(pubTime),
FromUrl = newsUrl,
FromSiteName = from,
Author = author,
CreateTime = DateTime.Now,
IsShow = 0,
};
return(news);
}
catch (Exception ex)
{
Log.Error(newsUrl + " 错误:" + ex.Message + ex.StackTrace);
}
return(null);
}
public List <DtoNewsUrlList> NewsUrlGathering(string newsListUrl, int newsType)
{
try
{
Log.Info(newsListUrl + " 抓取开始");
var strContent = HttpHelper.GetContentByMobileAgent(newsListUrl, Encoding.GetEncoding("gb2312"));
if (string.IsNullOrWhiteSpace(strContent))
{
Log.Info(newsListUrl + " 未抓取到任何内容");
return(null);
}
//取得标题列表
var strList = XpathHelper.GetInnerHtmlListByXPath(strContent, "//div[@class='leftList']/ul/li");
if (strList != null && strList.Count > 0)
{
foreach (var item in strList)
{
try
{
var url = XpathHelper.GetAttrValueByXPath(item, "//a", "href");
var title = XpathHelper.GetInnerHtmlByXPath(item, "//a", "");
title = StrHelper.FormatHtml(title).Trim();
var isHave = DalNews.IsExistsNews(title);
//如果已存在则跳过
if (isHave)
{
continue;
}
if (newsType == 100 || newsType == 200 || newsType == 300)
{
#region === 根据详细页地址取新闻内容 ===
var news = NewsGathering(url);
if (news != null)
{
news.NewsTypeId = newsType;
news.Title = title;
news.PubTime = StrHelper.ToDateTime(StrHelper.FormatPubTime(news.PubTime.ToString()));
//入库
var newsId = DalNews.Insert(news);
if (newsId < 1)
{
continue;
}
//从内容中提取img,存入newsmedia
var mediaList = ImgDeal.GetImgList(news.Contents);
if (mediaList != null && mediaList.Count > 0)
{
news.Contents = mediaList[0].Description;
foreach (var picitem in mediaList)
{
picitem.NewsId = newsId;
DalNews.InsertMedia(picitem);
}
}
//休眠 控制抓取的频率
Random rnd = new Random();
var sleepSeconds = rnd.Next(30, 90);
Thread.Sleep(sleepSeconds * 1000);
}
#endregion
}
if (newsType == 400)
{
#region === 根据详细页地址取图片内容 ===
var mediaList = NewsPicGathering(url);
var news = new DtoNews()
{
Title = title,
FromUrl = url,
NewsTypeId = newsType,
};
if (mediaList != null && mediaList.Count > 0)
{
news.Contents = mediaList[0].Description;
//入库
var newsId = DalNews.Insert(news);
foreach (var picitem in mediaList)
{
picitem.NewsId = newsId;
DalNews.InsertMedia(picitem);
}
}
//休眠 控制抓取的频率
Random rnd = new Random();
var sleepSeconds = rnd.Next(30, 90);
Thread.Sleep(sleepSeconds * 1000);
#endregion
}
Log.Info(url + " 抓取完成");
}
catch (Exception ex)
{
//Log.Error("内容: " + item);
Log.Error(ex.Message + ex.StackTrace);
}
}
}
Log.Info(newsListUrl + " 抓取结束");
return(null);
}
catch (Exception ex)
{
Log.Error(ex.Message + ex.StackTrace);
}
return(null);
}
public DtoNews NewsGathering(string newsUrl)
{
try
{
var title = "";
var content = "";
var pubTime = "";
var from = "";
var author = "";
var picUrl = "";
var strNewContent = HttpHelper.GetContent(newsUrl, Encoding.GetEncoding("gb2312"));
content = XpathHelper.GetInnerHtmlByXPath(strNewContent, "//div[@id='Cnt-Main-Article-QQ']", "");
//从content里去除最下面的广告部分
//var contentlast = StrHelper.GetStrByXPath(content, "//span[last()]", "");
//content = content.Replace(contentlast, "");
//content = Regex.Replace(content, contentlast, "", RegexOptions.IgnoreCase);
//从content里去除最上面的分享部分
//var contentfirst = StrHelper.GetStrByXPath(strNewContent, "//div[@class='tit-bar clearfix']", "");
//content = content.Replace(contentfirst, "");
//content = content.Replace("<div class='tit-bar clearfix' bosszone='titleDown'></div>","");
content = content.Trim();
pubTime = XpathHelper.GetInnerHtmlByXPath(strNewContent, "//span[@class='article-time']", "");
pubTime = StrHelper.FormatHtml(pubTime).Trim();
from = "腾讯佛学";
//from = StrHelper.GetStrByXPath(strNewContent, "//span[@bosszone='jgname']/a", "");
//from = StrHelper.FormatHtml(from);
var picUrlList = XpathHelper.GetAttrValueListByXPath(content, "//img", "src");
if (picUrlList != null && picUrlList.Count > 0)
{
picUrl = picUrlList[0];
}
author = XpathHelper.GetInnerHtmlByXPath(strNewContent, "//div[@id='C-Main-Article-QQ']/div[1]/div/div[1]/span[5]", "");
author = StrHelper.FormatHtml(author).Trim();
//*[@id="Cnt-Main-Article-QQ"]/p/div[@r='1']
content = DealContent(content);
var news = new DtoNews
{
Contents = content,
Title = title,
PubTime = StrHelper.ToDateTime(pubTime),
FromUrl = newsUrl,
FromSiteName = from,
Author = author,
CreateTime = DateTime.Now,
IsShow = 1,
LogoOriginalUrl = picUrl,
LogoUrl = picUrl
};
return(news);
}
catch (Exception ex)
{
Log.Error(newsUrl + " 错误:" + ex.Message + ex.StackTrace);
}
return(null);
}
