/// <summary>
/// 从给定的Html原始文本中获取正文信息
/// 只有单条链接情况，正文缺少推倒去重复
/// </summary>
/// <param name="html"></param>
/// <returns></returns>
public static Article GetArticle(string html)
{
string backupHtml = html;
// 过滤样式，脚本等不相干标签
foreach (var filter in Filters)
{
backupHtml = Regex.Replace(backupHtml, filter[0], filter[1]);
}
HtmlNode itemHtmlNode = HtmlUtility.getSafeHtmlRootNode(backupHtml, true, true);
Article article = new Article();
string title = GetTitle(itemHtmlNode);
if (!string.IsNullOrWhiteSpace(title))
{
article.Title = Regex.Replace(title, @"\s", "");
}
string dateTimeStr = GetPublishDate(backupHtml);
article.PubDate = DateTimeParser.Parser(dateTimeStr);
article.HtmlContent = GetHtmlContent(backupHtml, article.Title, dateTimeStr);
if (!string.IsNullOrWhiteSpace(article.HtmlContent))
{
article.Content = HTMLCleaner.CleanHTML(article.HtmlContent, false);
}
return(article);
}
internal int CountTextofNode(HtmlNode Node)
{
if (Node.XPath.Contains("/script") || Node.XPath.Contains("/style") || Node.XPath.Contains("/meta"))
{
return(0);
}
if (Node.Name.Contains("#") && !Node.Attributes.Contains("cutting"))
{
return(HTMLCleaner.GetCleanInnerText(Node).Length);
}
if (Node.ChildNodes.Count == 0)
{
return(0);
}
int count = 0;
foreach (HtmlNode node in Node.ChildNodes)
{
if (node.Attributes.Contains("cutting"))
{
continue;
}
if (node.Name.Contains("#"))
{
count += HTMLCleaner.GetCleanInnerText(node).Length;
}
else
{
count += CountTextofNode(node);
}
}
return(count);
}
private void FillUserInfo(string content, UserTweet userTweet)
{
var match = Regex.Match(content, RegexInfo, RegexOptions.Multiline | RegexOptions.IgnoreCase);
userTweet.Follow = int.Parse(match.Groups["Follow"].Value);
userTweet.Follower = int.Parse(match.Groups["Follower"].Value);
userTweet.TweetNum = int.Parse(match.Groups["TweetNum"].Value);
userTweet.Name = HTMLCleaner.CleanHTML(match.Groups["Name"].Value, true);
userTweet.Uid = match.Groups["Uid"].Value;
}
/// <summary>
/// Match2s the item.
/// </summary>
/// <param name="m">M.</param>
/// <param name="Item">Item.</param>
/// <param name="BaseUrl">Base URL.</param>
/// <param name="ItemUrlCaseSensitive">If set to <c>true</c> item URL case sensitive.</param>
public static void Match2Item(Match m, ref Article Item, string BaseUrl, bool ItemUrlCaseSensitive = false)
{
//url
Item.Url = new Uri(new Uri(BaseUrl), RegexUtility.TryGetString(m, "Url", Item.Url, false)).AbsoluteUri;
//title
Item.Title = RegexUtility.TryGetString(m, "Title", Item.Title);
//降低Clean级别
if (string.IsNullOrEmpty(Item.Title))
{
Item.Title = HTMLCleaner.CleanHTML(Item.Title, true);
}
//text
Item.HtmlContent = RegexUtility.TryGetString(m, "Text", Item.HtmlContent, false);
//Author Info
Item.Author = RegexUtility.TryGetString(m, "AuthorName", Item.Author);
Item.Source = RegexUtility.TryGetString(m, "Source", Item.Source);
if (!String.IsNullOrWhiteSpace(Item.Source))
{
Item.Source = TextCleaner.FullClean(Item.Source);
}
//Media Info
Item.MediaName = RegexUtility.TryGetString(m, "MediaName", Item.MediaName);
//time
if (m.Groups["PubDate"].Success)
{
Item.PubDate = DateTimeParser.Parser(HTMLCleaner.CleanHTML(m.Groups["PubDate"].Value, true));
}
if (Item.PubDate <= DateTime.MinValue)
{
Item.PubDate = DateTime.Now;
}
Match2ItemCount(m, Item.ViewDataList);
}
public static bool validateNextPage(string Html, string Pattern, ref string Url)
{
HtmlNode htmlNode = HtmlUtility.getSafeHtmlRootNode(Html);
List <HtmlNode> atagHtmlNodes = htmlNode.SelectNodes("//a[@href]").ToList();
List <testNextUrl> x = new List <testNextUrl>();
StringBuilder Result = new StringBuilder();
int intX = 0;
foreach (HtmlNode tmpNode in atagHtmlNodes)
{
intX++;
if (Regex.Match(tmpNode.InnerText, @".*[一二三四五六七八九十\d页].*").Success)
{
testNextUrl tmp = new testNextUrl();
tmp.index = intX;
tmp.urlText = HTMLCleaner.CleanHTML(tmpNode.InnerText, true);
tmp.urlLink = HtmlUtility.AbsoluteUrl(tmpNode.Attributes["href"].Value, Url, true);
x.Add(tmp);
}
}
if (x.Count > 0)
{
if (Url.Equals(TrianNextUrl(x).urlLink))
{
return(false);
}
else
{
Url = TrianNextUrl(x).urlLink;
}
}
if (Url.Length == 0)
{
return(false);
}
else
{
return(true);
}
}
/// <summary>
/// 验证标题是否合法
/// </summary>
/// <param name="Title"></param>
/// <returns></returns>
public bool ValidateTitle(HtmlNode Title)
{
return(ValidateTitle(XPathUtility.InnerTextNonDescendants(Title)) && !(Title.Attributes["href"] == null) && HTMLCleaner.isUrlGood(Title.Attributes["href"].Value));
}
private static void WechatCrawler()
{
using (var client = new RedisAccess(redisPool))
{
try
{
//产生临时库（临时库和缓存总库取并集）
string wechatItem = client.PopItemWithHighestScoreFromSortedSet(wechatSubsKey);
if (!string.IsNullOrEmpty(wechatItem))
{
WeChatItemModel wechatItemModel = JsonConvert.DeserializeObject <WeChatItemModel>(wechatItem);
Palas.Protocol.PFItemToAnalyze pfItemToAnalyze = new Palas.Protocol.PFItemToAnalyze();
Palas.Protocol.PFItem pfItem = new Palas.Protocol.PFItem();
if (wechatItemModel != null && !string.IsNullOrEmpty(wechatItemModel.Id))
{
pfItem.AuthorID = wechatItemModel.WechatSubId;
pfItem.AuthorName = wechatItemModel.WechatSubName;
pfItem.ContentDetailLevel = Palas.Protocol.PFContentDetailLevel.FullContent;
pfItem.Crawler = "WechatSubs";
pfItem.CrawlID = wechatItemModel.WechatSubId;
pfItem.FetchTime = DateTimeUtility.GetUnixTimeStamp(DateTime.Now);
pfItem.ItemID = wechatItemModel.Id;
pfItem.MediaID = "WeChat";
pfItem.MediaName = "微信";
pfItem.PubDate = DateTimeUtility.GetUnixTimeStamp(wechatItemModel.PubDate);
pfItem.Url = wechatItemModel.Url.Replace("&", "&");
pfItem.CleanTitle = wechatItemModel.Title;
if (wechatItemModel.Tags != null && wechatItemModel.Tags.Count > 0)
{
pfItem.Tag = string.Join(",", wechatItemModel.Tags);
}
string content = GetHtmlContent(pfItem.Url);
if (!string.IsNullOrEmpty(content))
{
pfItem.HTMLText = HtmlFormattor.FormatHtml(content, pfItem.Url);
pfItem.CleanText = HTMLCleaner.CleanHTML(pfItem.HTMLText, true);
pfItemToAnalyze.AnalyzeFlag = 312287;
pfItemToAnalyze.Item = pfItem;
pfItemToAnalyze.CrawlRecode = new Palas.Protocol.PFCrawlRecode(pfItem.Crawler, pfItem.CrawlID, "Common", "", "N", 312287, 1, "WeChatSync", 0, 0, 0, "", 0, 0, true, pfItem.MediaID, pfItem.MediaName, pfItem.MediaChannel);
pfItemToAnalyze.CrawlRecode.Tags = pfItem.Tag;
pfItemToAnalyze.CrawlRecode.JoinIssueIDs = string.Join(",", wechatItemModel.Customers);
pfItemToAnalyze.CrawlRecode.MediaType = 11;
pfItemToAnalyze.CrawlRecode.MediaWeight = 8;
// 临时追加
if (wechatItemModel.Tags != null && wechatItemModel.Tags.Count > 0)
{
pfItem.Tag = string.Join(",", wechatItemModel.Tags);
//pfItemToAnalyze.CrawlRecode.Tags = pfItem.Tag;
}
// 临时追加完毕
//pfItemToAnalyze.CrawlRecode = new Palas.Protocol.PFCrawlRecode(pfItem.Crawler, pfItem.CrawlID, "Common", "", "N", 312287, string.Join(",", wechatItemModel.Customers), 1, "WeChatSync", 0, 0, "", 0, "", "", 0, 0, "", true, 4, pfItem.MediaID, pfItem.MediaName, "", 11, 0, 8, 0, "", "",);
Palas.Protocol.PFItemToAnalyze analyzeResult = AnalyzeItem.Analyzer(pfItemToAnalyze, Palas.Protocol.PFAnalyzeFlag.Splite);
//6.使用AnalyzeFirst对文章进行第一次分析
MultriAnalyzeFlag analyzeFlags = AnalyzeItem.BuildAnalyzeFlag((Enums.AnalyzeFlag)analyzeResult.AnalyzeFlag);
Palas.Protocol.PFItemToAnalyze analyzeFirstResult = AnalyzeItem.AnalyzerFirst(analyzeResult, analyzeFlags);
//7.使用AnalyzeSecond对文章进行第二次分析
Palas.Protocol.PFItemToAnalyze analyzeSecondResult = AnalyzeItem.AnalyzeSecond(analyzeFirstResult, analyzeFlags);
//8.使用AnalyzeIssue对文章进行分Issue分析
Palas.Protocol.PFItemToAnalyze analyzeIssueResult = AnalyzeItem.IssueAnalyzer(analyzeSecondResult, analyzeFlags);
//9.使用IndexThenDup将文章去重索引到ES
if (analyzeIssueResult != null)
{
//此处对Item进行一次转换
Palas.Protocol.PFItem pfItem2 = analyzeIssueResult.Item;
Item _item = TypeExchangeUtility.ExchangeItem(pfItem2);
_item = FilterIssue.FilterExcludeExpression(_item);
Enums.ProcessStatus result = Enums.ProcessStatus.Failed;
int retry = 0;
do
{
try
{
//retry++;
ESAccess.IndexOnly(_item);
result = Enums.ProcessStatus.Succeeded;
//result = DupThenIndexBusiness.DupThenIndexItem(_item);
//if (result == Enums.ProcessStatus.Failed)
//Thread.Sleep(new TimeSpan(0, 0, 30));
}
catch //(Exception ex)
{
result = Enums.ProcessStatus.Failed;
Thread.Sleep(new TimeSpan(0, 1, 00));
}
}while (result == Enums.ProcessStatus.Failed && retry < 3);
//10.判断是否成功
//Assert.AreNotEqual(Enums.ProcessStatus.Failed, result);
Console.WriteLine(string.Format("Index paper: {0}, Status: {1}", _item.ItemID, result.ToString()));
}
return;
}
}
}
else
{
//if have nothing to emit, then sleep for a little while to release CPU
Thread.Sleep(50);
}
}
catch (Exception ex)
{
}
}
}
public static bool ParseItem(string Html, string Pattern, string Url, ref Article BaseArticle)
{
//输入检查
if (string.IsNullOrWhiteSpace(Html) || string.IsNullOrWhiteSpace(Pattern))
{
return(false);
}
//检查 Pattern 的格式，判断是否符合要求
XpathPattern xpathPattern = null;
try
{
xpathPattern = JsonConvert.DeserializeObject <XpathPattern>(Pattern);
}
catch (Exception ex)
{
Logger.Error(string.Format("Pattern 的格式不符合 Xpath Parser 的定义，请检查！Url:{0}, Pattern:{1}.", Url, Pattern), ex);
}
HtmlNode itempagenode = HtmlUtility.getSafeHtmlRootNode(Html, true, true);
//提取文章正文
if (string.IsNullOrEmpty(BaseArticle.HtmlContent) && !string.IsNullOrWhiteSpace(xpathPattern.ItemContentXPath))
{
try
{
BaseArticle.HtmlContent = HTMLCleaner.CleanContent(itempagenode.SelectNodes(xpathPattern.ItemContentXPath), Url, true);
BaseArticle.Content = HTMLCleaner.CleanHTML(BaseArticle.HtmlContent, false);
}
catch (Exception ex)
{
Logger.Error(string.Format("从详情页解析正文出错，Url:{0}, Pattern:{1}.", Url, xpathPattern.ItemContentXPath), ex);
}
}
//确认标题
if (string.IsNullOrEmpty(BaseArticle.Title) && !string.IsNullOrWhiteSpace(xpathPattern.ItemTitleXPath))
{
try
{
BaseArticle.Title = TextCleaner.FullClean(HTMLCleaner.GetCleanInnerText(itempagenode.SelectSingleNode(xpathPattern.ItemTitleXPath)));
}
catch (Exception ex)
{
Logger.Error(string.Format("从详情页解析标题出错，Url:{0}, Pattern:{1}.", Url, xpathPattern.ItemTitleXPath), ex);
}
}
//确认时间
if (!string.IsNullOrWhiteSpace(xpathPattern.ItemDateXPath))
{
try
{
DateTime Pubdate = DateTimeParser.Parser(HTMLCleaner.GetCleanInnerText(itempagenode.SelectSingleNode(xpathPattern.ItemDateXPath)));
if (BaseArticle.PubDate <= DateTime.MinValue.AddYears(1) && Pubdate.Year > 2000) //发布时间过旧
{
BaseArticle.PubDate = Pubdate;
}
else if (BaseArticle.PubDate.Hour == 0 && BaseArticle.PubDate.Minute == 0 && (Pubdate.Hour != 0 || Pubdate.Minute != 0) && Pubdate.Year > 2000) //发布时间没有时与分
{
BaseArticle.PubDate = Pubdate;
}
else if (Pubdate.Year > 2000 && (Pubdate.Hour != 0 || Pubdate.Minute != 0) && (BaseArticle.PubDate - Pubdate) > new TimeSpan(0, 1, 59) && BaseArticle.PubDate >= DateTime.Now.AddMinutes(-10)) //发布时间拒当前时间很近且相差较大
{
BaseArticle.PubDate = Pubdate;
}
}
catch (Exception ex)
{
Logger.Error(string.Format("从详情页解析标题出错，Url:{0}, Pattern:{1}.", Url, xpathPattern.ItemContentXPath), ex);
}
}
//确认媒体
if (string.IsNullOrEmpty(BaseArticle.MediaName) && !string.IsNullOrWhiteSpace(xpathPattern.ItemMediaNameXPath))
{
try
{
BaseArticle.MediaName = TextCleaner.FullClean(HTMLCleaner.GetCleanInnerText(itempagenode.SelectSingleNode(xpathPattern.ItemMediaNameXPath)));
}
catch (Exception ex)
{
Logger.Error(string.Format("从详情页解析媒体出错，Url:{0}, Pattern:{1}.", Url, xpathPattern.ItemMediaNameXPath), ex);
}
}
//确认作者
if (string.IsNullOrEmpty(BaseArticle.Author) && !string.IsNullOrWhiteSpace(xpathPattern.ItemAuthorXPath))
{
try
{
BaseArticle.Author = TextCleaner.FullClean(HTMLCleaner.GetCleanInnerText(itempagenode.SelectSingleNode(xpathPattern.ItemAuthorXPath)));
}
catch (Exception ex)
{
Logger.Error(string.Format("从详情页解析作者出错，Url:{0}, Pattern:{1}.", Url, xpathPattern.ItemAuthorXPath), ex);
}
}
return(true);
}
/// <summary>
/// 从List页面上根据各字段XPath提取内容集合
/// </summary>
/// <param name="Url">网址</param>
/// <param name="RootNode">Document的根节点</param>
/// <param name="Path">根据此ListPath来提取内容</param>
/// <param name="List_MinCountItem">至少List几个Item（用于判定旧网站中大量A堆砌在同一个元素下的情况）</param>
/// <param name="needscalepages">是否需要翻页，默认为否</param>
/// <returns></returns>
public static List <Article> ExtractItemFromList(string Url, HtmlNode RootNode, XpathPattern Path)
{
List <Article> Content = new List <Article>();
//fix a null bug by carey. 2014-09-10
HtmlNodeCollection rootNodes = RootNode.SelectNodes(Path.ItemRootXPath);
if (rootNodes != null && rootNodes.Count > 0)
{
foreach (HtmlNode BaseNode in rootNodes)
{
//正常情况下，每个BaseNode有一个Item，但是某些网站可能存在多个
if (string.IsNullOrWhiteSpace(Path.TitleXPath) || BaseNode.SelectNodes(Path.TitleXPath) == null)
{
continue;
}
//如果 BaseNode 的数量小于6，则判断是否存在多个可匹配的 Title 项；如果存在的话则记录数量
List <HtmlNode> nodecollection = new List <HtmlNode>();
int singleNodeItemCount = 0;
if (!string.IsNullOrWhiteSpace(Path.UrlXPath))
{
nodecollection = BaseNode.SelectNodes(Path.UrlXPath).Where(n => n.Attributes.Contains("href")).ToList();
}
else
{
nodecollection = BaseNode.SelectNodes(Path.TitleXPath).Where(n => n.Attributes.Contains("href")).ToList();
}
if (!string.IsNullOrWhiteSpace(Path.TitleXPath))
{
Path.TitleXPath = Path.UrlXPath;
}
singleNodeItemCount = nodecollection?.Count ?? 0;
if (nodecollection != null && nodecollection.Count() > 0 && nodecollection.Any(n => !string.IsNullOrEmpty(n.Attributes["href"].Value)))
{
Article[] articleNodeItems = new Article[singleNodeItemCount];
for (int i = 0; i < singleNodeItemCount; i++)
{
articleNodeItems[i] = new Article();
articleNodeItems[i].Title = ExtractInnerTextFromBaseNode(BaseNode, Path.TitleXPath, i);
if (articleNodeItems[i].Title != null)
{
try
{
articleNodeItems[i].Url = nodecollection.Where(n => !string.IsNullOrEmpty(n.Attributes["href"].Value)).ElementAt(i).Attributes["href"].Value;
if (articleNodeItems[i].Url.Contains(".pdf"))
{
continue;
}
if (articleNodeItems[i].Url.StartsWith("javascript:openArticle"))
{
articleNodeItems[i].Url = articleNodeItems[i].Url.Substring(articleNodeItems[i].Url.IndexOf("('") + 2);
articleNodeItems[i].Url = articleNodeItems[i].Url.Substring(0, articleNodeItems[i].Url.IndexOf("')"));
}
articleNodeItems[i].Url = HtmlUtility.AbsoluteUrl(articleNodeItems[i].Url, Url, true);
string url = HtmlUtility.AbsoluteUrl(articleNodeItems[i].Url, Url, true);
articleNodeItems[i].Url = url;
if (articleNodeItems[i].Url.Contains('@'))
{
continue;
}
}
catch (Exception ex)
{
articleNodeItems[i].Url = null;
}
}
if (!string.IsNullOrWhiteSpace(Path.MediaNameXPath))
{
articleNodeItems[i].MediaName = ExtractInnerTextFromBaseNode(BaseNode, Path.MediaNameXPath, i);
articleNodeItems[i].MediaName = ExtractSegmentFromInnerText(articleNodeItems[i].MediaName, MediaPrefixRegex);
articleNodeItems[i].MediaName = HTMLCleaner.CleanMediaName(articleNodeItems[i].MediaName);//清洗
}
if (!string.IsNullOrWhiteSpace(Path.AuthorXPath))
{
articleNodeItems[i].Author = ExtractInnerTextFromBaseNode(BaseNode, Path.AuthorXPath, i);
articleNodeItems[i].Author = ExtractSegmentFromInnerText(articleNodeItems[i].Author, AuthorPrefixRegex);
articleNodeItems[i].Author = HTMLCleaner.CleanAuthor(articleNodeItems[i].Author);//清洗
}
if (!string.IsNullOrWhiteSpace(Path.DateXPath))
{
articleNodeItems[i].PubDate = DateTimeParser.Parser(ExtractInnerTextFromBaseNode(BaseNode, Path.DateXPath, i));
}
if (!string.IsNullOrWhiteSpace(Path.AbsTractXPath))
{
articleNodeItems[i].AbsTract = ExtractInnerTextFromBaseNode(BaseNode, Path.AbsTractXPath, i);
}
//点击数的提取逻辑
string ViewString = string.Empty;
if (!string.IsNullOrWhiteSpace(Path.ViewXPath) || !string.IsNullOrWhiteSpace(Path.ReplyXPath))
{
ViewData currentViewData = new ViewData();
currentViewData.FetchTime = DateTime.Now;
ViewString = ExtractInnerTextFromBaseNode(BaseNode, Path.ViewXPath, i, false);
if (!string.IsNullOrEmpty(ViewString))
{
MatchCollection digiText = Regex.Matches(ViewString, @"\d{1,9}");
if (digiText.Count == 1)
{
currentViewData.View = int.Parse(digiText[0].Captures[0].Value);
}
else if (digiText.Count > 1 && Path.ViewXPath == Path.ReplyXPath) //View和Reply在一个格子里，这里容易出现多个的情况，不建议使用
{
int a = int.Parse(digiText[0].Captures[0].Value);
int b = int.Parse(digiText[1].Captures[0].Value);
currentViewData.View = a >= b ? a : b;
currentViewData.Reply = a >= b ? b : a;
}
}
//评论数的提取逻辑
if (!string.IsNullOrEmpty(Path.ReplyXPath) && Path.ViewXPath != Path.ReplyXPath)
{
string ReplyString = ExtractInnerTextFromBaseNode(BaseNode, Path.ReplyXPath, i, false);
if (!string.IsNullOrEmpty(ReplyString))
{
MatchCollection digiText = Regex.Matches(ReplyString, @"\d{1,9}");
if (digiText.Count > 0) //单独的Reply
{
currentViewData.Reply = int.Parse(digiText[0].Captures[0].Value);
}
}
}
if (articleNodeItems[i].ViewDataList == null)
{
articleNodeItems[i].ViewDataList = new List <ViewData>();
}
articleNodeItems[i].ViewDataList.Add(currentViewData);
}
}
Content.AddRange(articleNodeItems.Where(f => !string.IsNullOrWhiteSpace(f.Url)));
}
}
}
return(Content);
}