private static void EnsurePermissionAllowed(CQ body, MillRequest request)
{
if (body.Text() == "无权查看本主题")
{
throw new PermissionDeniedException(request);
}
}
public MillResult <ForumThread> ProcessThreadPage(MillRequest request)
{
if (request == null || string.IsNullOrWhiteSpace(request.HtmlContent))
{
throw new ArgumentNullException("request");
}
var root = CQ.CreateDocument(request.HtmlContent);
EnsureSignedIn(root, request);
var body = root.Select("div.wrap:first > div:eq(1)").FirstElement();
if (body == null || body.HasAttributes)
{
throw new ProcessFaultException(request, "无法定位包含内容的div元素");
}
var bodyCq = body.Cq();
EnsurePermissionAllowed(bodyCq, request);
var currentPageIndex = GetThreadPageCurrentIndex(request, root);
var isFirstPage = currentPageIndex == 1;
var thread = new ForumThread
{
Url = request.Url,
Id = request.Url.GetThreadId(),
CurrentPageIndex = currentPageIndex
};
if (isFirstPage)
{
ProcessFirstPost(request, bodyCq, thread, root);
}
var infobarNodes = bodyCq.Find("div.infobar");
var messageNodes = isFirstPage ? bodyCq.Find("div.message:gt(0)") : bodyCq.Find("div.message");
if (infobarNodes.Length != messageNodes.Length)
{
throw new ProcessFaultException(request, "infobar和message元素个数不一致。");
}
ProcessReplies(request, infobarNodes, messageNodes, thread);
var nextPageUrl = isFirstPage ? null : GetNextThreadPageUrl(request, root);
return(new MillResult <ForumThread>
{
Url = request.Url,
HumanReadableDescription = request.HumanReadableDescription,
NextPageUrl = nextPageUrl,
Result = thread
});
}
private static string GetNextThreadPageUrl(MillRequest request, CQ root)
{
var currentPageNode = root.Select("span.paging > span.s1").FirstElement();
if (currentPageNode == null)
{
throw new ProcessFaultException(request, "找不到分页元素。");
}
var previous = currentPageNode.Cq().Prev("a").FirstElement();
return(previous == null ? null : previous["href"]);
}
private static string GetNextForumPageUrl(CQ root, MillRequest request)
{
var currentPageIndexNode = root.Select("span.paging > span.s1").FirstOrDefault();
if (currentPageIndexNode == null)
{
throw new ProcessFaultException(request, "找不到分页元素。");
}
var nextPageNode = currentPageIndexNode.Cq().Next("a").FirstOrDefault();
return(nextPageNode == null ? null : nextPageNode["href"]);
}
private static int GetThreadPageCurrentIndex(MillRequest request, CQ root)
{
if (!request.Url.MatchPageIndex())
{
return(1);
}
var pagingNode = root.Select("span.paging:first").FirstOrDefault();
if (pagingNode == null)
{
return(1);
}
var currentPageIndexNode = pagingNode.Cq().Find("span.s1").Single();
return(int.Parse(currentPageIndexNode.InnerText.Replace("##", "")));
}
private static void ProcessReplies(MillRequest request, CQ infobarNodes, CQ messageNodes, ForumThread thread)
{
if (infobarNodes.Length <= 0)
{
return;
}
var replies = infobarNodes.Zip(messageNodes, (infobarNode, messageNode) => new { infobarNode, messageNode }).Select((x, index) =>
{
var post = new Post
{
ThreadId = thread.Id,
HtmlContent = HttpUtility.HtmlDecode(x.messageNode.InnerHTML)
};
var orderAnchor = x.infobarNode.Cq().Find("b a").FirstElement();
if (orderAnchor == null)
{
throw new ProcessFaultException(request, string.Format("第{0}个回复无法定位楼层锚元素。", index));
}
post.Id = orderAnchor["href"].GetPostId();
post.Order = int.Parse(orderAnchor.InnerText.Replace("#", ""));
var nextTextNode = orderAnchor.ParentNode.NextSibling;
if (nextTextNode == null || nextTextNode.NodeType != NodeType.TEXT_NODE)
{
throw new ProcessFaultException(request, string.Format("第{0}个回复无法定位作者元素。", index));
}
var text = nextTextNode.Cq().Text();
post.UserName = text.Trim().EndsWith("匿名")
? "匿名"
: orderAnchor.ParentNode.NextElementSibling.Cq().Text();
post.CreateDate = DateTime.Parse(x.infobarNode.Cq().Find("span.nf:first").Text());
post.ModifyDate = GetModifyDate(x.messageNode.Cq(), post.UserName);
var ratings = x.messageNode.GetRatings();
post.PositiveRate = ratings.Item1;
post.NegativeRate = ratings.Item2;
return(post);
});
thread.Posts.AddRange(replies);
}
public MillResult <List <ThreadHeader> > ProcessForumPage(MillRequest request)
{
if (request == null || string.IsNullOrWhiteSpace(request.HtmlContent))
{
throw new ArgumentNullException("request");
}
var root = CQ.CreateDocument(request.HtmlContent);
EnsureSignedIn(root, request);
var titles = root.Select("span.title");
var authors = root.Select("span.author");
if (titles.Length != authors.Length)
{
throw new ProcessFaultException(request, "forum page title和author元素个数不一样。");
}
if (titles.Length == 0)
{
throw new ProcessFaultException(request, "forum page 没有找到title和author元素。");
}
var headers = new List <ThreadHeader>(titles.Length);
for (var i = 0; i < titles.Length; i++)
{
var titleNode = titles[i];
var authorNode = authors[i];
headers.Add(GetThreadHeader(titleNode, authorNode, request, i));
}
return(new MillResult <List <ThreadHeader> >
{
Url = request.Url,
NextPageUrl = GetNextForumPageUrl(root, request),
Result = headers
});
}
private static void EnsureSignedIn(CQ root, MillRequest request)
{
var signedIn = true;
var footer = root.Select("div#footer").FirstOrDefault();
if (footer == null)
{
signedIn = false;
}
else
{
var anchorsInFooter = footer.Cq().Find("a");
if (anchorsInFooter.Any(link => link.Cq().Text() == "注册") ||
anchorsInFooter.Any(link => link.Cq().Text() == "登陆"))
{
signedIn = false;
}
}
if (!signedIn)
{
throw new NotSignedInException(request);
}
}
private static ThreadHeader GetThreadHeader(IDomObject titleNode, IDomObject authorNode, MillRequest request, int i)
{
var titleAnchor = titleNode.Cq().Find("a:first").FirstOrDefault();
if (titleAnchor == null)
{
throw new ProcessFaultException(request, string.Format("第{0}个title元素里面没有a元素。", i));
}
var url = titleAnchor["href"];
var titleText = titleAnchor.Cq().Text().Trim();
var threadId = url.GetThreadId();
if (0 == threadId)
{
throw new ProcessFaultException(request, string.Format("无法从第{0}个thread url中获取thread id。", i));
}
var header = new ThreadHeader
{
Id = threadId,
Url = url,
Title = titleText,
ReplyCount = -1
};
var authorText = authorNode.Cq().Text();
if (string.IsNullOrWhiteSpace(authorText))
{
return(header);
}
var values = authorText.Replace("[", "").Replace("]", "").Split('/');
if (values.Length != 4)
{
return(header);
}
header.UserName = values[0];
int replyCount;
if (int.TryParse(values[1], out replyCount))
{
header.ReplyCount = replyCount;
}
return(header);
}
private static void ProcessFirstPost(MillRequest request, CQ bodyCq, ForumThread thread, CQ root)
{
var titleNode = bodyCq.Find("b:first").FirstOrDefault();
if (titleNode == null)
{
throw new ProcessFaultException(request, "无法定位主题的title元素");
}
thread.Title = titleNode.Cq().Text();
var dateNode = titleNode.NextSibling;
if (dateNode == null)
{
throw new ProcessFaultException(request, "无法定位发帖日期元素");
}
dateNode = dateNode.NextSibling;
if (dateNode == null || dateNode.NodeType != NodeType.TEXT_NODE)
{
throw new ProcessFaultException(request, "无法定位发帖日期元素");
}
var createDate = DateTime.Parse(dateNode.NodeValue.Replace("时间:", "").Trim());
var authorLiteralNode = dateNode.NextSibling;
if (authorLiteralNode == null)
{
throw new ProcessFaultException(request, "无法定位主题作者元素");
}
authorLiteralNode = authorLiteralNode.NextSibling;
if (authorLiteralNode == null || authorLiteralNode.NodeType != NodeType.TEXT_NODE)
{
throw new ProcessFaultException(request, "无法定位主题作者元素");
}
var author = authorLiteralNode.NodeValue;
if (author == "作者:匿名")
{
thread.UserName = "******";
}
else if (author == "作者:")
{
var userNameAnchor = authorLiteralNode.NextElementSibling;
if (userNameAnchor == null || !"A".Equals(userNameAnchor.NodeName.ToUpper(), StringComparison.OrdinalIgnoreCase))
{
throw new ProcessFaultException(request, "无法定位主题作者元素");
}
thread.UserName = userNameAnchor.Cq().Text().Trim();
}
var messageNode = bodyCq.Find("div.message:first").FirstElement();
if (messageNode == null)
{
throw new ProcessFaultException(request, "无法定位主题内容元素");
}
var firstPostHtmlContent = HttpUtility.HtmlDecode(messageNode.InnerHTML);
var modifyDate = GetModifyDate(messageNode.Cq(), thread.UserName);
var pidAnchor = messageNode.Cq().NextAll("a").FirstElement();
if (pidAnchor == null || pidAnchor["href"] == null)
{
pidAnchor = root.Select("a:contains('引用')").FirstElement();
}
if (pidAnchor == null)
{
throw new ProcessFaultException(request, "无法定位主题中包含pid的锚元素，无法获取pid");
}
var pid = pidAnchor["href"].GetPostId();
var ratings = messageNode.GetRatings();
var post = new Post
{
Id = pid,
ThreadId = thread.Id,
UserName = thread.UserName,
Title = thread.Title,
Order = 1,
HtmlContent = firstPostHtmlContent,
CreateDate = createDate,
ModifyDate = modifyDate,
PositiveRate = ratings.Item1,
NegativeRate = ratings.Item2,
};
thread.Posts.Add(post);
}
public PermissionDeniedException(MillRequest request)
: base(request, "", null)
{
}
public ProcessFaultException(MillRequest request, string message, Exception e = null)
: base(request, message, e)
{
}
public NotSignedInException(MillRequest request)
: base(request, "", null)
{
}
protected PageMillException(MillRequest request, string message, Exception e)
: base(message, e)
{
Request = request;
}
public static MillStatus TryProcessPage <T>(this IPageProcessor pageProcessor, MillRequest request, out MillResult <T> result)
where T : class
{
result = new MillResult <T> {
Url = request.Url, HumanReadableDescription = request.HumanReadableDescription
};
try
{
if (typeof(T) == typeof(List <ThreadHeader>))
{
result = pageProcessor.ProcessForumPage(request) as MillResult <T>;
}
else if (typeof(T) == typeof(ForumThread))
{
result = pageProcessor.ProcessThreadPage(request) as MillResult <T>;
}
else
{
throw new NotSupportedException();
}
return(MillStatus.Success);
}
catch (NotSignedInException ne)
{
Logger.Info("解析网页时发现未登录，URL: {0}\r\n 内容:{1}", ne.Request.Url, ne.Request.HtmlContent);
return(MillStatus.NotSignedIn);
}
catch (PermissionDeniedException pde)
{
Logger.Info("解析网页时发现没有权限，URL: {0}\r\n 内容:{1}", pde.Request.Url, pde.Request.HtmlContent);
return(MillStatus.PermissionDenied);
}
catch (ProcessFaultException pfe)
{
Logger.Info("解析网页时发生错误，错误信息：{0}\r\n URL: {1}\r\n 内容:{2}\r\n 内部异常:{3}\r\n", pfe.Message, pfe.Request.Url,
pfe.Request.HtmlContent, pfe.InnerException);
return(MillStatus.FormatError);
}
catch (Exception e)
{
Logger.Info("Error URL:{0}\r\n{1}\r\n{2}", request.Url, e, request.HtmlContent);
return(MillStatus.FormatError);
}
}
