public void GetInfoFromHtml(int currentPage)
{
Lexer lexer = new Lexer(currentHtml);
Parser parser = new Parser(lexer);
NodeList poiHeadList = parser.Parse(poiListFilter);
if (poiHeadList.Count == 1)
{
NodeList poiNodeList = poiHeadList[0].Children.ExtractAllNodesThatMatch(poiFilter, false);
int numCount = 0;
for (int i = 0; i < poiNodeList.Count; i++)
{
POI poi = new POI();
DefinitionListBullet poiNode = (DefinitionListBullet)poiNodeList[i];
if (poiNode.TagName.Equals("DD"))
{
numCount++;
poi.Page = currentPage;
poi.Number = numCount;
#region 获取口味、环境和服务评分，以及获取星级
NodeList tasteNodeList = poiNode.Children.ExtractAllNodesThatMatch(tasteFilter, true);
NodeList environmentNodeList = poiNode.Children.ExtractAllNodesThatMatch(environmentFilter, true);
NodeList serviceNodeList = poiNode.Children.ExtractAllNodesThatMatch(serviceFilter, true);
if (tasteNodeList.Count == 1 && environmentNodeList.Count == 1 && serviceNodeList.Count == 1)
{
Span spanNode = (Span)tasteNodeList[0];
if (!spanNode.ToPlainTextString().Equals("-"))
{
poi.TasteRemark = Int32.Parse(spanNode.ToPlainTextString());
}
spanNode = (Span)environmentNodeList[0];
if (!spanNode.ToPlainTextString().Equals("-"))
{
poi.EnvironmentRemark = Int32.Parse(spanNode.ToPlainTextString());
}
spanNode = (Span)serviceNodeList[0];
if (!spanNode.ToPlainTextString().Equals("-"))
{
poi.ServiceRemark = Int32.Parse(spanNode.ToPlainTextString());
}
#region 获取星级
INode rankNodeOfParent = spanNode.Parent.NextSibling.NextSibling;
if (rankNodeOfParent.Children != null && rankNodeOfParent.Children.Count >= 1)
{
INode rankNodeCandidate = rankNodeOfParent.Children[0];
if (rankNodeCandidate.GetType().Equals(typeof(Span)))
{
Span rankNode = (Span)rankNodeCandidate;
string rank = rankNode.GetAttribute("TITLE");
if (rank.Contains("五"))
{
poi.Rank = 5;
}
else
{
if (rank.Contains("四"))
{
poi.Rank = 4;
}
else
{
if (rank.Contains("三"))
{
poi.Rank = 3;
}
else
{
if (rank.Contains("二"))
{
poi.Rank = 2;
}
else
{
if (rank.Contains("一"))
{
poi.Rank = 1;
}
}
}
}
}
}
}
#endregion
}
else
{
Console.WriteLine("第" + i + "条POI中，判断口味、环境和服务的标准出错！");
}
#endregion
#region 获取平均消费
NodeList averageNodeList = poiNode.Children.ExtractAllNodesThatMatch(averageFilter, true);
if (averageNodeList.Count == 1)
{
INode averageNode = averageNodeList[0];
if (averageNode.NextSibling.NextSibling.GetType().Equals(typeof(TextNode)))
{
string cost = ((TextNode)averageNode.NextSibling.NextSibling).ToPlainTextString();
poi.AverageCost = Int32.Parse(cost);
}
}
else
{
Console.WriteLine("第" + i + "条POI中，判断平均消费的标准出错！");
}
#endregion
#region 获取点评数
NodeList commentNodeList = poiNode.Children.ExtractAllNodesThatMatch(commentFilter, true);
if (commentNodeList.Count == 1)
{
INode commentNode = commentNodeList[0];
if (commentNode.GetType().Equals(typeof(ATag)))
{
string commentNum = ((ATag)commentNode).StringText;
if (commentNum.Substring(commentNum.Length - 3, 3).Equals("封点评"))
{
commentNum = commentNum.Substring(0, commentNum.Length - 3);
}
poi.CommentCount = Int32.Parse(commentNum);
}
}
else
{
Console.WriteLine("第" + i + "条POI中，判断点评数的标准出错！");
}
#endregion
#region 获取店名
NodeList nameNodeList = poiNode.Children.ExtractAllNodesThatMatch(nameFilter, true);
if (nameNodeList.Count == 1)
{
INode nameNode = nameNodeList[0];
if (nameNode.GetType().Equals(typeof(ATag)))
{
poi.Name = ((ATag)nameNode).StringText;
}
}
else
{
Console.WriteLine("第" + i + "条POI中，判断店名的标准出错！");
}
#endregion
#region 获取地址和电话
NodeList addressNodeList = poiNode.Children.ExtractAllNodesThatMatch(addressFilter, true);
if (addressNodeList.Count == 1)
{
NodeList districtNodeList = addressNodeList[0].Children.ExtractAllNodesThatMatch(new NodeClassFilter(typeof(ATag)));
if (districtNodeList.Count == 1)
{
ATag districtTag = (ATag)districtNodeList[0];
string address = districtTag.ToPlainTextString();
if (districtTag.NextSibling.GetType().Equals(typeof(TextNode)))
{
TextNode detailAddressNode = (TextNode)districtTag.NextSibling;
string detailAddress = detailAddressNode.ToPlainTextString();
detailAddress = detailAddress.Trim();
string phoneStr = detailAddress.Substring(detailAddress.Length - 8, 8);
poi.Phone = phoneStr;
address += detailAddress.Substring(0, detailAddress.Length - 8);
}
char[] removeChrVector = { ' ', '\n', '\t' };
address = address.Trim(removeChrVector);
foreach (char c in removeChrVector)
{
address = address.Replace(c.ToString(), "");
}
poi.Address = address;
}
else
{
Console.WriteLine("第" + i + "条POI中，判断含地址的<a>标记的标准出错！");
}
}
else
{
Console.WriteLine("第" + i + "条POI中，判断地址的标准出错！");
}
#endregion
#region 获取标签
NodeList tagsNodeList = poiNode.Children.ExtractAllNodesThatMatch(tagsFilter, true);
if (tagsNodeList.Count == 1)
{
INode tagsNode = tagsNodeList[0];
if (tagsNode.Children != null)
{
for (int j = 0; j < tagsNode.Children.Count; j++)
{
INode node = tagsNode.Children[j];
if (node.GetType().Equals(typeof(ATag)))
{
poi.Tags.Add(node.ToPlainTextString());
}
}
}
}
else
{
Console.WriteLine("第" + i + "条POI中，判断标签的标准出错！");
}
#endregion
poiList.Add(poi);
}
}
}
else
{
Console.WriteLine("获取POI列表出错");
}
}
/// <summary>
/// 从网页版微博中获取微博信息
/// </summary>
/// <param name="fansList">保存爬得的粉丝数组</param>
public void GetInfoFromHtml(List <Fan> fansList)
{
Lexer lexer = new Lexer(currentHtmlContent);
Parser parser = new Parser(lexer);
//获取包含每条微博的div标记列表
NodeList fansNodeList = parser.Parse(fanFilter);
for (int i = 0; i < fansNodeList.Size(); i++)
{
Fan fan = new Fan();
//获取包含一个粉丝的<li>标记
Bullet fanBullet = (Bullet)fansNodeList[i];
#region 获取该粉丝头像
NodeList fanPortraitNodeList = fanBullet.Children.ExtractAllNodesThatMatch(portraitFilter, true);
if (fanPortraitNodeList.Size() == 1)
{
Div fanPortraitDiv = (Div)fanPortraitNodeList[0];
NodeList imgNodeList = fanPortraitDiv.Children.ExtractAllNodesThatMatch(new NodeClassFilter(typeof(ImageTag)), true);
if (imgNodeList.Size() == 1)
{
ImageTag imgNode = (ImageTag)imgNodeList[0];
if (imgNode.Attributes.ContainsKey("SRC") && imgNode.Attributes.ContainsKey("ALT"))
{
string imgUrl = imgNode.GetAttribute("SRC");
string imgName = imgNode.GetAttribute("ALT");
fan.Name = imgName;
WebClient wc = new WebClient();//使用WebClient是因为下载用户头像不用登录cookie
wc.DownloadFileAsync(new Uri(imgUrl), @"portrait\" + imgName + ".jpg");
wc.DownloadFileCompleted += wc_DownloadFileCompleted;
}
else
{
Console.WriteLine("第" + i + "个粉丝中，<img>标记缺少必要的属性！");
}
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取img标记出错！");
}
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝头像的标准出错！");
}
#endregion
#region 获取该粉丝的关注数/粉丝数/微博数
NodeList fanConnectNodeList = fanBullet.Children.ExtractAllNodesThatMatch(fanConnectFilter, true);
if (fanConnectNodeList.Size() == 1)
{
NodeList ATagList = fanConnectNodeList[0].Children.ExtractAllNodesThatMatch(new NodeClassFilter(typeof(ATag)), true);
if (ATagList.Size() == 3)
{
for (int j = 0; j < 3; j++)
{
ATag aTag = (ATag)ATagList[j];
switch (j)
{
case 0:
if (aTag.Attributes.ContainsKey("HREF") && aTag.GetAttribute("HREF").Contains("follow"))
{
fan.FollowCount = Int32.Parse(aTag.StringText);
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝的关注数出错！");
}
break;
case 1:
if (aTag.Attributes.ContainsKey("HREF") && aTag.GetAttribute("HREF").Contains("fans"))
{
fan.FansCount = Int32.Parse(aTag.StringText);
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝的粉丝数出错！");
}
break;
default:
fan.FeedsCount = Int32.Parse(aTag.StringText);
break;
}
}
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝关注数/粉丝数/微博数的数量出错！");
}
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝关注数/粉丝数/微博数的标准出错！");
}
#endregion
#region 获取该粉丝的简介信息
NodeList fanInfoNodeList = fanBullet.Children.ExtractAllNodesThatMatch(fanInfoFilter, true);
if (fanInfoNodeList.Size() == 1)
{
//Console.WriteLine(fanInfoNodeList[0].Parent.ToHtml());
Div fanInfoDiv = (Div)fanInfoNodeList[0];
string intro = fanInfoDiv.StringText;
if (intro.Substring(0, 2).Equals("简介"))
{
fan.Introduction = intro.Substring(3, intro.Length - 3).Replace("\n", " ").Replace("\t", " ");
}
}
else
{
if (fanInfoNodeList.Size() == 0)
{
fan.Introduction = "";
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝简介的标准出错！");
}
}
#endregion
#region 获取该粉丝的UserID、地点和性别信息；校验该粉丝的用户名信息
NodeList fanLocationNodeList = fanBullet.Children.ExtractAllNodesThatMatch(fanNameFilter, true);
if (fanLocationNodeList.Size() == 1)
{
//获取粉丝的UserID信息；校验该粉丝的用户名信息
NodeList aTagNodeList = fanLocationNodeList[0].Children.ExtractAllNodesThatMatch(new NodeClassFilter(typeof(ATag)), true);
if (aTagNodeList.Size() >= 1)
{
ATag nameNode = (ATag)aTagNodeList[0];
if (nameNode.Attributes.ContainsKey("USERCARD") && nameNode.Attributes.ContainsKey("HREF"))
{
//获取粉丝的UserID信息
string uidStr = nameNode.GetAttribute("USERCARD");
if (uidStr.Substring(0, 3).Equals("id="))
{
fan.UserID = uidStr.Substring(3, uidStr.Length - 3);
}
//获取粉丝的微博链接
string linkUrl = nameNode.GetAttribute("HREF");
fan.LinkURL = "http://www.weibo.com" + linkUrl;
}
else
{
Console.WriteLine("第" + i + "个粉丝中，包含用户id和链接的<a>标记中缺少必要的属性！");
}
//校验该粉丝的用户名信息
if (!nameNode.StringText.Equals(fan.Name))
{
Console.WriteLine("第" + i + "个粉丝中，用户名与用户头像文字描述不一致！");
}
}
//获取粉丝的性别和地点信息
NodeList locationNodeList = fanLocationNodeList[0].Children.ExtractAllNodesThatMatch(new HasAttributeFilter("class", "addr"), true);
if (locationNodeList.Size() == 1)
{
string locationStr = "";
for (int j = 0; j < locationNodeList[0].Children.Size(); j++)
{
INode node = locationNodeList[0].Children[j];
if (node.GetType().Equals(typeof(TextNode)))
{
TextNode tNode = (TextNode)node;
locationStr += tNode.ToPlainTextString();
}
if (node.GetType().Equals(typeof(TagNode)))
{
TagNode tNode = (TagNode)node;
if (tNode.Attributes.ContainsKey("CLASS"))
{
if (tNode.GetAttribute("CLASS").Contains("female"))//必须先female，因为female中也含有male，如果male在前，则所有用户均符合该条件了= =
{
fan.Gender = "female";
}
else
{
if (tNode.GetAttribute("CLASS").Contains("male"))
{
fan.Gender = "male";
}
else
{
fan.Gender = "unknown";
Console.WriteLine("第" + i + "个粉丝性别不明！");
}
}
}
}
}
fan.Location = locationStr.Trim();
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝地点的标准出错！");
}
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取该粉丝的UserID、地点和性别信息的标准出错！");
}
#endregion
#region 获取该粉丝关注用户的方式
NodeList followMethodNodeList = fanBullet.Children.ExtractAllNodesThatMatch(followMethodFilter, true);
if (followMethodNodeList.Size() == 1)
{
NodeList methodNodeList = followMethodNodeList[0].Children.ExtractAllNodesThatMatch(new NodeClassFilter(typeof(ATag)));
if (methodNodeList.Size() == 1)
{
ATag methodNode = (ATag)methodNodeList[0];
fan.FollowMethod = methodNode.StringText.Trim();
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取该粉丝关注用户的方式的数量出错！");
}
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取该粉丝关注用户的方式的标准出错！");
}
#endregion
fansList.Add(fan);
}
}
