public List<Link> GetLinks(string urlHead,int pageNeededNumber)
{
List<Link> links = new List<Link>();
string url ;
for (int i = 0; i < pageNeededNumber; i++)
{
url = urlHead + 50 * i;
WebPage page = new WebPage(url);
links.AddRange(page.getSpecialLinksByUrl("/p/", 50));
}
StreamWriter sw = new StreamWriter("C:\\Users\\LENOVO\\Desktop\\urls3.0.txt",false,Encoding.UTF8);
for (int i = 0; i < links.Count; i++)
{
sw.Write(links[i].NavigateUrl + "\r\n");
}
sw.Close();
return links;
}
private void bt_go_Click(object sender, EventArgs e)
{
startUrl = "http://tieba.baidu.com/f?kw=%E7%BD%91%E7%BB%9C%E6%B5%81%E8%A1%8C%E8%AF%AD&ie=utf-8&pn=";
List<Link> links =GetLinks(startUrl, 50);
if (File.Exists("C:\\Users\\LENOVO\\Desktop\\回复提取3.0.txt"))
File.Delete("C:\\Users\\LENOVO\\Desktop\\回复提取3.0.txt");
for (int i = 0; i < links.Count; i++)
{
string fstUrl = links[i].NavigateUrl;
WebPage page = new WebPage(fstUrl);
int pageNumber = GetPageNumber(page.Context);
if (pageNumber == 0)
continue;
for (int j = 0; j < pageNumber; j++)
{
string currentUrl = fstUrl + "?pn=" + (j + 1).ToString();
WebPage currentPage = new WebPage(currentUrl);
GetAndWrite(currentPage.Context,"回复提取3.0.txt",true);
}
}
}
