public List<Link> GetLinks(string urlHead,int pageNeededNumber) { List<Link> links = new List<Link>(); string url ; for (int i = 0; i < pageNeededNumber; i++) { url = urlHead + 50 * i; WebPage page = new WebPage(url); links.AddRange(page.getSpecialLinksByUrl("/p/", 50)); } StreamWriter sw = new StreamWriter("C:\\Users\\LENOVO\\Desktop\\urls3.0.txt",false,Encoding.UTF8); for (int i = 0; i < links.Count; i++) { sw.Write(links[i].NavigateUrl + "\r\n"); } sw.Close(); return links; }
private void bt_go_Click(object sender, EventArgs e) { startUrl = "http://tieba.baidu.com/f?kw=%E7%BD%91%E7%BB%9C%E6%B5%81%E8%A1%8C%E8%AF%AD&ie=utf-8&pn="; List<Link> links =GetLinks(startUrl, 50); if (File.Exists("C:\\Users\\LENOVO\\Desktop\\回复提取3.0.txt")) File.Delete("C:\\Users\\LENOVO\\Desktop\\回复提取3.0.txt"); for (int i = 0; i < links.Count; i++) { string fstUrl = links[i].NavigateUrl; WebPage page = new WebPage(fstUrl); int pageNumber = GetPageNumber(page.Context); if (pageNumber == 0) continue; for (int j = 0; j < pageNumber; j++) { string currentUrl = fstUrl + "?pn=" + (j + 1).ToString(); WebPage currentPage = new WebPage(currentUrl); GetAndWrite(currentPage.Context,"回复提取3.0.txt",true); } } }