private void refreshPage()
{
IURL urlX = URLFactory.CreateURL(m_LastURLType);
Uri url = urlX.GetURL(m_Location);
webBrowser.Navigate(url);
}
private void IsInclude(object obj)
{
IURL iurl = obj as IURL;
int i = iurl.i;
string url = iurl.url;
StreamReader sr;
try
{
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create("http://www.baidu.com/s?wd=" + url);
HttpWebResponse webResponse = (HttpWebResponse)webRequest.GetResponse();
Stream srtStream = webResponse.GetResponseStream();
sr = new StreamReader(webResponse.GetResponseStream(), Encoding.GetEncoding("utf-8"));
String totalLines = sr.ReadToEnd();
//匹配标题
Regex reg = new Regex("没有找到");
Match mat = reg.Match(totalLines);
if (!mat.Success)
{
dt1.Rows[i][2] = "收录";
}
else
{
dt1.Rows[i][2] = "——";
}
}
catch
{
MessageBox.Show("对不起！网络没有连接。");
return;
}
finally
{
y++;
//修改图标控件--在线程中修改界面需要使用委托
//OutDelegateQ outdelegate = new OutDelegateQ(OutTextQ);
//this.Dispatcher.BeginInvoke(outdelegate, new object[] { "xx" });
//实时更新界面元素
Thread trd = new Thread(new ThreadStart(this.ThreadTask));
trd.IsBackground = true;
trd.Start();
}
Thread.Sleep(500);
}
private void IsRank(object obj)
{
IURL iurl = obj as IURL;
int i = iurl.i;
string key = iurl.url;
StreamReader sr;
try
{
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create("http://www.baidu.com/s?wd=" + key);
HttpWebResponse webResponse = (HttpWebResponse)webRequest.GetResponse();
Stream srtStream = webResponse.GetResponseStream();
sr = new StreamReader(webResponse.GetResponseStream(), Encoding.GetEncoding("utf-8"));
}
catch
{
MessageBox.Show("对不起！网络没有连接。");
return;
}
finally
{
}
String totalLines = sr.ReadToEnd();
bool pageinclude = false;
for (int n = 1; n <= 10; n++)
{
string str = " id=\"" + n.ToString() + "\"";
string html = Common.BDSegment(totalLines, n); //totalLines.Substring(totalLines.IndexOf(str), 1592);
string nohtmlstr = Common.StripHT(html);
if (nohtmlstr.Contains(domain(dt1.Rows[i][0].ToString())))
{
dt1.Rows[i][3] = n.ToString();
pageinclude = true;
break;
}
}
Thread.Sleep(Common.RandomInt(500, 700));
//-------------------------------------
//获得百度搜索结果页的2-10的url
if (!pageinclude && bdpage != 1)//如果首页没有收录和排名设置不为10时
{
string baiduurls = Common.MatchURL(totalLines, "<p id=\"page\"", "</p>");
MatchCollection mcbaiduurls = Common.MatchURLs(baiduurls, "<a href=\"", "\">");
List <string> BDurl = new List <string>();
j = 0;
foreach (Match ma in mcbaiduurls)
{
if (j < 9)
{
BDurl.Add("http://www.baidu.com" + ma.Value);
j++;
}
}
int m = 1;
for (int p = 0; p < BDurl.Count; p++)
{
if (m < bdpage)
{
if (GetRank(BDurl[p], m, domain(dt1.Rows[i][0].ToString()), i) == 1)
{
break;
}
m++;
}
}
}
//-------------------------------------
OutDelegateR outdelegate = new OutDelegateR(OutTextR);
this.Dispatcher.BeginInvoke(outdelegate, new object[] { "xx" });
}
private void Gather(object obj)
{
IURL iurl = obj as IURL;
int i = iurl.i;
string url = iurl.url;
try
{
string pageUrl = url;
WebClient wc = new WebClient();
byte[] pageSourceBytes = wc.DownloadData(new Uri(pageUrl));
//获取编码
string encode = "";
var r_utf8 = new System.IO.StreamReader(new System.IO.MemoryStream(pageSourceBytes), Encoding.UTF8); //将html放到utf8编码的StreamReader内
var r_gbk = new System.IO.StreamReader(new System.IO.MemoryStream(pageSourceBytes), Encoding.Default); //将html放到gbk编码的StreamReader内
var t_utf8 = r_utf8.ReadToEnd(); //读出html内容
var t_gbk = r_gbk.ReadToEnd(); //读出html内容
if (!isLuan(t_utf8)) //判断utf8是否有乱码
{
encode = "utf8";
}
else
{
encode = "gbk";
}
//根据编码读取
string pageSource = "";
if (encode == "utf8")
{
pageSource = Encoding.GetEncoding("utf-8").GetString(pageSourceBytes);
}
else
{
pageSource = Encoding.GetEncoding("gb2312").GetString(pageSourceBytes);
}
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(pageSource);
//获取标题
HtmlNodeCollection keyNodes = doc.DocumentNode.SelectNodes("/html[1]/head[1]/title[1]");
List <string> keyWords = new List <string>();
foreach (HtmlNode keyNode in keyNodes)
{
dt1.Rows[i][1] = replace(keyNode.InnerText.Trim()).Trim();
}
}
catch
{
}
finally
{
//修改图标控件--在线程中修改界面需要使用委托
OutDelegate outdelegate = new OutDelegate(OutText);
this.Dispatcher.BeginInvoke(outdelegate, new object[] { "xx" });
++y;
//实时更新界面元素
Thread trd = new Thread(new ThreadStart(this.ThreadTask));
trd.IsBackground = true;
trd.Start();
}
}
