private void BlogWebSpider()
{
#region 处理关键字
MySqlCmd cmd = new MySqlCmd();
//得到关键字列表
DataTable dtkey;
DataTable dtParts;
if (selectKID == -1)
{
//全部
dtkey = cmd.GetTabel("select * from Keywords");
}
else
{
dtkey = cmd.GetTabel("select * from Keywords where kid=" + selectKID);
}
dtParts = cmd.GetTabel("SELECT * FROM partword");
#endregion
HtmlParse.Parse parse = new HtmlParse.Parse();
//parse.ReportCatchProcess += new HtmlParse.ReportCatchProcessEventHandler(Blog_ReportCatchProcess);
List<ModelReleaseInfo> webDatas = new List<ModelReleaseInfo>();
//按关键字循环
for (int kw = 0; kw < dtkey.Rows.Count; kw++)
{
//处理关键字
if (selectKID != -1 && selectKwName != "全部")
{
if (dtkey.Rows[kw]["name"].ToString().Trim() != selectKwName) continue;
}
string keyword = dtkey.Rows[kw]["KeyWord"].ToString().Trim();
string keyTitle = dtkey.Rows[kw]["Name"].ToString().Trim();
int kid = 0;
int.TryParse(dtkey.Rows[kw]["kid"].ToString().Trim(), out kid);
#region 按关键字检索
//组成查询字串
string url = "http://www.sogou.com/web?interation=196647&query=" + keyword + "&ie=utf8";
string html = HtmlUtil.HttpGet(url, Encoding.UTF8);
#endregion
List<ModelReleaseInfo> mris = parse.ParseSogouBlog(html, keyword, kid);
if (mris != null && mris.Count() > 0)
{
DataPersistenceControl.GetInstance().Add(mris);
}
//防止拉黑
Thread.Sleep(Interval2m);
}
}
