private void MediaWebSpider()
{
#region 处理关键字
MySqlCmd cmd = new MySqlCmd();
//得到关键字列表
DataTable dtkey;
DataTable dtParts;
if (selectKID == -1)
{
//全部
dtkey = cmd.GetTabel("select * from Keywords");
}
else
{
dtkey = cmd.GetTabel("select * from Keywords where kid=" + selectKID);
}
dtParts = cmd.GetTabel("SELECT * FROM partword");
#endregion
HtmlParse.Parse parse = new HtmlParse.Parse();
//parse.ReportCatchProcess += new HtmlParse.ReportCatchProcessEventHandler(parse_ReportCatchProcess);
List<ModelReleaseInfo> webDatas = new List<ModelReleaseInfo>();
//按关键字循环
for (int kw = 0; kw < dtkey.Rows.Count; kw++)
{
//处理关键字
if (selectKID != -1 && selectKwName != "全部")
{
if (dtkey.Rows[kw]["name"].ToString().Trim() != selectKwName) continue;
}
string keyword = dtkey.Rows[kw]["KeyWord"].ToString().Trim();
//string keyTitle = dtkey.Rows[kw]["Name"].ToString().Trim();
int kid = 0;
int.TryParse(dtkey.Rows[kw]["kid"].ToString().Trim(), out kid);
#region 百度检索
//组成查询字串
string url = "http://news.baidu.com/ns?rn=100&word=" + keyword;
string html = HtmlUtil.HttpGet(url, Encoding.UTF8);
List<ModelReleaseInfo> mris = parse.ParseBaiduNews(html, keyword, kid);
if (mris != null && mris.Count() > 0)
{
//写入数据库
DataPersistenceControl.GetInstance().Add(mris);
}
#endregion
#region bing检索
for (int i = 0; i < 10; i++)
{
//组成查询字串
url = "";
html = "";
mris = null;
url = string.Format("http://cn.bing.com/news/search?q={0}&first={1}&FORM=PENR", keyword, i * 10 + 1);
html = HtmlUtil.HttpGet(url, Encoding.UTF8);
mris = parse.ParseBingNews(html, keyword, kid);
if (mris != null && mris.Count() > 0)
{
//写入数据库
DataPersistenceControl.GetInstance().Add(mris);
}
else
{
break;
}
//防止拉黑
Thread.Sleep(Interval50s);
}
#endregion
#region 搜狗新闻
for (int i = 0; i < 10; i++)
{
//组成查询字串
url = "";
html = "";
mris = null;
url = string.Format("http://news.sogou.com/news?query={0}&page={1}", keyword, i + 1);
html = HtmlUtil.HttpGet(url, Encoding.Default);
mris = parse.ParseSogouNews(html, keyword, kid);
if (mris != null && mris.Count() > 0)
{
//写入数据库
DataPersistenceControl.GetInstance().Add(mris);
}
else
{
break;
}
//防止拉黑（2分钟）
Thread.Sleep(Interval2m);
}
#endregion
#region 中搜新闻
for (int i = 0; i < 10; i++)
{
//组成查询字串
url = "";
html = "";
mris = null;
url = string.Format("http://zixun.zhongsou.com/n?w={0}&b={1}", keyword, i + 1);
html = HtmlUtil.HttpGet(url, Encoding.Default);
mris = parse.ParseZhongsouNews(html, keyword, kid);
if (mris != null && mris.Count() > 0)
{
//写入数据库
DataPersistenceControl.GetInstance().Add(mris);
}
else
{
break;
}
//防止拉黑（2分钟）
Thread.Sleep(Interval2m);
}
#endregion
#region 好搜新闻
for (int i = 0; i < 10; i++)
{
//组成查询字串
url = "";
html = "";
mris = null;
url = string.Format("http://news.haosou.com/ns?q={0}&pn={1}&tn=news&rank=rank&j=0", keyword, i + 1);
html = HtmlUtil.HttpGet(url, Encoding.UTF8);
mris = parse.ParseHaosouNews(html, keyword, kid);
if (mris != null && mris.Count() > 0)
{
//写入数据库
DataPersistenceControl.GetInstance().Add(mris);
}
else
{
break;
}
//防止拉黑
Thread.Sleep(Interval50s);
}
#endregion
}
}
