private void UpdataPartWord(string dataName, string dataValue, string instr) { string sql = "update partword set " + dataName + " = '" + dataValue + "' where uid in (" + instr + ")"; cmd.ExecuteNonQuery(sql); GetPartWordData(); }
private bool AddKeyWord(string kw, int meal, int mual, int has) { if (checkHasData("keyword", kw)) { MessageBox.Show("关键词 :" + kw + " 已存在!", "提示:", MessageBoxButtons.OK, MessageBoxIcon.Information); return(false); } string sql = "insert into keywords (keyword,messagealarm,musicalarm,has) values ('" + kw + "'," + meal + "," + mual + "," + has + ")"; cmd.ExecuteNonQuery(sql); return(true); }
private bool AddKeyWord(string url, string uname, string likeurl, int upid) { if (checkHasData("url", url)) { MessageBox.Show("网址 :" + url + " 已存在!", "提示", MessageBoxButtons.OK, MessageBoxIcon.Information); return(false); } string sql = "insert into webaddress (url,name,likeurl,pid) values ('" + url + "','" + uname + "','" + likeurl + "'," + upid + ")"; cmd.ExecuteNonQuery(sql); return(true); }
private void GetBaiduInfo() { lbAll.Text = ""; lbAll.Visible = true; MySqlCmd cmd = new MySqlCmd(); ; //得到关键字列表 DataTable dtkey = new DataTable(); dtkey = cmd.GetTabel("select * from Keywords"); dtParts = cmd.GetTabel("SELECT * FROM partword"); //链接的正则 string aa = "http://.[^\"]+"; string[] sDate; sb = new StringBuilder(); sb.Append(""); //TbReleaseInfo ri = new TbReleaseInfo(); //按关键字循环 for (int kw = 0; kw < dtkey.Rows.Count; kw++) { lbAll.Text = "正在搜索关键字为<" + dtkey.Rows[kw]["KeyWord"].ToString().Trim() + ">的数据."; lbAll.ForeColor = Color.DarkBlue; //取得关键字 string keys = dtkey.Rows[kw]["KeyWord"].ToString().Trim(); //组成查询字串 string url = "http://www.baidu.com/s?wd=\"" + keys + "\"&rn=50"; //得到结果放在数组内 List<string> lis = new List<string>(); lis = HtmlUtil.GetElementsByClassList(getHtml(url, "utf-8"), "result"); //如果没取到,就结束本次循环 if (lis == null) return; //webBrowser1.Navigate(url); //循环时判断是否要验证 bool isThere = false; for (int i = 0; i < lis.Count; i++) { if (Program.ProClose == true) break; ModelReleaseInfo mri = new ModelReleaseInfo(); //发布日期的赋值 sDate = HtmlUtil.GetElementsByTagAndClass(lis[i], "span", "g"); if (sDate.Length <= 0) continue; mri.ReleaseDate = HtmlUtil.NoHTML(sDate[0]); mri.ReleaseDate = mri.ReleaseDate.Substring(mri.ReleaseDate.Length - 10, 10); //判断日期 DateTime ddt; if (DateTime.TryParse(mri.ReleaseDate, out ddt)) { } else { //百度的快照日期有时会是9位或8位,如果是这种情况,那么按规则去掉 mri.ReleaseDate = mri.ReleaseDate.Substring(1, 9); if (DateTime.TryParse(mri.ReleaseDate, out ddt)) { } else { mri.ReleaseDate = mri.ReleaseDate.Substring(1, 8); } } //处理日期 try { mri.ReleaseDate = DateTime.Parse(mri.ReleaseDate).ToString("yyyy-MM-dd HH:mm:ss"); } catch (Exception ex) { StreamWriter sw = File.AppendText("log.txt"); sw.WriteLine(DateTime.Now.ToLongDateString()); sw.WriteLine("begin"); sw.WriteLine(ex.Message); sw.WriteLine(sb.ToString()); sw.WriteLine("end"); sw.WriteLine(""); sw.Close(); } //只拿取三天的内的数据 try { if (DateTime.Parse(mri.ReleaseDate) < DateTime.Now.AddDays(-3)) continue; } catch (Exception ex) { continue; } try { //得到标题 mri.Title = HtmlUtil.NoHTML(HtmlUtil.GetElementsByTagName(lis[i], "h3")[0]); string[] temp = HtmlUtil.GetElementsByClass(lis[i], "c-abstract"); //如果未取到内容部分,就跳出 if (temp.Length == 0) continue; mri.Contexts = HtmlUtil.NoHTML(temp[0]); mri.InfoSource = HtmlUtil.GetListByHtml("", HtmlUtil.GetElementsByTagName(lis[i], "a")[0], aa)[0]; //去掉重复 if (isThere) { continue; } else { if (UrlThereare(mri.Title, this.dtqueryinfo, dtWebQueryInfo, false) != 0) { isThere = true; continue; } } mri.KeyWords = dtkey.Rows[kw]["KeyWord"].ToString().Trim(); mri.CollectDate = string.Format(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); mri.Snapshot = ""; mri.ReleaseName = ""; mri.WebName = "百度"; mri.Pid = 0; mri.Part = GetParts(mri.Contexts); mri.Comments = 0; mri.Reposts = 0; DataRow dr = dtqueryinfo.NewRow(); if (dvAll.RowCount == 0) { dr[0] = 1; } else { dr[0] = int.Parse(dvAll.Rows[dvAll.RowCount - 1].Cells[0].Value.ToString()) + 1; } dr[1] = mri.Title; dr[2] = mri.Contexts; dr[3] = mri.ReleaseDate; dr[4] = mri.InfoSource; dr[5] = mri.KeyWords; dr[6] = mri.ReleaseName; dr[7] = mri.CollectDate; dr[8] = mri.Snapshot; dr[9] = mri.WebName; dr[10] = mri.Pid; dr[11] = mri.Part; dr[12] = mri.Reposts; dr[13] = mri.Comments; dtqueryinfo.Rows.InsertAt(dr, 0); if (dtqueryinfo.Rows.Count >= 500) { dtqueryinfo.Rows.RemoveAt(500); } dvAll.Refresh(); } catch (Exception ex) { StreamWriter sw = File.AppendText("log.txt"); sw.WriteLine(DateTime.Now.ToLongDateString()); sw.WriteLine("begin"); sw.WriteLine(ex.Message); sw.WriteLine(sb.ToString()); sw.WriteLine("end"); sw.WriteLine(""); sw.Close(); } try { //得到插入语句 if (isThere) { continue; } else { sb.Append(tri.GetInsString(mri) + ";"); } //每10次执行一次插入数据库 if (sb.ToString().Length != 0) { if (i % 10 == 0) { //执行插入 cmd.ExecuteNonQuery(sb.ToString()); //清除插入字段串 sb.Clear(); } } } catch (Exception ex) { StreamWriter sw = File.AppendText("log.txt"); sw.WriteLine(DateTime.Now.ToLongDateString()); sw.WriteLine("begin"); sw.WriteLine(ex.Message); sw.WriteLine(sb.ToString()); sw.WriteLine("end"); sw.WriteLine(""); sw.Close(); } } } try { if (sb.ToString().Length != 0) { //执行插入 cmd.ExecuteNonQuery(sb.ToString()); //清除插入字段串 sb.Clear(); } } catch (Exception ex) { StreamWriter sw = File.AppendText("log.txt"); sw.WriteLine(DateTime.Now.ToLongDateString()); sw.WriteLine("begin"); sw.WriteLine(ex.Message); sw.WriteLine(sb.ToString()); sw.WriteLine("end"); sw.WriteLine(""); sw.Close(); } //执行完毕后,重新获取一次数据库的数据 dtWebQueryInfo = tri.SelReleaseInfo(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), DateTime.Now.AddDays(-15).ToString("yyyy-MM-dd HH:mm:ss"), "0 AND webName='百度'"); //MessageBox.Show("ok"); lbAll.Text = "一轮搜索完毕!"; lbAll.ForeColor = Color.Red; }
/// <summary> /// 得到网站的新闻类数据 /// </summary> private void GetWebNewsInfo() { lbWeb.Text = ""; lbWeb.Visible = true; //相似链接 string Similar = ""; DataBaseServer.MySqlCmd cmd = new MySqlCmd(); //得到相似表 DataTable dtXs = new DataTable(); dtXs = cmd.GetTabel("Select * from WebAddress WHERE pid=0"); dtParts = cmd.GetTabel("SELECT * FROM partword"); DataTable dtkey = new DataTable(); dtkey = cmd.GetTabel("select * from Keywords"); //相似表中的被抓取网址 string webInfo = ""; //要过滤链接中首页的正则 string strTopFormat = "http://.+/"; List<string> strTop = new List<string>(); sb = new StringBuilder(); sb.Append(""); string filterStr = ""; #region 读取相似度表中的数据据,循环抓取 for (int xs = 0; xs < dtXs.Rows.Count; xs++) { lbWeb.ForeColor = Color.DarkBlue; lbWeb.Text = "正在搜索:" + dtXs.Rows[xs]["name"].ToString(); //读取相似表中要抓取的网址 webInfo = getHtml(dtXs.Rows[xs]["url"].ToString(), ""); //读取相似链接 Similar = dtXs.Rows[xs]["likeurl"].ToString(); //取出 //string[] strA = HtmlUtil.GetElementsByTagName(webInfo, "a"); List<string> strList = HtmlUtil.GetElementsByTagNameList(webInfo, "a"); string strURLformat = "http://.[^\"]+"; TbReleaseInfo ri = new TbReleaseInfo(); string[] strA = GetLIstDate(strList.Distinct()); #region 逐个链接判断 //循环时判断是否要验证 bool isThere = false; for (int i = 0; i < strA.Length; i++) { if (Program.ProClose == true) break; Application.DoEvents(); try { //得到目标网址中的所有链接,如果未得到,那么就继续读取下一个 strA[i] = HtmlUtil.GetListByHtml(dtXs.Rows[xs]["url"].ToString(), strA[i], strURLformat)[0]; //处理含有单引号的链接 strA[i] = UrlCl(strA[i]); //处理单引号的链接 if (strA[i].IndexOf("'") != -1) { strA[i] = GetstringByHtmlArray(strA[i], "http://.[^\']+"); } } catch (Exception) { continue; } //得到相似值,大于0.70的认为相同,并开始抓取 if (HtmlUtil.getSimilarDegree(Similar, strA[i]) >= 0.60) { //判断这个链接是否已经在库中或者列表中,如果存在,此次就不再执行 strTop = HtmlUtil.GetListByHtmlArray(strA[i], strTopFormat); if (strTop.Count != 0) { //if (strTop[0] == "http://blog.sohu.com/") continue;//同新闻,如果将首页去掉 } if (isThere) { continue; } else { //if (strA[i] == "http://news.ifeng.com/mainland/detail_2013_10/18/30459577_0.shtml'>[详细]</a>") //{ // strA[i] = strA[i]; //} if (UrlThereare(strA[i], this.dtnewsinfo, dtWebNewsInfo, true) != 0) { isThere = true; continue; } } //得到此链接的源码 webInfo = getHtml(strA[i], ""); if (webInfo.Length == 0) { continue; } //创建数据对象 ModelReleaseInfo newsInfo = new ModelReleaseInfo(); try { //流水+1 newsInfo.Uid = this.dvAll.Rows.Count + 1; //标题 string[] strT = HtmlUtil.GetElementsByTagName(webInfo, "title"); if (strT.Length == 0) { continue; } else { newsInfo.Title = HtmlUtil.NoHTML(HtmlUtil.GetElementsByTagName(webInfo, "title")[0]); } //得到正文,以P标签来区分 string[] strContext = HtmlUtil.GetElementsByTagName(webInfo, "p"); newsInfo.Contexts = ""; for (int j = 0; j < strContext.Length; j++) { //循环累加正文信息 newsInfo.Contexts += HtmlUtil.NoHTML(strContext[j]); } //如果正文信息为空,那么将无法做关键字对照,此条数据舍弃 if (newsInfo.Contexts.Length == 0) { continue; } //网站链接 newsInfo.InfoSource = strA[i].Trim(); //关键字的设置 newsInfo.KeyWords = ""; for (int j = 0; j < dtkey.Rows.Count; j++) { Application.DoEvents(); if (newsInfo.Contexts.IndexOf(dtkey.Rows[j][1].ToString()) > 0) { newsInfo.KeyWords += dtkey.Rows[j][1].ToString() + ","; } else { } } if (newsInfo.KeyWords.Length == 0) { continue; } newsInfo.KeyWords = newsInfo.KeyWords.Substring(0, newsInfo.KeyWords.Length - 1); //收集日期 newsInfo.CollectDate = string.Format(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); //发布人和发布日期暂时无法取到,手工赋值为空 newsInfo.ReleaseDate = ""; newsInfo.ReleaseName = ""; //网页快照,这里为用户指定生成,如果未选择生成,那么为空 newsInfo.Snapshot = ""; //网站名 newsInfo.WebName = dtXs.Rows[xs]["Name"].ToString(); //pid newsInfo.Pid = 0; //part正负判断 newsInfo.Part = GetParts(newsInfo.Contexts); //reposts newsInfo.Reposts = 0; //comments newsInfo.Comments = 0; //新建数据行 DataRow dr = dtnewsinfo.NewRow(); if (dvWeb.RowCount == 0) { dr[0] = 1; } else { dr[0] = int.Parse(dvWeb.Rows[dvWeb.RowCount - 1].Cells[0].Value.ToString()) + 1; } //dr[0] = newsInfo.Uid; dr[1] = newsInfo.Title; dr[2] = newsInfo.Contexts; dr[3] = newsInfo.ReleaseDate; dr[4] = newsInfo.InfoSource; dr[5] = newsInfo.KeyWords; dr[6] = newsInfo.ReleaseName; dr[7] = newsInfo.CollectDate; dr[8] = newsInfo.Snapshot; dr[9] = newsInfo.WebName; dr[10] = newsInfo.Pid; dr[11] = newsInfo.Part; dr[12] = newsInfo.Reposts; dr[13] = newsInfo.Comments; //把行加到DT中 dtnewsinfo.Rows.InsertAt(dr, 0); //数据源刷新 if (dtnewsinfo.Rows.Count >= 500) { dtnewsinfo.Rows.RemoveAt(500); } dvWeb.Refresh(); } catch (Exception ex) { StreamWriter sw = File.AppendText("log.txt"); sw.WriteLine(DateTime.Now.ToLongDateString()); sw.WriteLine("begin"); sw.WriteLine(ex.Message); sw.WriteLine(sb.ToString()); sw.WriteLine("end"); sw.WriteLine(""); sw.Close(); } ////总表刷新 //dt.Rows.Add(dr); //dvAll.Refresh(); //得到插入语句 try { if (isThere) { continue; } else { sb.Append(ri.GetInsString(newsInfo) + ";"); } //每10次执行一次插入数据库 if (sb.ToString().Length != 0) { if (i % 10 == 0) { filterStr = sb.ToString(); filterStr = filterStr.Replace("[ ", "["); filterStr = filterStr.Replace(" ]", "]"); //执行插入 cmd.ExecuteNonQuery(filterStr); //清除插入字段串 sb.Clear(); filterStr = ""; } } } catch (Exception ex) { StreamWriter sw = File.AppendText("log.txt"); sw.WriteLine(DateTime.Now.ToLongDateString()); sw.WriteLine("begin"); sw.WriteLine(ex.Message); sw.WriteLine(sb.ToString()); sw.WriteLine("end"); sw.WriteLine(""); sw.Close(); } } } #endregion } #endregion try { if (sb.ToString().Length != 0) { filterStr = sb.ToString(); filterStr = filterStr.Replace("[ ", "["); filterStr = filterStr.Replace(" ]", "]"); //执行插入 cmd.ExecuteNonQuery(filterStr); //清除插入字段串 sb.Clear(); filterStr = ""; } } catch (Exception ex) { StreamWriter sw = File.AppendText("log.txt"); sw.WriteLine(DateTime.Now.ToLongDateString()); sw.WriteLine("begin"); sw.WriteLine(ex.Message); sw.WriteLine(sb.ToString()); sw.WriteLine("end"); sw.WriteLine(""); sw.Close(); } //执行完毕后,重新获取一次数据库的数据 dtWebNewsInfo = tri.SelReleaseInfo(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), DateTime.Now.AddDays(-15).ToString("yyyy-MM-dd HH:mm:ss"), "0 AND webName<>'百度'"); //MessageBox.Show("ok"); lbWeb.Text = "一轮搜索完毕!"; lbWeb.ForeColor = Color.Red; }
public void StartWrite() { Action actionG = () => { List<ModelReleaseInfo> data; while (true) { if (Program.ProClose) { break; } if (Queue.TryDequeue(out data)) { //// 写入数据库 #region 数据入库 try { StringBuilder sb = new StringBuilder(); sb.Append(""); TbReleaseInfo tri = new TbReleaseInfo(); MySqlCmd cmd = new MySqlCmd(); foreach (var mri in data) { #region 2016.11.16 加入精确匹配的判断 string keywords = mri.KeyWords; string title = mri.Title; string context = mri.Contexts; if (!string.IsNullOrEmpty(keywords)) { bool isFundTitle = true; bool isFundContext = true; string[] keyw = keywords.Split(' '); if (keyw != null && keyw.Count() > 0) { foreach (string key in keyw) { if (title.IndexOf(key) < 0) { isFundTitle = false; } if (context.IndexOf(key) < 0) { isFundContext = false; } } } if (!isFundTitle && !isFundContext) { //如果标题或者内容没有匹配全部关键字则去掉该条数据 continue; } } #endregion if (tri.GetReleaseInfoCount(mri.InfoSource, mri.KeyWords) > 0) continue; string sql = tri.GetInsertStr(mri); if (!sql.Trim().EndsWith(";")) { sql += sql + ";"; } sb.Append(sql); } if (sb.ToString().Length > 0) { //执行插入 cmd.ExecuteNonQuery(sb.ToString()); //清除插入字段串 sb.Clear(); } } catch (Exception ex) { Comm.WriteErrorLog(ex.Message); Comm.WriteErrorLog(ex.StackTrace); } #endregion log.Info("数据层写入数据库成功"); } else { System.Threading.Thread.Sleep(1000); } } }; Parallel.Invoke(actionG, actionG); }