//推送评论url. /// <summary> /// 推送评论url /// </summary> private void urlPingLun() { string curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 00:05:00"); //curenttime = DateTime.Now.ToString("yyyy-MM-dd 10:14:00"); richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + ":【周报评论】采集......\r\n请不要关闭程序......\r\n如有意外,请联系采集人员......"); richtextwriteR(this.richTextBox_plurl, "【周报评论】下次时间:" + curenttime + "...0000"); while (true) { if (DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") == curenttime) { try { //查找需要周度推送的品类 string weekCategory = ReadXML("WeekCategory");//周度品类 richtextwriteR(this.richTextBox_plurl, "周度采集的品类:" + weekCategory + ""); if (weekCategory != "") { weekCategory = " AND 品类 NOT IN " + weekCategory + " "; } string monthCategory = ReadXML("MonthCategory"); richtextwriteR(this.richTextBox_plurl, "月度采集的品类:" + monthCategory + ""); if (monthCategory != "") { monthCategory = " AND 品类 NOT IN " + monthCategory + " "; } //推送所有url string strSql = "SELECT '{\"url\": \"'+页面信息+'\",\"attr\": {\"urlleibie\": \"'+品类+'\",\"urlweb\": \"'+电商+'\",\"brand\": \"'+品牌+'\",\"model\": \"'+机型+'\"}}' FROM URLDATA WHERE NEED=0 " + weekCategory + monthCategory + " AND 电商!='天猫商城'"; DataTable dt = db.selectDatas(strSql); List <string> urlList = new List <string>(); foreach (DataRow item in dt.Rows) { urlList.Add(item[0].ToString()); } richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + ":URL插入redis 共有:" + urlList.Count + ""); bool isredisclient = true; while (isredisclient) { try { client.AddRangeToList(plRedisKey, urlList); isredisclient = false; } catch (Exception er) { richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + ":url 注入redis 失败 :" + er.Message + ""); //等等两秒,重新连接 Thread.Sleep(2000); OpenRedisServer(); } } //开始推送一次天猫品类url string tmCategory = ReadXML("TMCategory"); # region 天猫评论url,部分品类 url if (tmCategory != "")//如果为空,则天猫url 不再采集评论 { tmCategory = " AND 品类 IN " + tmCategory + ""; dt = new DataTable(); urlList = new List <string>(); strSql = "SELECT '{\"url\": \"'+页面信息+'\",\"attr\": {\"urlleibie\": \"'+品类+'\",\"urlweb\": \"'+电商+'\",\"brand\": \"'+品牌+'\",\"model\": \"'+机型+'\"}}' FROM URLDATA WHERE NEED=0 " + tmCategory + " AND 电商='天猫商城'"; dt = db.selectDatas(strSql); foreach (DataRow item in dt.Rows) { urlList.Add(item[0].ToString()); } //client.AddRangeToList(plRedisKey, urlList); isredisclient = true; while (isredisclient) { try { client.AddRangeToList(plRedisKey, urlList); isredisclient = false; } catch (Exception er) { richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + ":url 注入redis 失败 :" + er.Message + ""); //等等两秒,重新连接 Thread.Sleep(2000); OpenRedisServer(); } } richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + ":天猫URL插入redis 共有:" + urlList.Count + ""); } #endregion dt = new DataTable(); urlList = new List <string>(); strSql = "SELECT '{\"url\": \"'+A.页面信息+'\",\"attr\": {\"urlleibie\": \"'+A.品类+'\",\"urlweb\": \"'+A.电商+'\",\"brand\": \"\",\"model\": \"\"}}' FROM (SELECT 页面信息,电商,品类 FROM URLDATAJD WHERE (NEED=0 OR NEED IS NULL) AND ISNULL(品类,'')!='' AND 电商!='天猫商城' " + weekCategory + monthCategory + " " + ")A LEFT JOIN (SELECT 页面信息,电商,品类 FROM URLDATA WHERE NEED=0 AND 电商!='天猫商城' " + weekCategory + monthCategory + ")B " + "ON A.页面信息 = B.页面信息 WHERE B.页面信息 IS NULL"; dt = db.selectDatas(strSql); foreach (DataRow item in dt.Rows) { urlList.Add(item[0].ToString()); } //client.AddRangeToList(plRedisKey, urlList); isredisclient = true; while (isredisclient) { try { client.AddRangeToList(plRedisKey, urlList); isredisclient = false; } catch (Exception er) { richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + ":url 注入redis 失败 :" + er.Message + ""); //等等两秒,重新连接 Thread.Sleep(2000); OpenRedisServer(); } } richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + ":查询临时url结束, 共: " + urlList.Count + ""); if (weekCategory != "") { #region //周一 凌晨开始采集的url(不包含天猫url) if (DateTime.Now.DayOfWeek.ToString() == "Monday") { dt = new DataTable(); urlList = new List <string>(); strSql = "SELECT '{\"url\": \"'+页面信息+'\",\"attr\": {\"urlleibie\": \"'+品类+'\",\"urlweb\": \"'+电商+'\",\"brand\": \"'+品牌+'\",\"model\": \"'+机型+'\"}}' FROM URLDATA WHERE NEED=0 " + weekCategory.ToUpper().ToString().Replace("NOT", "") + " AND 电商!='天猫商城'"; dt = db.selectDatas(strSql); foreach (DataRow item in dt.Rows) { urlList.Add(item[0].ToString()); } isredisclient = true; while (isredisclient) { try { client.AddRangeToList(plRedisKey, urlList); isredisclient = false; } catch (Exception er) { richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + ":url 注入redis 失败 :" + er.Message + ""); //等等两秒,重新连接 Thread.Sleep(2000); OpenRedisServer(); } } //client.AddRangeToList(plRedisKey, urlList); dt = new DataTable(); urlList = new List <string>(); strSql = "SELECT '{\"url\": \"'+A.页面信息+'\",\"attr\": {\"urlleibie\": \"'+A.品类+'\",\"urlweb\": \"'+A.电商+'\",\"brand\": \"\",\"model\": \"\"}}' FROM (SELECT 页面信息,电商,品类 FROM URLDATAJD WHERE (NEED=0 OR NEED IS NULL) AND ISNULL(品类,'')!='' AND 电商!='天猫商城' " + weekCategory.ToUpper().ToString().Replace("NOT", "") + " " + ")A LEFT JOIN (SELECT 页面信息,电商,品类 FROM URLDATA WHERE NEED=0 AND 电商!='天猫商城' " + weekCategory.ToUpper().ToString().Replace("NOT", "") + ")B " + "ON A.页面信息 = B.页面信息 WHERE B.页面信息 IS NULL"; dt = db.selectDatas(strSql); foreach (DataRow item in dt.Rows) { urlList.Add(item[0].ToString()); } isredisclient = true; while (isredisclient) { try { client.AddRangeToList(plRedisKey, urlList); isredisclient = false; } catch (Exception er) { richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + ":url 注入redis 失败 :" + er.Message + ""); //等等两秒,重新连接 Thread.Sleep(2000); OpenRedisServer(); } } //client.AddRangeToList(plRedisKey, urlList); richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + ":周度品类查询临时url结束, 共: " + urlList.Count + ""); } #endregion } richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + ":over"); curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 00:05:00"); richtextwriteR(this.richTextBox_plurl, "【周报评论】下次时间:" + curenttime + "..."); }
//每天9点 private void NewMethod() { string curenttime = DateTime.Now.ToString(); if (DateTime.Parse(curenttime).Hour >= 9 && DateTime.Parse(curenttime).Minute >= 5) { curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 09:00:00"); } this.richTextBox1.Invoke(new ThreadStart(delegate() { richTextBox1.Text = "【卖家网】采集......\r\n请不要关闭程序......\r\n如有意外,请联系采集人员......\r\n"; richTextBox1.SelectionStart = richTextBox1.TextLength; richTextBox1.ScrollToCaret(); })); writeR(this.richTextBox1, "【卖家网】下次时间:" + curenttime + "..."); while (true) { if (DateTime.Now >= DateTime.Parse(curenttime)) { try { this.richTextBox1.Invoke(new ThreadStart(delegate() { if (this.richTextBox1.Lines.Length > 40) { richTextBox1.Clear(); } richTextBox1.Text = "【卖家网】采集......\r\n请不要关闭程序......\r\n如有意外,请联系采集人员......\r\n"; richTextBox1.SelectionStart = richTextBox1.TextLength; richTextBox1.ScrollToCaret(); })); writeR(this.richTextBox1, "开始查询URL..."); string sqlStr = "select '{\"category\": \"'+category+'\",\"url\": \"'+url+'\"}' from dba.dbo.卖家网采集页面信息定时推送"; DataTable dt = db.selectDatas(sqlStr); List <string> urlList = new List <string>(); foreach (DataRow item in dt.Rows) { urlList.Add(item[0].ToString()); } writeR(this.richTextBox1, "URL插入redis 共有:" + urlList.Count + " 条"); RedisClient client = new RedisClient("192.168.2.245", 6379); client.AddRangeToList("dpc_maijiawang:url_list", urlList); writeR(this.richTextBox1, "成功,等待下次..."); if (DateTime.Parse(curenttime).Hour >= 9 && DateTime.Parse(curenttime).Minute >= 5) { curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 09:00:00"); } writeR(this.richTextBox1, "【卖家网】下次时间:" + curenttime + "..."); } catch (Exception ecppp) { writeR(this.richTextBox1, "报错:" + ecppp.ToString()); } } else { Thread.Sleep(1000); } this.label1.Invoke(new ThreadStart(delegate() { label1.Text = "当前时间:" + DateTime.Now.ToString() + ""; })); } }
// wgh : 0:05、10:05、16:05、21:05 修改采集时间 private void NewMethod() { string curenttime = DateTime.Now.ToString(); if (DateTime.Parse(curenttime).Hour >= 13 && DateTime.Parse(curenttime).Minute >= 5) { curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 0:10:00"); } // //curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 0:05:00"); else if (DateTime.Parse(curenttime).Hour >= 16 && DateTime.Parse(curenttime).Minute >= 5) { curenttime = DateTime.Now.ToString("yyyy-MM-dd 21:05:00"); } else if (DateTime.Parse(curenttime).Hour >= 10 && DateTime.Parse(curenttime).Minute >= 5) { curenttime = DateTime.Now.ToString("yyyy-MM-dd 16:05:00"); } else { curenttime = DateTime.Now.ToString("yyyy-MM-dd 10:05:00"); } this.richTextBox1.Invoke(new ThreadStart(delegate() { richTextBox1.Text = "【周报】采集......\r\n请不要关闭程序......\r\n如有意外,请联系采集人员......\r\n"; richTextBox1.SelectionStart = richTextBox1.TextLength; richTextBox1.ScrollToCaret(); })); writeR(this.richTextBox1, "【周报】下次时间:" + curenttime + "..."); while (true) { if (DateTime.Now >= DateTime.Parse(curenttime)) { try { this.richTextBox1.Invoke(new ThreadStart(delegate() { if (this.richTextBox1.Lines.Length > 40) { richTextBox1.Clear(); } richTextBox1.Text = "【周报】采集......\r\n请不要关闭程序......\r\n如有意外,请联系采集人员......\r\n"; richTextBox1.SelectionStart = richTextBox1.TextLength; richTextBox1.ScrollToCaret(); })); writeR(this.richTextBox1, "开始查询URL..."); string sqlStr = "select '{\"Urls\": \"'+REPLACE(Urls,';',';')+'\",\"Urlleibie\": \"'+Urlleibie+'\",\"Urlweb\": \"'+Urlweb+'\",\"spbjpinpai\": \"'+ISNULL(spbjpinpai,'')+'\",\"spbjjixing\": \"'+ISNULL(spbjjixing,'')+'\",\"pc\":\"" + curenttime + "\"}' from urldata_tm where need=1"; DataTable dt = db.selectDatas(sqlStr); List <string> urlList = new List <string>(); foreach (DataRow item in dt.Rows) { urlList.Add(item[0].ToString()); } writeR(this.richTextBox1, "URL插入redis 共有:" + urlList.Count + " 条"); RedisClient client = new RedisClient("117.122.192.50", 6479); //RedisClient client = new RedisClient("192.168.2.245", 6379); client.AddRangeToList("sellCountSpider:start_urls", urlList); //client.AddRangeToList("DailySellCountSpider:start_urls", urlList); writeR(this.richTextBox1, "成功,等待下次..."); if (DateTime.Parse(curenttime).Hour >= 13 && DateTime.Parse(curenttime).Minute >= 5) { //if (DateTime.Parse(curenttime).Hour >= 20 && DateTime.Parse(curenttime).Minute >= 5) curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 00:10:00"); } else if (DateTime.Parse(curenttime).Hour >= 16 && DateTime.Parse(curenttime).Minute >= 5) { curenttime = DateTime.Now.ToString("yyyy-MM-dd 21:05:00"); } else if (DateTime.Parse(curenttime).Hour >= 10 && DateTime.Parse(curenttime).Minute >= 5) { curenttime = DateTime.Now.ToString("yyyy-MM-dd 16:05:00"); } else { curenttime = DateTime.Now.ToString("yyyy-MM-dd 10:05:00"); } writeR(this.richTextBox1, "【周报】下次时间:" + curenttime + "..."); } catch (Exception ecppp) { writeR(this.richTextBox1, "报错:" + ecppp.ToString()); } } else { Thread.Sleep(1000); } this.label1.Invoke(new ThreadStart(delegate() { label1.Text = "当前时间:" + DateTime.Now.ToString() + ""; })); } }
private void NewMethod() { while (true) { RedisClient client = new RedisClient("192.168.100.200", 6379); try { byte[] taskbyte = client.RPop("datatoredis:task"); if (taskbyte != null) { string taskinfo = System.Text.Encoding.UTF8.GetString(taskbyte); if (taskinfo != "") { var jo = JObject.Parse(taskinfo); string datatype = Convert.ToString(jo["datatype"]); string datasource = Convert.ToString(jo["datasource"]); string redishost = Convert.ToString(jo["redishost"]); int redisport = Convert.ToInt32(jo["redisport"]); string rediskey = Convert.ToString(jo["rediskey"]); string pattern = Convert.ToString(jo["pattern"]); string starttime = Convert.ToString(jo["starttime"]); int nextaddmin = Convert.ToInt32(jo["nextaddmin"]); if (datasource.ToLower() == "sql") { if (DateTime.Now >= Convert.ToDateTime(starttime)) { if (nextaddmin > 0) { starttime = Convert.ToDateTime(starttime).AddMinutes(nextaddmin).ToString("yyyy-MM-dd HH:mm:ss"); jo["starttime"] = starttime; taskinfo = JsonConvert.SerializeObject(jo); client.LPush("datatoredis:task", Encoding.UTF8.GetBytes(taskinfo)); writeR(this.richTextBox1, "LPush to [datatoredis:task] " + taskinfo); } DataTable dtResult = db.selectDatas(pattern); writeR(this.richTextBox1, "Get pattern for result [" + pattern + "]"); writeR(this.richTextBox1, "Get result " + dtResult.Rows.Count); List <string> urlList = new List <string>(); foreach (DataRow item in dtResult.Rows) { urlList.Add(item[0].ToString()); } RedisClient dataClient = new RedisClient(redishost, redisport); dataClient.AddRangeToList(rediskey, urlList); writeR(this.richTextBox1, "Souccess"); writeR(this.richTextBox1, "***************"); writeR(this.richTextBox1, "***************"); } else { taskinfo = JsonConvert.SerializeObject(jo); client.LPush("datatoredis:task", Encoding.UTF8.GetBytes(taskinfo)); } } } } } catch (Exception ecp) { writeR(this.richTextBox1, "[Error]" + ecp.Message); } Thread.Sleep(1000 * 60 * 5); } }