//推送评论url.
/// <summary>
/// 推送评论url
/// </summary>
private void urlPingLun()
{
string curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 00:05:00");
//curenttime = DateTime.Now.ToString("yyyy-MM-dd 10:14:00");
richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + "：【周报评论】采集......\r\n请不要关闭程序......\r\n如有意外，请联系采集人员......");
richtextwriteR(this.richTextBox_plurl, "【周报评论】下次时间:" + curenttime + "...0000");
while (true)
{
if (DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") == curenttime)
{
try
{
//查找需要周度推送的品类
string weekCategory = ReadXML("WeekCategory");//周度品类
richtextwriteR(this.richTextBox_plurl, "周度采集的品类:" + weekCategory + "");
if (weekCategory != "")
{
weekCategory = " AND 品类 NOT IN " + weekCategory + " ";
}
string monthCategory = ReadXML("MonthCategory");
richtextwriteR(this.richTextBox_plurl, "月度采集的品类:" + monthCategory + "");
if (monthCategory != "")
{
monthCategory = " AND 品类 NOT IN " + monthCategory + " ";
}
//推送所有url
string strSql = "SELECT '{\"url\": \"'+页面信息+'\",\"attr\": {\"urlleibie\": \"'+品类+'\",\"urlweb\": \"'+电商+'\",\"brand\": \"'+品牌+'\",\"model\": \"'+机型+'\"}}' FROM URLDATA WHERE NEED=0 " + weekCategory + monthCategory + " AND 电商!='天猫商城'";
DataTable dt = db.selectDatas(strSql);
List <string> urlList = new List <string>();
foreach (DataRow item in dt.Rows)
{
urlList.Add(item[0].ToString());
}
richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + "：URL插入redis 共有：" + urlList.Count + "");
bool isredisclient = true;
while (isredisclient)
{
try
{
client.AddRangeToList(plRedisKey, urlList);
isredisclient = false;
}
catch (Exception er)
{
richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + "：url 注入redis 失败 ：" + er.Message + "");
//等等两秒，重新连接
Thread.Sleep(2000);
OpenRedisServer();
}
}
//开始推送一次天猫品类url
string tmCategory = ReadXML("TMCategory");
# region 天猫评论url,部分品类 url
if (tmCategory != "")//如果为空，则天猫url 不再采集评论
{
tmCategory = " AND 品类 IN " + tmCategory + "";
dt = new DataTable();
urlList = new List <string>();
strSql = "SELECT '{\"url\": \"'+页面信息+'\",\"attr\": {\"urlleibie\": \"'+品类+'\",\"urlweb\": \"'+电商+'\",\"brand\": \"'+品牌+'\",\"model\": \"'+机型+'\"}}' FROM URLDATA WHERE NEED=0 " + tmCategory + " AND 电商='天猫商城'";
dt = db.selectDatas(strSql);
foreach (DataRow item in dt.Rows)
{
urlList.Add(item[0].ToString());
}
//client.AddRangeToList(plRedisKey, urlList);
isredisclient = true;
while (isredisclient)
{
try
{
client.AddRangeToList(plRedisKey, urlList);
isredisclient = false;
}
catch (Exception er)
{
richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + "：url 注入redis 失败 ：" + er.Message + "");
//等等两秒，重新连接
Thread.Sleep(2000);
OpenRedisServer();
}
}
richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + "：天猫URL插入redis 共有：" + urlList.Count + "");
}
#endregion
dt = new DataTable();
urlList = new List <string>();
strSql = "SELECT '{\"url\": \"'+A.页面信息+'\",\"attr\": {\"urlleibie\": \"'+A.品类+'\",\"urlweb\": \"'+A.电商+'\",\"brand\": \"\",\"model\": \"\"}}' FROM (SELECT 页面信息,电商,品类 FROM URLDATAJD WHERE (NEED=0 OR NEED IS NULL) AND ISNULL(品类,'')!='' AND 电商!='天猫商城' " + weekCategory + monthCategory + " " +
")A LEFT JOIN (SELECT 页面信息,电商,品类 FROM URLDATA WHERE NEED=0 AND 电商!='天猫商城' " + weekCategory + monthCategory + ")B " +
"ON A.页面信息 = B.页面信息 WHERE B.页面信息 IS NULL";
dt = db.selectDatas(strSql);
foreach (DataRow item in dt.Rows)
{
urlList.Add(item[0].ToString());
}
//client.AddRangeToList(plRedisKey, urlList);
isredisclient = true;
while (isredisclient)
{
try
{
client.AddRangeToList(plRedisKey, urlList);
isredisclient = false;
}
catch (Exception er)
{
richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + "：url 注入redis 失败 ：" + er.Message + "");
//等等两秒，重新连接
Thread.Sleep(2000);
OpenRedisServer();
}
}
richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + "：查询临时url结束， 共： " + urlList.Count + "");
if (weekCategory != "")
{
#region //周一 凌晨开始采集的url(不包含天猫url)
if (DateTime.Now.DayOfWeek.ToString() == "Monday")
{
dt = new DataTable();
urlList = new List <string>();
strSql = "SELECT '{\"url\": \"'+页面信息+'\",\"attr\": {\"urlleibie\": \"'+品类+'\",\"urlweb\": \"'+电商+'\",\"brand\": \"'+品牌+'\",\"model\": \"'+机型+'\"}}' FROM URLDATA WHERE NEED=0 " + weekCategory.ToUpper().ToString().Replace("NOT", "") + " AND 电商!='天猫商城'";
dt = db.selectDatas(strSql);
foreach (DataRow item in dt.Rows)
{
urlList.Add(item[0].ToString());
}
isredisclient = true;
while (isredisclient)
{
try
{
client.AddRangeToList(plRedisKey, urlList);
isredisclient = false;
}
catch (Exception er)
{
richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + "：url 注入redis 失败 ：" + er.Message + "");
//等等两秒，重新连接
Thread.Sleep(2000);
OpenRedisServer();
}
}
//client.AddRangeToList(plRedisKey, urlList);
dt = new DataTable();
urlList = new List <string>();
strSql = "SELECT '{\"url\": \"'+A.页面信息+'\",\"attr\": {\"urlleibie\": \"'+A.品类+'\",\"urlweb\": \"'+A.电商+'\",\"brand\": \"\",\"model\": \"\"}}' FROM (SELECT 页面信息,电商,品类 FROM URLDATAJD WHERE (NEED=0 OR NEED IS NULL) AND ISNULL(品类,'')!='' AND 电商!='天猫商城' " + weekCategory.ToUpper().ToString().Replace("NOT", "") + " " +
")A LEFT JOIN (SELECT 页面信息,电商,品类 FROM URLDATA WHERE NEED=0 AND 电商!='天猫商城' " + weekCategory.ToUpper().ToString().Replace("NOT", "") + ")B " +
"ON A.页面信息 = B.页面信息 WHERE B.页面信息 IS NULL";
dt = db.selectDatas(strSql);
foreach (DataRow item in dt.Rows)
{
urlList.Add(item[0].ToString());
}
isredisclient = true;
while (isredisclient)
{
try
{
client.AddRangeToList(plRedisKey, urlList);
isredisclient = false;
}
catch (Exception er)
{
richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + "：url 注入redis 失败 ：" + er.Message + "");
//等等两秒，重新连接
Thread.Sleep(2000);
OpenRedisServer();
}
}
//client.AddRangeToList(plRedisKey, urlList);
richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + "：周度品类查询临时url结束， 共： " + urlList.Count + "");
}
#endregion
}
richtextwriteR(this.richTextBox_plurl, DateTime.Now.ToString() + "：over");
curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 00:05:00");
richtextwriteR(this.richTextBox_plurl, "【周报评论】下次时间:" + curenttime + "...");
}
//每天9点
private void NewMethod()
{
string curenttime = DateTime.Now.ToString();
if (DateTime.Parse(curenttime).Hour >= 9 && DateTime.Parse(curenttime).Minute >= 5)
{
curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 09:00:00");
}
this.richTextBox1.Invoke(new ThreadStart(delegate()
{
richTextBox1.Text = "【卖家网】采集......\r\n请不要关闭程序......\r\n如有意外，请联系采集人员......\r\n";
richTextBox1.SelectionStart = richTextBox1.TextLength;
richTextBox1.ScrollToCaret();
}));
writeR(this.richTextBox1, "【卖家网】下次时间:" + curenttime + "...");
while (true)
{
if (DateTime.Now >= DateTime.Parse(curenttime))
{
try
{
this.richTextBox1.Invoke(new ThreadStart(delegate()
{
if (this.richTextBox1.Lines.Length > 40)
{
richTextBox1.Clear();
}
richTextBox1.Text = "【卖家网】采集......\r\n请不要关闭程序......\r\n如有意外，请联系采集人员......\r\n";
richTextBox1.SelectionStart = richTextBox1.TextLength;
richTextBox1.ScrollToCaret();
}));
writeR(this.richTextBox1, "开始查询URL...");
string sqlStr = "select '{\"category\": \"'+category+'\",\"url\": \"'+url+'\"}' from dba.dbo.卖家网采集页面信息定时推送";
DataTable dt = db.selectDatas(sqlStr);
List <string> urlList = new List <string>();
foreach (DataRow item in dt.Rows)
{
urlList.Add(item[0].ToString());
}
writeR(this.richTextBox1, "URL插入redis 共有：" + urlList.Count + " 条");
RedisClient client = new RedisClient("192.168.2.245", 6379);
client.AddRangeToList("dpc_maijiawang:url_list", urlList);
writeR(this.richTextBox1, "成功，等待下次...");
if (DateTime.Parse(curenttime).Hour >= 9 && DateTime.Parse(curenttime).Minute >= 5)
{
curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 09:00:00");
}
writeR(this.richTextBox1, "【卖家网】下次时间:" + curenttime + "...");
}
catch (Exception ecppp)
{
writeR(this.richTextBox1, "报错：" + ecppp.ToString());
}
}
else
{
Thread.Sleep(1000);
}
this.label1.Invoke(new ThreadStart(delegate()
{
label1.Text = "当前时间：" + DateTime.Now.ToString() + "";
}));
}
}
// wgh : 0:05、10:05、16:05、21:05 修改采集时间
private void NewMethod()
{
string curenttime = DateTime.Now.ToString();
if (DateTime.Parse(curenttime).Hour >= 13 && DateTime.Parse(curenttime).Minute >= 5)
{
curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 0:10:00");
}
// //curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 0:05:00");
else if (DateTime.Parse(curenttime).Hour >= 16 && DateTime.Parse(curenttime).Minute >= 5)
{
curenttime = DateTime.Now.ToString("yyyy-MM-dd 21:05:00");
}
else if (DateTime.Parse(curenttime).Hour >= 10 && DateTime.Parse(curenttime).Minute >= 5)
{
curenttime = DateTime.Now.ToString("yyyy-MM-dd 16:05:00");
}
else
{
curenttime = DateTime.Now.ToString("yyyy-MM-dd 10:05:00");
}
this.richTextBox1.Invoke(new ThreadStart(delegate()
{
richTextBox1.Text = "【周报】采集......\r\n请不要关闭程序......\r\n如有意外，请联系采集人员......\r\n";
richTextBox1.SelectionStart = richTextBox1.TextLength;
richTextBox1.ScrollToCaret();
}));
writeR(this.richTextBox1, "【周报】下次时间:" + curenttime + "...");
while (true)
{
if (DateTime.Now >= DateTime.Parse(curenttime))
{
try
{
this.richTextBox1.Invoke(new ThreadStart(delegate()
{
if (this.richTextBox1.Lines.Length > 40)
{
richTextBox1.Clear();
}
richTextBox1.Text = "【周报】采集......\r\n请不要关闭程序......\r\n如有意外，请联系采集人员......\r\n";
richTextBox1.SelectionStart = richTextBox1.TextLength;
richTextBox1.ScrollToCaret();
}));
writeR(this.richTextBox1, "开始查询URL...");
string sqlStr = "select '{\"Urls\": \"'+REPLACE(Urls,'；',';')+'\",\"Urlleibie\": \"'+Urlleibie+'\",\"Urlweb\": \"'+Urlweb+'\",\"spbjpinpai\": \"'+ISNULL(spbjpinpai,'')+'\",\"spbjjixing\": \"'+ISNULL(spbjjixing,'')+'\",\"pc\":\"" + curenttime + "\"}' from urldata_tm where need=1";
DataTable dt = db.selectDatas(sqlStr);
List <string> urlList = new List <string>();
foreach (DataRow item in dt.Rows)
{
urlList.Add(item[0].ToString());
}
writeR(this.richTextBox1, "URL插入redis 共有：" + urlList.Count + " 条");
RedisClient client = new RedisClient("117.122.192.50", 6479);
//RedisClient client = new RedisClient("192.168.2.245", 6379);
client.AddRangeToList("sellCountSpider:start_urls", urlList);
//client.AddRangeToList("DailySellCountSpider:start_urls", urlList);
writeR(this.richTextBox1, "成功，等待下次...");
if (DateTime.Parse(curenttime).Hour >= 13 && DateTime.Parse(curenttime).Minute >= 5)
{
//if (DateTime.Parse(curenttime).Hour >= 20 && DateTime.Parse(curenttime).Minute >= 5)
curenttime = DateTime.Now.AddDays(1).ToString("yyyy-MM-dd 00:10:00");
}
else if (DateTime.Parse(curenttime).Hour >= 16 && DateTime.Parse(curenttime).Minute >= 5)
{
curenttime = DateTime.Now.ToString("yyyy-MM-dd 21:05:00");
}
else if (DateTime.Parse(curenttime).Hour >= 10 && DateTime.Parse(curenttime).Minute >= 5)
{
curenttime = DateTime.Now.ToString("yyyy-MM-dd 16:05:00");
}
else
{
curenttime = DateTime.Now.ToString("yyyy-MM-dd 10:05:00");
}
writeR(this.richTextBox1, "【周报】下次时间:" + curenttime + "...");
}
catch (Exception ecppp)
{
writeR(this.richTextBox1, "报错：" + ecppp.ToString());
}
}
else
{
Thread.Sleep(1000);
}
this.label1.Invoke(new ThreadStart(delegate()
{
label1.Text = "当前时间：" + DateTime.Now.ToString() + "";
}));
}
}
private void NewMethod()
{
while (true)
{
RedisClient client = new RedisClient("192.168.100.200", 6379);
try
{
byte[] taskbyte = client.RPop("datatoredis:task");
if (taskbyte != null)
{
string taskinfo = System.Text.Encoding.UTF8.GetString(taskbyte);
if (taskinfo != "")
{
var jo = JObject.Parse(taskinfo);
string datatype = Convert.ToString(jo["datatype"]);
string datasource = Convert.ToString(jo["datasource"]);
string redishost = Convert.ToString(jo["redishost"]);
int redisport = Convert.ToInt32(jo["redisport"]);
string rediskey = Convert.ToString(jo["rediskey"]);
string pattern = Convert.ToString(jo["pattern"]);
string starttime = Convert.ToString(jo["starttime"]);
int nextaddmin = Convert.ToInt32(jo["nextaddmin"]);
if (datasource.ToLower() == "sql")
{
if (DateTime.Now >= Convert.ToDateTime(starttime))
{
if (nextaddmin > 0)
{
starttime = Convert.ToDateTime(starttime).AddMinutes(nextaddmin).ToString("yyyy-MM-dd HH:mm:ss");
jo["starttime"] = starttime;
taskinfo = JsonConvert.SerializeObject(jo);
client.LPush("datatoredis:task", Encoding.UTF8.GetBytes(taskinfo));
writeR(this.richTextBox1, "LPush to [datatoredis:task] " + taskinfo);
}
DataTable dtResult = db.selectDatas(pattern);
writeR(this.richTextBox1, "Get pattern for result [" + pattern + "]");
writeR(this.richTextBox1, "Get result " + dtResult.Rows.Count);
List <string> urlList = new List <string>();
foreach (DataRow item in dtResult.Rows)
{
urlList.Add(item[0].ToString());
}
RedisClient dataClient = new RedisClient(redishost, redisport);
dataClient.AddRangeToList(rediskey, urlList);
writeR(this.richTextBox1, "Souccess");
writeR(this.richTextBox1, "***************");
writeR(this.richTextBox1, "***************");
}
else
{
taskinfo = JsonConvert.SerializeObject(jo);
client.LPush("datatoredis:task", Encoding.UTF8.GetBytes(taskinfo));
}
}
}
}
}
catch (Exception ecp)
{
writeR(this.richTextBox1, "[Error]" + ecp.Message);
}
Thread.Sleep(1000 * 60 * 5);
}
}