/// <summary>
/// 点击查看手机
/// </summary>
public static void ClickLookPhone_PDD(ChromiumWebBrowser webBrowser2)
{
Browser.Delay(1000);
CaiJi.JsXml_No("PDDClickPhone", webBrowser2);
//Browser.JS_CEFBrowser_NoReturn(CaiJi.GetJsByXmlValue("PDDClickPhone"), webBrowser2);
Browser.Delay(1000);
}
private void button2_Click(object sender, EventArgs e)
{
string url = txt_url.Text.Trim();
WebBrowser web = new WebBrowser();
web.Navigate(url);
if (Browser.WaitWebPageLoad(web))
{
Debug.WriteLine(CaiJi.GetShopType(web).ToString());
}
}
/// <summary>
/// 根据店铺商品搜索页面获取商品基本信息
/// </summary>
/// <param name="DateCaiJi">采集时间</param>
/// <param name="webBrowser1">浏览器</param>
/// <returns>List<Entity.wanggong_goods></returns>
List <Entity.wanggong_goods> SearchPage_GetWangGongGoods(DateTime DateCaiJi, WebBrowser webBrowser1)
{
List <wanggong_goods> list = new List <wanggong_goods>();
Browser.jsstr = " getClassName('photo').length; ";
string res = Browser.JS_WebBrowser(webBrowser1);
Debug.WriteLine("本页面商品数量为：" + res);//输出
int list_count1 = 0;
//如果没有商品，则返回空
if (!int.TryParse(res, out list_count1))
{
Debug.WriteLine("没有获取到商品");//输出
return(list);
}
else
{
//遍历每个商品
for (int i = 0; i < list_count1; i++)
{
//进度条+1
toolStripProgressBar1.PerformStep();
wanggong_goods wg = new wanggong_goods();
string tbid = CaiJi.SearchPage_GetTBID(i, webBrowser1);
IList <wanggong_goods> list_temp = BLL2.wanggong_goodsManager.Search(1, 10, tbid, 0, new DateTime(), new DateTime(), "");
if (list_temp.Count > 0)
{
wg = list_temp[0];
}
wg.wgcollectdate = DateCaiJi;
wg.wgimg = CaiJi.SearchPage_GetImgUrl(i, webBrowser1);
wg.wgprice2 = CaiJi.SearchPage_GetPrice(i, webBrowser1);
wg.wgtittle = CaiJi.SearchPage_GetTitle(i, webBrowser1);
wg.wgurl = CaiJi.SearchPage_GetGoodsUrl(i, webBrowser1);
wg.wgxiaoliang = CaiJi.SearchPage_GetSales(i, webBrowser1);
wg.wgTBid = CaiJi.SearchPage_GetTBID(i, webBrowser1);
//下载图片
string imgurl = CaiJi.GetImgSizeByUrl(wg.wgimg, 150);
string imgname = !string.IsNullOrEmpty(wg.wgremark) ? wg.wgremark : DateTime.Now.ToString("yyyyMMddhhmmss") + Manager.RandomNumber(1000, 99999).ToString() + ".jpg";
Manager.DownloadFile(imgurl, Manager.imgsite_wanggong, imgname, 1000);
wg.wgremark = imgname;
list.Add(wg);
dgv1.DataSource = Manager.BindingSort_wanggong_goods(list);
}
}
return(list);
}
public static string CheckPR(string m_Url)
{
string text = "http://toolbarqueries.google.com/search?client=navclient-auto&googleip=O;937&ie=UTF-8&oe=UTF-8&features=Rank&ch=" + OutputCheckSum(m_Url, 1) + "&q=info:" + m_Url;
string text2 = CaiJi.GetHtmlSource(text);
if (text2.Length < 20)
{
Regex regex = new Regex("Rank_(?:[0-9]):(?:[0-9]):(?<PageRank>[0-9]+)");
string value = regex.Match(text2).Groups["PageRank"].ToString();
value = value.Length == 0 ? "0" : value;
return(value);
}
return("-1");
}
private void btn_img_Click(object sender, EventArgs e)
{
toolStripProgressBar1.Value = 0;
Common.BindingCollection <wanggong_goods> list = (BindingCollection <wanggong_goods>)dgv1.DataSource;
//设置状态栏的最大值
toolStripProgressBar1.Maximum = list.Count;
//判断图片文件夹是否存在
if (!System.IO.Directory.Exists(Manager.imgsite_wanggong))
{
//新建图片文件夹
System.IO.Directory.CreateDirectory(Manager.imgsite_wanggong);
}
foreach (wanggong_goods goods in list)
{
toolStripProgressBar1.PerformStep();
if (!string.IsNullOrEmpty(goods.wgimg))
{
//判断是否有该图片
if (!System.IO.File.Exists(Manager.imgsite_wanggong + "\\" + goods.wgremark))
{
if (string.IsNullOrEmpty(goods.wgremark))
{
goods.wgremark = DateTime.Now.ToString("yyyyMMddhhmmss") + Manager.RandomNumber(111, 9999).ToString();
BLL2.wanggong_goodsManager.Update(goods);
}
string imgurl = CaiJi.GetImgSizeByUrl(goods.wgimg, 150);
Manager.DownloadFile(imgurl, Manager.imgsite_wanggong, goods.wgremark, 5000);
}
}
}
//设置状态栏全部完成
toolStripProgressBar1.Value = list.Count;
dgv1.DataBindings.Clear();
dgv1.DataSource = Manager.BindingSort_wanggong_goods(list);
MessageBox.Show("图片采集完成");
}
void BuildDG()
{
Regex re;
string Pattern;
//取本站的链接并筛选出友情链接块
string tempstr = CaiJi.GetHtmlSource(txtMyURL.Text);
Pattern = "<a[^<>]+href\\s*=\\s*[\"']*" + txtS.Text + @"[\s\S]+" + txtD.Text + "[^<>]*?>[\\s\\S]+?</a>";
re = new Regex(Pattern, RegexOptions.IgnoreCase);
Match match = re.Match(tempstr);
tempstr = match.Value;
//从友情链接块里取出链接地址及链接名称
Pattern = "<a[^<>]+href\\s*=\\s*[\"']*([^<>\"' ]+/*)[\"']*[^<>]*?>([\\s\\S]+?)</a>";
re = new Regex(Pattern, RegexOptions.IgnoreCase);
MatchCollection matchs = re.Matches(tempstr);
foreach (Match m in matchs)
{
this.dataGridView1.Rows.Add(m.Groups[2].Value, m.Groups[1].Value);
}
}
private void btn_caiji1_Click(object sender, EventArgs e)
{
GC.Collect();
dgv1.Rows.Clear();
string str = txt_key.Text.Trim();
//判断是否已经选择采集店铺
string temp_dp = cb_dianpu.SelectedValue.ToString();
if (!string.IsNullOrEmpty(temp_dp))
{
IList <opponent_dianpu> temp_listdp2 = BLL2.opponent_dianpuManager.Search(1, 100, temp_dp, "");
if (temp_listdp2.Count > 0)
{
str = temp_listdp2[0].odwangwang;
}
}
//str = "https://item.taobao.com/item.htm?spm=a1z10.3-c-s.w4002-14460596614.23.XM7KUF&id=540733715384";//sunny家网址
if (string.IsNullOrEmpty(str))
{
MessageBox.Show("请输入要分析的店铺");
return;
//str = "miss原创定制";
}
//判断是否是网址
int temp_isurl = str.IndexOf("http");
string id = Manager.GetValueByURL(str, "id");
string datenow = DateTime.Now.ToString("yyyy年MM月dd日HH时mm分");
//如果是网址
if (temp_isurl > -1 && !string.IsNullOrEmpty(id))
{
//string url = Manager.GetDPurlByWangwang(str);
Browser.urlstr = str;
webBrowser1.Navigate(Browser.urlstr);
//判断是否加载完成
if (Browser.WaitWebPageLoad(webBrowser1))
{
try
{
if (webBrowser1.Document != null)
{
//判断是否是邮费链接
HtmlElementCollection elem1 = webBrowser1.Document.GetElementById("attributes").GetElementsByTagName("ul")[0].GetElementsByTagName("li");
if (elem1.Count > 0)
{
//string datenow = DateTime.Now.ToString("yyyy年MM月dd日HH时mm分");
//先采集商品
opponent_goods og = CaiJi.CaijiGoods(datenow, webBrowser1);
List <opponent_goods> list = new List <opponent_goods>();
if (og != null)
{
list.Add(og);
dgv1.DataSource = Manager.BindingSort_opponent_goods(list);
//再采集店铺信息，需要跳转页面
Entity.opponent_dianpu dp = CaiJi.CaijiDP(datenow, webBrowser1);
BLL2.opponent_dianpuManager.Insert(dp);
opponent_dianpu_info dpinfo = CaiJi.CaijiDPSDR(datenow, webBrowser1);
dpinfo = bind_dianpu_info(list, dpinfo);
BLL2.opponent_dianpu_infoManager.Insert(dpinfo);
BLL2.opponent_goodsManager.Insert(og);
GC.Collect();//释放内存
}
}
else
{
Debug.WriteLine("======================================================================");
Debug.WriteLine("网址：" + Browser.urlstr);
Debug.WriteLine("该商品可能是 邮费链接 ");
}
}
}
catch (Exception ex)
{
//MessageBox.Show("采集信息失败：" + ex.Message);
Debug.WriteLine("======================================================================");
Debug.WriteLine("网址：" + Browser.urlstr);
Debug.WriteLine(DateTime.Now.ToString());
Debug.WriteLine("单品采集信息失败：" + ex.Message);
}
}
}
else
{
//如果不是网址
string url = CaiJi.GetShopSearchURL(str);
url = Browser.UrlEncode(url, Encoding.UTF8);
//Debug.WriteLine(url);
//跳转到搜索店铺页面
Browser.urlstr = url;
webBrowser1.Navigate(Browser.urlstr);
//判断是否加载完成
if (Browser.WaitWebPageLoad(webBrowser1))
{
try
{
if (webBrowser1.Document != null)
{
string url_shop = CaiJi.GetShopUrlByWangWang(str, webBrowser1);
//如果没有找到店铺网址，或者没有开店信息
if (string.IsNullOrEmpty(url_shop))
{
Debug.WriteLine("采集店铺时，没有找到店铺网址，或者没有开店信息");
MessageBox.Show("没有采集到该店铺信息");
return;
}
List <string> list_url = new List <string>();
List <opponent_goods> list_goods = new List <opponent_goods>();
//默认查询20页
for (int i = 0; i < 20; i++)
{
string url_goodslist = CaiJi.GetShopGoodsListURL(url_shop, i + 1);
Browser.urlstr = url_goodslist;
//跳转到商品列表页面
webBrowser1.Navigate(Browser.urlstr);
//break;
//判断是否加载完成
if (Browser.WaitWebPageLoad(webBrowser1))
{
if (webBrowser1.Document != null)
{
//判断是否还有商品
Browser.jsstr = " getClassName('no-result-new').length; ";
if (Browser.JS_WebBrowser(webBrowser1) == "1")
{
break;
}
else
{
//================
//list_url = CaiJi.GetGoodsUrlList(webBrowser1);
//获取本页面商品网址List
List <string> temp_list_url = CaiJi.GetGoodsUrlList(webBrowser1);
Debug.WriteLine("获取本页面商品网址List" + temp_list_url.Count.ToString());
if (temp_list_url.Count > 0)
{
foreach (string temp_url1 in temp_list_url)
{
Browser.urlstr = temp_url1;
//跳转到商品页面
webBrowser1.Navigate(Browser.urlstr);
if (Browser.WaitWebPageLoad(webBrowser1))
{
if (webBrowser1.Document != null)
{
//判断是否是邮费链接
HtmlElementCollection elem1 = webBrowser1.Document.GetElementById("attributes").GetElementsByTagName("ul")[0].GetElementsByTagName("li");
if (elem1.Count > 0)
{
opponent_goods og = CaiJi.CaijiGoods(datenow, webBrowser1);
if (og != null)
{
list_goods.Add(og);
dgv1.DataSource = Manager.BindingSort_opponent_goods(list_goods);
BLL2.opponent_goodsManager.Insert(og);
Debug.WriteLine("添加商品：" + og.ogtittle);
//绑定店铺信息
opponent_dianpu_info dpinfo2 = new opponent_dianpu_info();
dpinfo2 = bind_dianpu_info(list_goods, dpinfo2);
GC.Collect();//释放内存
}
}
else
{
Debug.WriteLine("======================================================================");
Debug.WriteLine("网址：" + Browser.urlstr);
Debug.WriteLine("该商品可能是 邮费链接 ");
}
}
}
}
}
}
}
}
}
dgv1.DataSource = Manager.BindingSort_opponent_goods(list_goods);
//在本页获取店铺信息
opponent_dianpu dp = CaiJi.CaijiDP(datenow, webBrowser1);
//在本页获取店铺信息
opponent_dianpu_info dpinfo = CaiJi.CaijiDPSDR(datenow, webBrowser1);
//从商品列表获取店铺信息
dpinfo = bind_dianpu_info(list_goods, dpinfo);
BLL2.opponent_dianpu_infoManager.Insert(dpinfo);
BLL2.opponent_dianpuManager.Insert(dp);
GC.Collect();//释放内存
}
}
catch (Exception ex)
{
Debug.WriteLine("======================================================================");
Debug.WriteLine("网址：" + Browser.urlstr);
Debug.WriteLine(DateTime.Now.ToString());
Debug.WriteLine("全店采集信息失败：" + ex.Message);
MessageBox.Show("全店采集信息失败：" + ex.Message);
}
}
}
bind_dianpu();
MessageBox.Show("采集完成");
}
public void ThreadProc()
{
try
{
bool flags = false;
string tempstr = "";
string myurl = txtMyURL.Text.Trim();
string mydomain;
if (myurl.Substring(myurl.Length - 1) == "/")
{
myurl = myurl.Substring(0, myurl.Length - 1);
}
mydomain = myurl;
mydomain = Regex.Match(mydomain, "http://[^/]+").Value;
//友情链接检测工具http://regexlib.com/RETester.aspx
//<a[^<>]+href\s*=\s*["']*([^"' ]*)["']*[^<>]*?>([\s\S]+?)</a>
// <a[\s\S]*?href=("(?<href>[^"]*)"|'(?<href>[^']*)'|(?<href>[^>\s]*))[^>]*?>(?<title>[\s\S]*?)</a>
// @"<a[\s\S]*?href=(""(?<href>[^""]*)""|'(?<href>[^']*)'|(?<href>[^>\s]*))[^>]*?>(?<title>[\s\S]*?)</a>",RegexOptions.IgnoreCase | RegexOptions.Compiled);
//string pattion = "<a[^<>]+href\\s*=\\s*[\"']*(" + mydomain + "/*)[\"']*[^<>]*?>([\\s\\S]+?)</a>";
//string pattion = @"<a[\s\S]+?href=(""[^""]*"")|('[^']*')|([^>\s]*)[^>]*?>[\s\S]*?</a>";
string pattion = "<a[^<>]+href\\s*=\\s*[\"']*(\"" + mydomain + "/*\")|('" + mydomain + "/*')|(" + mydomain + "/*)[^<>]*?>([\\s\\S]+?)</a>";
for (int i = 0; i < this.dataGridView1.RowCount; i++)
{
if (Convert.ToBoolean(this.dataGridView1.Rows[i].Cells["blFlag"].Value) != true)
{
flags = true;
string nowurl;
this.dataGridView1.Rows[i].Cells["blFlag"].Value = !Convert.ToBoolean(this.dataGridView1.Rows[i].Cells["blFlag"].Value);
nowurl = Convert.ToString(this.dataGridView1.Rows[i].Cells["txtWebURL"].Value).ToLower();
tempstr = CaiJi.GetHtmlSource(nowurl);
if (Regex.IsMatch(tempstr, pattion, RegexOptions.IgnoreCase))
{
this.dataGridView1.Rows[i].Cells["txtStatus"].Value = "正常";
}
else
{
this.dataGridView1.Rows[i].Cells["txtStatus"].Value = "不正常";
}
if (chkPR.Checked)
{
this.dataGridView1.Rows[i].Cells["txtPR"].Value = PageRank.GetGooglePR(nowurl);
}
}
}
if (flags)
{
ThreadProc();
}
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
}
/// <summary>
/// 采集网供数据
/// </summary>
void CaiJiWG()
{
GC.Collect();
dgv1.Rows.Clear();
//设置采集时间
DateCaiJi = DateTime.Now;
//设置店铺ID
int dpid = 0;
string str = cb_dianpu.Text.Trim();
string temp_shopid = cb_dianpu.SelectedValue == null ? "" : cb_dianpu.SelectedValue.ToString();
//判断是否是网址
int temp_isurl = str.IndexOf("http");
//string id = Manager.GetValueByURL(str, "id");
if (temp_isurl > -1 || str == "请选择")
{
MessageBox.Show("请输入正确的旺旺名称或者店铺名称");
return;
}
//如果是选择下拉菜单中的店铺
if (temp_shopid != "0")
{
if (int.TryParse(temp_shopid, out dpid))
{
try
{
wanggong_dianpu temp_dp = BLL2.wanggong_dianpuManager.SearchByID(dpid);
//wanggong_dianpu temp_dp = BLL2.wanggong_dianpuManager.Search(1, 10, temp_shopid, "", new DateTime(), new DateTime(), "")[0];
str = temp_dp.wdwangwang;
}
catch
{ }
}
}
string pagenum = txt_page.Text.Trim();
int page_temp = 0;
if (!int.TryParse(pagenum, out page_temp))
{
MessageBox.Show("请输入正确的采集页数");
return;
}
//设置进度条
toolStripProgressBar1.Maximum = page_temp * 24;
toolStripProgressBar1.Value = 0;
//根据旺旺查询店铺ID
string url = CaiJi.GetShopSearchURL(str);
//url = Browser.UrlEncode(url, Encoding.UTF8);
//Debug.WriteLine(url);
//跳转到搜索店铺页面
Browser.urlstr = url;
webBrowser1.Navigate(Browser.urlstr);
//判断是否加载完成
if (Browser.WaitWebPageLoad(webBrowser1))
{
try
{
if (webBrowser1.Document != null)
{
string url_shop = CaiJi.GetShopUrlByWangWang(str, webBrowser1);
//如果没有找到店铺网址，或者没有开店信息
if (string.IsNullOrEmpty(url_shop))
{
Debug.WriteLine("采集店铺时，没有找到店铺网址，或者没有开店信息");
MessageBox.Show("没有采集到该店铺信息");
return;
}
List <string> list_url = new List <string>();
List <wanggong_goods> list_goods = new List <wanggong_goods>();
//默认查询20页
for (int i = 0; i < page_temp; i++)
{
string url_goodslist = CaiJi.GetShopGoodsListURL_NewOn(url_shop, i + 1);
Browser.urlstr = url_goodslist;
//跳转到商品列表页面
webBrowser1.Navigate(Browser.urlstr);
//判断是否加载完成
if (Browser.WaitWebPageLoad(webBrowser1))
{
if (webBrowser1.Document != null)
{
//设置店铺ID
if (dpid == 0)
{
//在本页获取店铺信息
wanggong_dianpu dp = CaiJi.SearchPage_DianPu(DateCaiJi, webBrowser1);
BLL2.wanggong_dianpuManager.Insert(dp);
IList <wanggong_dianpu> list_dianpu = BLL2.wanggong_dianpuManager.Search(1, 1, dp.wdTBID, "", DateCaiJi, DateCaiJi, "");
if (list_dianpu.Count > 0)
{
dpid = list_dianpu[0].wid;
}
else
{
MessageBox.Show("采集失败，获取店铺信息失败");
return;
}
}
//判断是否还有商品
Browser.jsstr = " getClassName('no-result-new').length; ";
if (Browser.JS_WebBrowser(webBrowser1) == "1")
{
break;
}
else
{
//开始采集商品
//List<Entity.wanggong_goods> list_wg = SearchPage_GetWangGongGoods(DateCaiJi, webBrowser1);
#region
//开始采集
//List<wanggong_goods> list_wg = new List<wanggong_goods>();
Browser.jsstr = " getClassName('photo').length; ";
string res = Browser.JS_WebBrowser(webBrowser1);
Debug.WriteLine("本页面商品数量为：" + res);//输出
int list_count1 = 0;
//如果没有商品，则返回空
if (!int.TryParse(res, out list_count1))
{
Debug.WriteLine("没有获取到商品");//输出
break;
}
else
{
//遍历每个商品
for (int j = 0; j < list_count1; j++)
{
//进度条+1
toolStripProgressBar1.PerformStep();
wanggong_goods wg = new wanggong_goods();
string tbid = CaiJi.SearchPage_GetTBID(j, webBrowser1);
IList <wanggong_goods> list_temp = BLL2.wanggong_goodsManager.Search(1, 10, tbid, 0, new DateTime(), new DateTime(), "");
if (list_temp.Count > 0)
{
wg = list_temp[0];
}
wg.wgcollectdate = DateCaiJi;
wg.wgimg = CaiJi.SearchPage_GetImgUrl(j, webBrowser1);
wg.wgprice2 = CaiJi.SearchPage_GetPrice(j, webBrowser1);
wg.wgtittle = CaiJi.SearchPage_GetTitle(j, webBrowser1);
wg.wgurl = CaiJi.SearchPage_GetGoodsUrl(j, webBrowser1);
wg.wgxiaoliang = CaiJi.SearchPage_GetSales(j, webBrowser1);
wg.wgTBid = CaiJi.SearchPage_GetTBID(j, webBrowser1);
//下载图片
string imgurl = CaiJi.GetImgSizeByUrl(wg.wgimg, 150);
string imgname = !string.IsNullOrEmpty(wg.wgremark) ? wg.wgremark : DateTime.Now.ToString("yyyyMMddhhmmss") + Manager.RandomNumber(1000, 99999).ToString() + ".jpg";
Manager.DownloadFile(imgurl, Manager.imgsite_wanggong, imgname, 1000);
wg.wgremark = imgname;
list_goods.Add(wg);
dgv1.DataSource = Manager.BindingSort_wanggong_goods(list_goods);
}
}
#endregion
}
}
}
}
foreach (wanggong_goods wg in list_goods)
{
wg.wid = dpid;
//把每个采集到的商品添加到数据库
BLL2.wanggong_goodsManager.Insert(wg);
//dgv1.DataSource = Manager.BindingSort_wanggong_goods(list_goods);
}
dgv1.DataSource = Manager.BindingSort_wanggong_goods(list_goods);
GC.Collect();//释放内存
}
}
catch (Exception ex)
{
Debug.WriteLine("======================================================================");
Debug.WriteLine("网址：" + Browser.urlstr);
Debug.WriteLine(DateTime.Now.ToString());
Debug.WriteLine("全店采集信息失败：" + ex.Message);
MessageBox.Show("全店采集信息失败：" + ex.Message);
}
}
toolStripProgressBar1.Value = page_temp * 24;
bind_dianpu();
}