public void Run()
{
List <string> orderList = new List <string>();
string configPath = AppDomain.CurrentDomain.BaseDirectory + "IP代理/Config/TaobaoProxyIpConfig.txt";
StreamReader sr = new StreamReader(configPath);
while (true)
{
string str = sr.ReadLine();
if (str == null)
{
break;
}
else
{
orderList.Add(str);
}
}
sr.Close();
sr.Dispose();
string url = "http://121.199.38.28/ip/?tid={0}&num=1&ports=80,808,3128&filter=on";
string error = "ERROR|订单剩余数量不足";
SysData_ProxyIp existsObj = new SysData_ProxyIp();
string message = "";
foreach (string orderStr in orderList)
{
string requestUrl = string.Format(url, orderStr);
while (true)
{
string ipHtml = "";
begin:
try
{
ipHtml = SpiderHelp.GetHtml(requestUrl, "utf-8");
}
catch (Exception ex)
{
System.Threading.Thread.Sleep(3000);
goto begin;
}
if (ipHtml.Contains(error))
{
break;
}
int result = ProxyIpHelp.ImportProxyIp(ipHtml, "匿名", out existsObj, out message);
if (result != 1)
{
log.Debug(string.Format("{0},订单:{1},url:{2},ip:{3}", message, orderStr, url, ipHtml == null ? "null" : ipHtml));
continue;
}
else
{
log.Debug(string.Format("ip插入成功,订单:{0},url:{1},ip:{2}", orderStr, url, ipHtml == null ? "null" : ipHtml));
}
}
log.Debug(string.Format("订单:{0}导入完成,url:{1}", orderStr, url));
}
log.Debug("所有订单:导入完成,url:{1}");
}
/// <summary>
/// 将当前网站对应的代理ip设置为不可用
/// </summary>
/// <param name="webId"></param>
/// <param name="ip"></param>
public static void SetNotEffectiveProxyIp(int webId, string ip, DataClass _dc = null)
{
if (!string.IsNullOrEmpty(ip))
{
DataClass dc = new DataClass(_dc);
string sql = new StringBuilder().Append("select top 1 * from SysData_WebJoinProxyIp with(nolock) where WebId=")
.Append(webId).Append(" and ProxyIp in (select ID from SysData_ProxyIp with(nolock) where Ip='").Append(ip).Append("')").ToString();
SysData_WebJoinProxyIp obj = dc.DB.ExecuteQuery <SysData_WebJoinProxyIp>(sql).FirstOrDefault();
if (obj == null)
{
SysData_ProxyIp ipObj = GetProxyIpByIp(ip, dc);
if (ipObj != null)
{
obj = new SysData_WebJoinProxyIp {
WebId = webId, ProxyIp = ipObj.ID, CreateTime = DateTime.Now, Status = WebProxyIpStatus2
};
obj = WebJoinProxyIpManager.Insert(obj, dc);
}
}
else
{
obj.Status = WebProxyIpStatus2;
WebJoinProxyIpManager.Update(obj, dc);
}
//dc.DB.SubmitChanges();
dc.Connection_Close();
dc.Dispose();
}
}
public void RunPageList(object url)
{
string _url = Convert.ToString(url);
string pageUrlPara = url + "&pageid={0}";
try
{
SysData_ProxyIp existsObj = new SysData_ProxyIp();
bool notStop = true;
string message = "";
int pageIndex = 1;
while (notStop)
{
string paegUrl = string.Format(pageUrlPara, pageIndex);
int reqCount = 1;
reqBegin:
Dictionary <string, List <string> > dicValueList = SpiderHelp.GetHtmlByRegexNotProxyIp(paegUrl, "gb2312", regexDic);
List <string> ipInfoList = dicValueList["regex_ipInfo"];//所有ip集合
if (ipInfoList == null || ipInfoList.Count < 1)
{
if (reqCount < 3)
{
reqCount = reqCount + 1;
goto reqBegin;
}
else
{
pageIndex = 1;
log.Debug(string.Format("未获取到IP列表,url:{0}", url));
continue;
}
}
foreach (string ipInfo in ipInfoList)
{
Dictionary <string, List <string> > infoListDic = SpiderHelp.GetStrByRegex(ipInfo, regexDic);
string ip = infoListDic["regex_ip"].Count < 1 ? "" : infoListDic["regex_ip"][0];
ip = ip.TrimBlank();
int result = ProxyIpHelp.ImportProxyIp(ip, "匿名", out existsObj, out message);
if (result != 1)
{
log.Debug(string.Format("{0},url:{1},ip:{2}", message, url, ip == null ? "null" : ip));
continue;
}
else
{
log.Debug(string.Format("ip插入成功,url:{0},ip:{1}", url, ip == null ? "null" : ip));
}
}
pageIndex = pageIndex + 1;
}
}
catch (Exception ex)
{
log.Error("系统异常", ex);
}
}
/// <summary>
/// 根据ip获取ip信息
/// </summary>
/// <param name="ip"></param>
/// <param name="_dc"></param>
/// <returns></returns>
public static SysData_ProxyIp GetProxyIpByIp(string ip, DataClass _dc = null)
{
DataClass dc = new DataClass(_dc);
string sql = string.Format("select top 1 * from SysData_ProxyIp with(nolock) where Ip='{0}'", ip);
SysData_ProxyIp existsObj = dc.DB.ExecuteQuery <SysData_ProxyIp>(sql).FirstOrDefault();
dc.Connection_Close();
dc.Dispose();
return(existsObj);
}
public static SysData_ProxyIp Insert(SysData_ProxyIp obj, DataClass _db = null)
{
DataClass db = new DataClass(_db);
if (obj != null)
{
long nowID = 0;
db.DB.SysData_ProxyIp_Insert(obj.Ip, obj.CreateTime, obj.IpArea, out nowID);
obj.ID = nowID;
}
return(obj);
}
/// <summary>
///
/// </summary>
/// <param name="ip"></param>
/// <param name="ipArea">ip所在地区,比如:广东省广州市电信</param>
/// <param name="existsObj">已经存在或者插入成功的数据</param>
/// <param name="checkExists">是否监测数据是否存在</param>
/// <param name="message"></param>
/// <param name="_dc"></param>
/// <returns>1:成功,2:失败-ip已经存在,0:失败-其他失败</returns>
public static int InsertProxyIp(string ip, string ipArea, out SysData_ProxyIp existsObj, out string message, bool checkExists = true, DataClass _dc = null)
{
existsObj = null;
message = "";
ip = ip.TrimBlank();
if (string.IsNullOrEmpty(ip) || string.IsNullOrEmpty(ip.ToLower().Replace("http://", "")))
{
message = "ip不能为空";
return(0);
}
if (string.IsNullOrEmpty(ip.TrimBlank()))
{
message = "ip所在地区不能为空";
return(0);
}
ip = ip.Trim();
DataClass dc = new DataClass(_dc);
ip = ip.ToLower().Replace("http://", "");
//如果需要检测数据是否存在
if (checkExists)
{
existsObj = GetProxyIpByIp(ip, dc);
if (existsObj != null)
{
dc.Connection_Close();
dc.Dispose();
message = "ip已存在";
return(2);
}
}
SysData_ProxyIp ipObj = new SysData_ProxyIp {
CreateTime = DateTime.Now, Ip = ip, IpArea = ipArea
};
ipObj = Insert(ipObj, dc);
//dc.DB.SysData_ProxyIp.InsertOnSubmit(ipObj);
//dc.DB.SubmitChanges();
dc.Connection_Close();
dc.Dispose();
existsObj = ipObj;
return(1);
}
/// <summary>
/// 根据网站获取可用的代理IP
/// </summary>
/// <param name="webId"></param>
/// <param name="_dc"></param>
/// <returns></returns>
public static string GetEffectiveProxyIp(int webId, DataClass _dc = null)
{
//return null;
string nowIp = null;
DataClass dc = new DataClass(_dc);
//View_WebJoinProxyIp obj = dc.DB.ExecuteQuery<View_WebJoinProxyIp>(string.Format("select top 1 * from View_WebJoinProxyIp where WebId={0} and Status={1} order by newId()",webId,WebProxyIpStatus1)).FirstOrDefault();// dc.DB.View_WebJoinProxyIp.Where(tbl => tbl.WebId == webId && tbl.Status == WebProxyIpStatus1).FirstOrDefault();
//SysData_ProxyIp obj = dc.DB.ExecuteQuery<SysData_ProxyIp>("select top 1 * from SysData_ProxyIp order by newId()").FirstOrDefault();// dc.DB.View_WebJoinProxyIp.Where(tbl => tbl.WebId == webId && tbl.Status == WebProxyIpStatus1).FirstOrDefault();
string sql = new StringBuilder().Append("select top 1 * from SysData_ProxyIp as tbl1 with(nolock) where ")
.Append("not exists (select * from SysData_WebJoinProxyIp with(nolock) where tbl1.ID=ProxyIp and WebId=").Append(webId)
.Append(" and [Status]=").Append(WebProxyIpStatus2)
.Append(") order by newId()").ToString();
SysData_ProxyIp obj = dc.DB.ExecuteQuery <SysData_ProxyIp>(sql).FirstOrDefault();
if (obj != null)
{
nowIp = obj.Ip;
}
dc.Connection_Close();
dc.Dispose();
return(nowIp);
}
/// <summary>
/// 导入IP
/// </summary>
/// <param name="ip"></param>
/// <param name="ipArea"></param>
/// <param name="existsObj"></param>
/// <param name="message"></param>
/// <returns></returns>
public static int ImportProxyIp(string ip, string ipArea, out SysData_ProxyIp existsObj, out string message)
{
existsObj = null;
message = "";
if (string.IsNullOrEmpty(ip))
{
message = "ip不能为空";
return(0);
}
work:
if (!WorkItemManager.CheckPassSpider())//****检查数据库是否有维护程序在执行******//
{
System.Threading.Thread.Sleep(60000);
goto work;
}
existsObj = ProxyIpManager.GetProxyIpByIp(ip);
if (existsObj != null)
{
message = "ip已存在";
return(0);
}
if (!CheckProxyIp(ip))
{
message = "ip不可用";
return(0);
}
int result = ProxyIpManager.InsertProxyIp(ip, ipArea, out existsObj, out message, checkExists: false);
if (result != 1)
{
return(0);
}
else
{
message = "ip插入成功";
}
return(1);
}
public void Run()
{
//ip列表页ip信息
RegexInfo regex_ipInfo = new RegexInfo("([^<>]+\\@[^<>]+</font>)", "$1");
//ip列表页ip
RegexInfo regex_ip = new RegexInfo("([^<>]+)\\@[^<>]+</font>", "$1");
//ip列表页ip区域
RegexInfo regex_area = new RegexInfo("[^<>]+\\#([^<>]+)</font>", "$1");
Dictionary <string, RegexInfo> regexDic1 = new Dictionary <string, RegexInfo>();
regexDic1.Add("regex_ipInfo", regex_ipInfo);
Dictionary <string, RegexInfo> regexDic2 = new Dictionary <string, RegexInfo>();
regexDic2.Add("regex_ip", regex_ip);
regexDic2.Add("regex_area", regex_area);
//从文本文件中获取爬取ip分类页面列表
List <string> urlList = new List <string>();
string configPath = AppDomain.CurrentDomain.BaseDirectory + "IP代理/Config/Lastro网IP代理ProxyIpConfig.txt";
StreamReader sr = new StreamReader(configPath);
while (true)
{
string str = sr.ReadLine();
if (str == null)
{
break;
}
else
{
urlList.Add(str);
}
}
sr.Close();
sr.Dispose();
//但从当前页面中获取爬取ip分类列表
RegexInfo regex_urllist = new RegexInfo("<a href=\"([^\"]+)\"[^<>]*>【国内代理】[^<>]+</a>", "$1");
RegexInfo regex_urllist_pagecount = new RegexInfo("<span title=\"共[^\"]+页\"> /([^<>]+)页", "$1");
Dictionary <string, RegexInfo> regexDic3 = new Dictionary <string, RegexInfo>();
regexDic3.Add("regex_urllist", regex_urllist);
regexDic3.Add("regex_urllist_pagecount", regex_urllist_pagecount);
int urllist_index = 1;
int urllist_max_index = 1;
string urllist_nextPage_para = "http://www.httpip.net/forum-36-{0}.html"; //下一页
string urllist_nextPage = string.Format(urllist_nextPage_para, urllist_index); //下一页
try
{
begin_nextpage:
Dictionary <string, List <string> > dicValueList3 = SpiderHelp.GetHtmlByRegexNotProxyIp(urllist_nextPage, "gb2312", regexDic3);
List <string> urlList2 = dicValueList3["regex_urllist"];//所有ip列表集合
urlList.AddRange(urlList2);
urllist_max_index = dicValueList3["regex_urllist_pagecount"].Count < 1 ? 0 : Convert.ToInt32(dicValueList3["regex_urllist_pagecount"][0].TrimBlank());
if (urllist_index < urllist_max_index)
{
urllist_index = urllist_index + 1;
urllist_nextPage = string.Format(urllist_nextPage_para, urllist_index);
}
else
{
urllist_nextPage = "";
}
//开始爬取当前页列表ip页面
SysData_ProxyIp existsObj = new SysData_ProxyIp();
string message = "";
for (int i = 0; i < urlList.Count(); i++)
{
string urlInfo = urlList[i];
string url = urlInfo.Split('$')[0];
string urlHost = urlInfo.Split('$').Length < 2 ? "http://www.httpip.net/" : urlInfo.Split('$')[1];
requestpage:
if (!url.ToLower().Contains("http://"))
{
url = urlHost + url;
}
Dictionary <string, List <string> > dicValueList = SpiderHelp.GetHtmlByRegexNotProxyIp(url, "gb2312", regexDic1);
List <string> ipInfoList = dicValueList["regex_ipInfo"]; //所有ip集合
string nextPage = ""; //下一页链接
if (ipInfoList == null || ipInfoList.Count < 1)
{
log.Debug(string.Format("未获取到IP列表,url:{0}", url));
continue;
}
foreach (string ipInfo in ipInfoList)
{
Dictionary <string, List <string> > infoListDic = SpiderHelp.GetStrByRegex(ipInfo, regexDic2);
string ip = infoListDic["regex_ip"].Count < 1 ? "" : infoListDic["regex_ip"][0];
string ipArea = infoListDic["regex_area"].Count < 1 ? "" : infoListDic["regex_area"][0];
ip = ip.TrimBlank();
int result = ProxyIpHelp.ImportProxyIp(ip, ipArea, out existsObj, out message);
if (result != 1)
{
log.Debug(string.Format("{0},url:{1},ip:{2}", message, url, ip == null ? "null" : ip));
continue;
}
else
{
log.Debug(string.Format("ip插入成功,url:{0},ip:{1}", url, ip == null ? "null" : ip));
}
}
if (!string.IsNullOrEmpty(nextPage))
{
if (!nextPage.ToLower().Contains("http://"))
{
url = urlHost + nextPage;
}
goto requestpage;
}
}
if (!string.IsNullOrEmpty(urllist_nextPage))
{
urlList = new List <string>();
goto begin_nextpage;
}
}
catch (Exception ex)
{
log.Error("系统异常", ex);
}
}
/// <summary>
/// 给网站插入代理IP(如果ip在ip表里存在则读取后插入到网站ip表,否则新增ip)
/// </summary>
/// <param name="ip"></param>
/// <param name="ipArea">ip所在地区,比如:广东省广州市电信</param>
/// <param name="webIds"></param>
/// <param name="addList">新增成功的网站代理ip信息</param>
/// <param name="message"></param>
/// <param name="_dc"></param>
/// <returns></returns>
public static int InsertWebJoinProxyIp(string ip, string ipArea, int[] webIds, out List <SysData_WebJoinProxyIp> addList, out string message, DataClass _dc = null)
{
message = "";
addList = new List <SysData_WebJoinProxyIp>();
if (string.IsNullOrEmpty(ip) || webIds == null || webIds.Length < 1)
{
message = "请输入ip和网站";
return(0);
}
DataClass dc = new DataClass(_dc);
try
{
SysData_ProxyIp ipObj = null;
int result = ProxyIpManager.InsertProxyIp(ip, ipArea, out ipObj, out message, _dc: dc);
if (ipObj == null)
{
dc.Connection_Close();
dc.Dispose();
message = "ip不能为空";
return(0);
}
List <SysData_WebJoinProxyIp> list = dc.DB.SysData_WebJoinProxyIp.Where(tbl => tbl.ProxyIp == ipObj.ID && webIds.Contains(tbl.WebId)).ToList();
if (list.Count == webIds.Length)
{
dc.Connection_Close();
dc.Dispose();
message = "此网站代理IP已存在";
return(0);
}
foreach (int webId in webIds)
{
if (list.Where(p => p.WebId == webId).FirstOrDefault() == null)
{
SysData_WebJoinProxyIp wipObj = new SysData_WebJoinProxyIp();
wipObj.ProxyIp = ipObj.ID;
wipObj.WebId = webId;
wipObj.Status = ProxyIpManager.WebProxyIpStatus1;
wipObj.CreateTime = DateTime.Now;
addList.Add(wipObj);
}
}
if (addList != null && addList.Count > 0)
{
dc.DB.SysData_WebJoinProxyIp.InsertAllOnSubmit <SysData_WebJoinProxyIp>(addList);
dc.DB.SubmitChanges();
}
dc.Connection_Close();
dc.Dispose();
}
catch (Exception ex)
{
dc.Connection_Close();
dc.Dispose();
message = "系统异常";
log.Error(string.Format("(插入网站代理ip失败)InsertWebJoinProxyIp(string ip={0}, int[] webIds={1},out string message, DataClass _dc = null)"
, ip == null ? "null" : null, webIds == null ? 0 : webIds.Length),
ex
);
return(0);
}
return(1);
}
public void Run()
{
//ip列表页ip信息
RegexInfo regex_ipInfo = new RegexInfo("(\\ \\;[^<>]+<br />)", "$1");
//ip列表页ip
RegexInfo regex_ip = new RegexInfo("\\ \\;([^<>\\s]+) ([^<>\\s]+) [^<>]+<br />", "$1:$2");
//ip列表页ip区域
RegexInfo regex_area = new RegexInfo("\\ \\;[^<>\\s]+ [^<>\\s]+ ([^<>\\s]+) [^<>]+<br />", "$1");
Dictionary <string, RegexInfo> regexDic1 = new Dictionary <string, RegexInfo>();
regexDic1.Add("regex_ipInfo", regex_ipInfo);
Dictionary <string, RegexInfo> regexDic2 = new Dictionary <string, RegexInfo>();
regexDic2.Add("regex_ip", regex_ip);
regexDic2.Add("regex_area", regex_area);
//从文本文件中获取爬取ip分类页面列表
List <string> urlList = new List <string>();
string configPath = AppDomain.CurrentDomain.BaseDirectory + "IP代理/Config/猫扑网IP代理ProxyIpConfig.txt";
StreamReader sr = new StreamReader(configPath);
while (true)
{
string str = sr.ReadLine();
if (str == null)
{
break;
}
else
{
urlList.Add(str);
}
}
sr.Close();
sr.Dispose();
//但从当前页面中获取爬取ip分类列表
RegexInfo regex_urllist = new RegexInfo("<DT><a href=\"([^\"]+)\" target=\"_blank\">[^<>]+</a></DT>", "$1");
RegexInfo regex_urllist_pagecount = new RegexInfo("<a href=\"http://www.itmop.com/proxy/catalog.asp\\?page=(\\d+)\">\\»\\;</a></DIV>", "$1");
Dictionary <string, RegexInfo> regexDic3 = new Dictionary <string, RegexInfo>();
regexDic3.Add("regex_urllist", regex_urllist);
regexDic3.Add("regex_urllist_pagecount", regex_urllist_pagecount);
int urllist_index = 1;
int urllist_max_index = 1;
string urllist_nextPage_para = "http://www.itmop.com/proxy/catalog.asp?page={0}"; //下一页
string urllist_nextPage = string.Format(urllist_nextPage_para, urllist_index); //下一页
try
{
begin_nextpage:
//获取ip列表页url
Dictionary <string, List <string> > dicValueList3 = SpiderHelp.GetHtmlByRegexNotProxyIp(urllist_nextPage, "utf-8", regexDic3);
List <string> urlList2 = dicValueList3["regex_urllist"];//所有ip列表集合
urlList.AddRange(urlList2);
//获取ip列表列表页的最大页数
urllist_max_index = dicValueList3["regex_urllist_pagecount"].Count < 1 ? 0 : Convert.ToInt32(dicValueList3["regex_urllist_pagecount"][0].TrimBlank());
if (urllist_index < urllist_max_index)
{
urllist_index = urllist_index + 1;
urllist_nextPage = string.Format(urllist_nextPage_para, urllist_index);
}
else
{
urllist_nextPage = "";
}
//开始爬取当前页列表ip页面
SysData_ProxyIp existsObj = new SysData_ProxyIp();
string message = "";
for (int i = 0; i < urlList.Count(); i++)
{
string urlInfo = urlList[i];
urlInfo = urlInfo.Replace("&", "&");
string url = urlInfo.Split('$')[0];
string urlHost = urlInfo.Split('$').Length < 2 ? "http://www.itmop.com" : urlInfo.Split('$')[1];
requestpage:
//根据ip列表页url爬取ip信息
if (!url.ToLower().Contains("http://"))
{
url = urlHost + url;
}
Dictionary <string, List <string> > dicValueList = SpiderHelp.GetHtmlByRegexNotProxyIp(url, "utf-8", regexDic1);
List <string> ipInfoList = dicValueList["regex_ipInfo"];//所有ip集合
string nextPage = "";
if (ipInfoList == null || ipInfoList.Count < 1)
{
log.Debug(string.Format("未获取到IP列表,url:{0}", url));
continue;
}
foreach (string ipInfo in ipInfoList)
{
Dictionary <string, List <string> > infoListDic = SpiderHelp.GetStrByRegex(ipInfo, regexDic2);
string ip = infoListDic["regex_ip"].Count < 1 ? "" : infoListDic["regex_ip"][0];
string ipArea = infoListDic["regex_area"].Count < 1 ? "" : infoListDic["regex_area"][0];
ipArea = ipArea.RemoveHeml();
ip = ip.TrimBlank();
int result = ProxyIpHelp.ImportProxyIp(ip, ipArea, out existsObj, out message);
if (result != 1)
{
log.Debug(string.Format("{0},url:{1},ip:{2}", message, url, ip == null ? "null" : ip));
continue;
}
else
{
log.Debug(string.Format("ip插入成功,url:{0},ip:{1}", url, ip == null ? "null" : ip));
}
}
if (!string.IsNullOrEmpty(nextPage))
{
url = urlHost + nextPage;
goto requestpage;
}
}
if (!string.IsNullOrEmpty(urllist_nextPage))
{
urlList = new List <string>();
goto begin_nextpage;
}
}
catch (Exception ex)
{
log.Error("系统异常", ex);
}
}
public void ImportIp(object param)
{
string fileName = Convert.ToString(param);
int count = 0;//成功个数
//禁用其他按钮
panel查询条件.Enabled = false;
btnExcel2.Enabled = false;
btnImportIp.Visible = false;
//进度条位置设置
panelImport.Visible = true;
panelImport.Top = btnImportIp.Top;
panelImport.Left = btnImportIp.Left;
labelImportBar.Text = "0%";
progressBarImportIp.Value = 0;
SysData_ProxyIp existsObj = new SysData_ProxyIp();
string message = "";
object missing = System.Reflection.Missing.Value;
Excel.Application excel = new Excel.Application();
if (excel == null)
{
MessageBox.Show("Can't access excel");
}
else
{
excel.Visible = false; excel.UserControl = true;
// 以只读的形式打开EXCEL文件
Excel.Workbook wb = excel.Application.Workbooks.Open(fileName, missing, true, missing, missing, missing,
missing, missing, missing, true, missing, missing, missing, missing, missing);
//取得第一个工作薄
Excel.Worksheet ws = (Excel.Worksheet)wb.Worksheets.get_Item(1);
//取得总记录行数
int rowsint = ws.UsedRange.Cells.Rows.Count; //得到行数
//设置进度条范围
progressBarImportIp.Maximum = rowsint;
progressBarImportIp.Minimum = 0;
//取得数据范围区域 (从第一行第一列到最后一行第二列)
Excel.Range rng1 = ws.Cells.get_Range("A1", "B" + rowsint); //item
object[,] arryItem = (object[, ])rng1.Value2;
for (int i = 1; i <= rowsint - 1; i++)
{
string ip = arryItem[i, 1].ToString();
string ipArea = arryItem[i, 2].ToString();
int result = ProxyIpHelp.ImportProxyIp(ip, ipArea, out existsObj, out message);
if (result == 1)
{
count = count + 1;
}
progressBarImportIp.Value = progressBarImportIp.Value + 1;
labelImportBar.Text = 计算百分比(progressBarImportIp.Value, progressBarImportIp.Maximum);
}
progressBarImportIp.Value = progressBarImportIp.Maximum;
labelImportBar.Text = 计算百分比(progressBarImportIp.Value, progressBarImportIp.Maximum);
}
excel.Quit(); excel = null;
Process[] procs = Process.GetProcessesByName("excel");
foreach (Process pro in procs)
{
pro.Kill();//没有更好的方法,只有杀掉进程
}
GC.Collect();
MessageBox.Show(string.Format("导入完成,成功导入可用的代理IP{0}个", count));
//进度条设置
panelImport.Visible = false;
btnExcel2.Enabled = true;
btnImportIp.Visible = true;
panel查询条件.Enabled = true;
labelImportBar.Text = "";
}
public void Run()
{
RegexInfo regex_ipInfo = new RegexInfo("(<tr class=\"[^\"]*\">(?:(?!</tr>).)*</tr>)", "$1");
RegexInfo regex_nextPage = new RegexInfo("<a class=\"next_page\" rel=\"next\" href=\"([^\"]+)\">下一页[^<>]*</a>", "$1");
RegexInfo regex_ip = new RegexInfo("<tr class=\"[^\"]*\"><td>(?:(?!</td>).)*</td><td>([^<>]+)</td><td>([^<>]+)</td><td><a href=\"[^\"]*\">[^<>]+</a></td>" +
"<td>[^<>]*</td><td>[^<>]*</td><td>(?:(?!</td>).)*</td><td>(?:(?!</td>).)*</td><td>[^<>]+</td>", "$1:$2");
RegexInfo regex_area = new RegexInfo("<tr class=\"[^\"]*\"><td>(?:(?!</td>).)*</td><td>[^<>]+</td><td>[^<>]+</td><td><a href=\"[^\"]*\">([^<>]+)</a></td>" +
"<td>[^<>]*</td><td>[^<>]*</td><td>(?:(?!</td>).)*</td><td>(?:(?!</td>).)*</td><td>[^<>]+</td>", "$1");
RegexInfo regex_date = new RegexInfo("<tr class=\"[^\"]*\"><td>(?:(?!</td>).)*</td><td>[^<>]+</td><td>[^<>]+</td><td><a href=\"[^\"]*\">[^<>]+</a></td>" +
"<td>[^<>]*</td><td>[^<>]*</td><td>(?:(?!</td>).)*</td><td>(?:(?!</td>).)*</td><td>([^<>]+)</td>", "$1");
Dictionary <string, RegexInfo> regexDic1 = new Dictionary <string, RegexInfo>();
regexDic1.Add("regex_ipInfo", regex_ipInfo);
regexDic1.Add("regex_nextPage", regex_nextPage);
Dictionary <string, RegexInfo> regexDic2 = new Dictionary <string, RegexInfo>();
regexDic2.Add("regex_ip", regex_ip);
regexDic2.Add("regex_area", regex_area);
regexDic2.Add("regex_date", regex_date);
//从文本文件中获取爬取配置
List <string> urlList = new List <string>();
string configPath = AppDomain.CurrentDomain.BaseDirectory + "IP代理/Config/西刺ProxyIpConfig.txt";
StreamReader sr = new StreamReader(configPath);
while (true)
{
string str = sr.ReadLine();
if (str == null)
{
break;
}
else
{
urlList.Add(str);
}
}
sr.Close();
sr.Dispose();
//urlList.Add("http://www.xici.net.co/nt/$http://www.xici.net.co/$2014-5-13");
try
{
SysData_ProxyIp existsObj = new SysData_ProxyIp();
string message = "";
foreach (string urlInfo in urlList)
{
string url = urlInfo.Split('$')[0];
string urlHost = urlInfo.Split('$')[1];
DateTime maxDate = Convert.ToDateTime(urlInfo.Split('$')[2]);
requestpage:
Dictionary <string, List <string> > dicValueList = SpiderHelp.GetHtmlByRegexNotProxyIp(url, "utf-8", regexDic1);
List <string> ipInfoList = dicValueList["regex_ipInfo"]; //所有ip集合
string nextPage = dicValueList["regex_nextPage"].Count < 1 ? "" : dicValueList["regex_nextPage"][0]; //下一页链接
if (ipInfoList == null || ipInfoList.Count < 1)
{
log.Debug(string.Format("未获取到IP列表,url:{0}", url));
continue;
}
foreach (string ipInfo in ipInfoList)
{
Dictionary <string, List <string> > infoListDic = SpiderHelp.GetStrByRegex(ipInfo, regexDic2);
string ip = infoListDic["regex_ip"].Count < 1 ? "" : infoListDic["regex_ip"][0];
string ipArea = infoListDic["regex_area"].Count < 1 ? "" : infoListDic["regex_area"][0];
DateTime ipDate = infoListDic["regex_date"].Count < 1 ? DateTime.Now : Convert.ToDateTime(infoListDic["regex_date"][0]);
ip = ip.TrimBlank();
int result = ProxyIpHelp.ImportProxyIp(ip, ipArea, out existsObj, out message);
if (result != 1)
{
log.Debug(string.Format("{0},url:{1},ip:{2}", message, url, ip == null ? "null" : ip));
continue;
}
else
{
log.Debug(string.Format("ip插入成功,url:{0},ip:{1}", url, ip == null ? "null" : ip));
}
//如果自定日期
if (ipDate < maxDate)
{
nextPage = null;
break;
}
}
if (!string.IsNullOrEmpty(nextPage))
{
url = urlHost + nextPage;
goto requestpage;
}
}
}
catch (Exception ex)
{
log.Error("系统异常", ex);
}
}