//这是一个通讯的接口方法,不做采集规则的处理,所有需要采集的网页均调用此防范 //由此方法调用cGatherWeb.GetGatherData,做次方法的目的是为了可以处理错误重试 private DataTable GetGatherData(string Url, cGlobalParas.WebCode webCode, string cookie, string startPos, string endPos, string sPath, bool IsAjax) { cGatherWeb gWeb = new cGatherWeb(); gWeb.CutFlag = m_TaskSplitData.CutFlag; DataTable tmpData; int AgainTime = 0; GatherAgain: try { tmpData = gWeb.GetGatherData(Url, m_WebCode, m_Cookie, m_gStartPos, m_gEndPos, m_SavePath, IsAjax); } catch (System.Exception ex) { AgainTime++; if (AgainTime > m_AgainNumber) { if (m_IsErrorLog == true) { //保存出错日志 } throw ex; } else { if (m_Ignore404 == true && ex.Message.Contains("404")) { if (m_IsErrorLog == true) { //保存出错日志 } throw ex; } else { e_Log(this, new cGatherTaskLogArgs(m_TaskID, ((int)cGlobalParas.LogType.Error).ToString() + "网址:" + Url + "访问发生错,错误信息:" + ex.Message + ",等待3秒重试\n", this.IsErrorLog)); Thread.Sleep(3000); e_Log(this, new cGatherTaskLogArgs(m_TaskID, ((int)cGlobalParas.LogType.Warning).ToString() + Url + "正在进行第" + AgainTime + "次重试\n", this.IsErrorLog)); //返回重试 goto GatherAgain; } } } return(tmpData); }
//对Url中文部分进行编码,返回编码后的Url, //注意:只对中文进行编码 static public string UrlEncode(string Url, cGlobalParas.WebCode uEncoding) { string DemoUrl = Url; Regex re = new Regex("[\\u4e00-\\u9fa5]", RegexOptions.None); MatchCollection mc = re.Matches(DemoUrl); switch (uEncoding) { case cGlobalParas.WebCode.utf8: foreach (Match ma in mc) { DemoUrl = DemoUrl.Replace(ma.Value.ToString(), HttpUtility.UrlEncode(ma.Value.ToString(), Encoding.UTF8)); } break; case cGlobalParas.WebCode.gb2312: foreach (Match ma in mc) { DemoUrl = DemoUrl.Replace(ma.Value.ToString(), HttpUtility.UrlEncode(ma.Value.ToString(), Encoding.GetEncoding("gb2312"))); } break; case cGlobalParas.WebCode.gbk: foreach (Match ma in mc) { DemoUrl = DemoUrl.Replace(ma.Value.ToString(), HttpUtility.UrlEncode(ma.Value.ToString(), Encoding.GetEncoding("gbk"))); } break; case cGlobalParas.WebCode.big5: foreach (Match ma in mc) { DemoUrl = DemoUrl.Replace(ma.Value.ToString(), HttpUtility.UrlEncode(ma.Value.ToString(), Encoding.GetEncoding("big5"))); } break; default: foreach (Match ma in mc) { DemoUrl = DemoUrl.Replace(ma.Value.ToString(), HttpUtility.UrlEncode(ma.Value.ToString(), Encoding.UTF8)); } break; } return(DemoUrl); }
///根据指定的导航规则进行页面导航,在1.6版本中,增加了多层导航的功能 ///网址导航是属于一对多的关系,即每一级别的导航都是属于一对多(也会是一对一的关系) ///在此无论是几级导航,返回的都是最终的需要采集内容的网址 ///因为是多层导航,所以是属于递归的一种算法 ///解析网址后返回的都是标准网址,不会存在相对网址的情况 public List <string> ParseUrlRule(string Url, List <cNavigRule> nRules, cGlobalParas.WebCode webCode, string cookie) { List <string> pUrls = new List <string>(); List <string> Urls = new List <string>(); pUrls.Add(Url); //第一层导航分解都是从一个单一网址进行,之所以 //选择集合,是为了统一调用接口参数 try { Urls = PUrlRule(pUrls, 1, nRules, webCode, cookie); } catch (System.Exception ex) { //导航失败,无法解析导航规则 return(null); } return(Urls); }
//根据指定网址判断当前页面的编码 static public string GetWebpageCode(string url, cGlobalParas.WebCode WebCode) { string charSet = ""; WebClient myWebClient = new WebClient(); myWebClient.Credentials = CredentialCache.DefaultCredentials; //从资源下载数据并返回字节数组。(加@是因为网址中间有"/"符号) byte[] myDataBuffer = myWebClient.DownloadData(url); string strWebData = Encoding.Default.GetString(myDataBuffer); //获取网页字符编码描述信息 Match charSetMatch = Regex.Match(strWebData, "<meta([^<]*)charset=([^<]*)\"", RegexOptions.IgnoreCase | RegexOptions.Multiline); string webCharSet = charSetMatch.Groups[2].Value; if (charSet == null || charSet == "") { charSet = webCharSet; } return(charSet); }
/// <summary> /// 获取指定网址源码 /// </summary> /// <param name="url">网址</param> /// <param name="webCode">网页编码</param> /// <param name="cookie">网页cookie</param> /// <param name="startPos">获取网页源码的起始位置</param> /// <param name="endPos">获取网页源码的终止位置</param> /// <param name="IsCutnr">是否截取回车换行符,默认为true,截取</param> /// <returns></returns> public string GetHtml(string url, cGlobalParas.WebCode webCode, string cookie, string startPos, string endPos, bool IsCutnr, bool IsAjax) { //判断网页编码 Encoding wCode; string PostPara = ""; CookieContainer CookieCon = new CookieContainer(); HttpWebRequest wReq; if (Regex.IsMatch(url, @"<POST>.*</POST>", RegexOptions.IgnoreCase)) { wReq = (HttpWebRequest)WebRequest.Create(@url.Substring(0, url.IndexOf("<POST>"))); } else { wReq = (HttpWebRequest)WebRequest.Create(@url); } wReq.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.0; .NET CLR 1.1.4322; .NET CLR 2.0.50215;)"; //wReq.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5"); //wReq.Headers.Add("Accept-Encoding", "gzip, deflate"); //wReq.SendChunked = true; //wReq.TransferEncoding = "utf-8"; Match a = Regex.Match(url, @"(http://).[^/]*[?=/]", RegexOptions.IgnoreCase); string url1 = a.Groups[0].Value.ToString(); wReq.Referer = url1; //判断是否有cookie if (cookie != "") { CookieCollection cl = new CookieCollection(); //foreach (string sc in cookie.Split(';')) //{ // string ss = sc.Trim(); // cl.Add(new Cookie(ss.Split('=')[0].Trim(), ss.Split('=')[1].Trim(), "/")); //} foreach (string sc in cookie.Split(';')) { string ss = sc.Trim(); if (ss.IndexOf("&") > 0) { foreach (string s1 in ss.Split('&')) { string s2 = s1.Trim(); string s4 = s2.Substring(s2.IndexOf("=") + 1, s2.Length - s2.IndexOf("=") - 1); cl.Add(new Cookie(s2.Split('=')[0].Trim(), s4, "/")); } } else { string s3 = sc.Trim(); cl.Add(new Cookie(s3.Split('=')[0].Trim(), s3.Split('=')[1].Trim(), "/")); } } CookieCon.Add(new Uri(url), cl); wReq.CookieContainer = CookieCon; } //判断是否含有POST参数 if (Regex.IsMatch(url, @"(?<=<POST>)[\S\s]*(?=</POST>)", RegexOptions.IgnoreCase)) { Match s = Regex.Match(url, @"(?<=<POST>).*(?=</POST>)", RegexOptions.IgnoreCase); PostPara = s.Groups[0].Value.ToString(); byte[] pPara = Encoding.ASCII.GetBytes(PostPara); wReq.ContentType = "application/x-www-form-urlencoded"; wReq.ContentLength = pPara.Length; wReq.Method = "POST"; System.IO.Stream reqStream = wReq.GetRequestStream(); reqStream.Write(pPara, 0, pPara.Length); reqStream.Close(); } else { wReq.Method = "GET"; } //设置页面超时时间为12秒 wReq.Timeout = 12000; HttpWebResponse wResp = (HttpWebResponse)wReq.GetResponse(); System.IO.Stream respStream = wResp.GetResponseStream(); string strWebData = ""; switch (webCode) { case cGlobalParas.WebCode.auto: try { wCode = Encoding.Default; string cType = wResp.ContentType.ToLower(); Match charSetMatch = Regex.Match(cType, "(?<=charset=)([^<]*)*", RegexOptions.IgnoreCase | RegexOptions.Multiline); string webCharSet = charSetMatch.ToString(); wCode = System.Text.Encoding.GetEncoding(webCharSet); } catch { wCode = Encoding.Default; } break; case cGlobalParas.WebCode.gb2312: wCode = Encoding.GetEncoding("gb2312"); break; case cGlobalParas.WebCode.gbk: wCode = Encoding.GetEncoding("gbk"); break; case cGlobalParas.WebCode.utf8: wCode = Encoding.UTF8; break; default: wCode = Encoding.UTF8; break; } if (wResp.ContentEncoding == "gzip") { GZipStream myGZip = new GZipStream(respStream, CompressionMode.Decompress); System.IO.StreamReader reader; reader = new System.IO.StreamReader(myGZip, wCode); strWebData = reader.ReadToEnd(); reader.Close(); reader.Dispose(); } else { System.IO.StreamReader reader; reader = new System.IO.StreamReader(respStream, wCode); strWebData = reader.ReadToEnd(); reader.Close(); reader.Dispose(); } //去除回车换行符号 if (IsCutnr == true) { strWebData = Regex.Replace(strWebData, "([\\r\\n])[\\s]+", "", RegexOptions.IgnoreCase | RegexOptions.Multiline); strWebData = Regex.Replace(strWebData, "\\n", "", RegexOptions.IgnoreCase | RegexOptions.Multiline); strWebData.Replace("\\r\\n", ""); } //获取此页面的编码格式,并对源码进行一次判断,无论用户是否指定了网页代码 //Match charSetMatch = Regex.Match(strWebData, "<meta([^<]*)charset=([^<]*)\"", RegexOptions.IgnoreCase | RegexOptions.Multiline); //string webCharSet = charSetMatch.Groups[2].Value; //string charSet = webCharSet; //if (charSet != null && charSet != "" && Encoding.GetEncoding(charSet) != wCode) //{ // byte[] myDataBuffer; // myDataBuffer = System.Text.Encoding.GetEncoding(charSet).GetBytes(strWebData); // strWebData = Encoding.GetEncoding(charSet).GetString(myDataBuffer); //} //按照截取网页的起始标志和终止标志进行截取 //如果起始或终止截取标识有一个为空,则不进行截取 if (startPos != "" && endPos != "") { string Splitstr = "(" + startPos + ").*?(" + endPos + ")"; Match aa = Regex.Match(strWebData, Splitstr); strWebData = aa.Groups[0].ToString(); } if (IsAjax == true) { strWebData = System.Web.HttpUtility.UrlDecode(strWebData, Encoding.UTF8); } this.m_WebpageSource = strWebData; return(strWebData); }
/// <summary> /// 采集网页数据 /// </summary> /// <param name="Url">网页地址</param> /// <param name="StartPos">起始采集位置</param> /// <param name="EndPos">终止采集位置</param> /// <returns></returns> public DataTable GetGatherData(string Url, cGlobalParas.WebCode webCode, string cookie, string startPos, string endPos, string sPath, bool IsAjax) { tempData = new DataTable("tempData"); int i; int j; string strCut = ""; bool IsDownloadFile = false; #region 构建表结构,并构建截取正则 //根据页面截取的标志构建表结构 for (i = 0; i < this.CutFlag.Count; i++) { tempData.Columns.Add(new DataColumn(this.CutFlag[i].Title, typeof(string))); if (this.CutFlag[i].DataType != (int)cGlobalParas.GDataType.Txt && IsDownloadFile == false) { IsDownloadFile = true; } } //根据用户指定的页面截取位置构造正则表达式 for (i = 0; i < this.CutFlag.Count; i++) { strCut += "(?<" + this.CutFlag[i].Title + ">" + cTool.RegexReplaceTrans(this.CutFlag[i].StartPos) + ")"; //strCut += "(?<=" + cTool.RegexReplaceTrans(this.CutFlag[i].StartPos) + ")"; switch (this.CutFlag[i].LimitSign) { case (int)cGlobalParas.LimitSign.NoLimit: strCut += ".*?"; break; case (int)cGlobalParas.LimitSign.NoWebSign: strCut += "[^<>]*?"; break; case (int)cGlobalParas.LimitSign.OnlyCN: strCut += "[\\u4e00-\\u9fa5]*?"; break; case (int)cGlobalParas.LimitSign.OnlyDoubleByte: strCut += "[^\\x00-\\xff]*?"; break; case (int)cGlobalParas.LimitSign.OnlyNumber: strCut += "[\\d]*?"; break; case (int)cGlobalParas.LimitSign.OnlyChar: strCut += "[\\x00-\\xff]*?"; break; case (int)cGlobalParas.LimitSign.Custom: //strCut += cTool.RegexReplaceTrans(this.CutFlag[i].RegionExpression.ToString()); strCut += this.CutFlag[i].RegionExpression.ToString(); break; default: strCut += "[\\S\\s]*?"; break; } strCut += "(?=" + cTool.RegexReplaceTrans(this.CutFlag[i].EndPos) + ")|"; } #endregion #region 获取网页源码 int rowCount = this.CutFlag.Count; //去掉最后一个“|” strCut = strCut.Substring(0, strCut.Length - 1); //获取网页信息 //判断传入的Url是否正确,如果不正确,则返回空数据 if (Regex.IsMatch(Url, "[\"\\s]")) { Match aa = Regex.Match(Url, "[\"\\s]"); tempData = null; return(tempData); } try { GetHtml(Url, webCode, cookie, startPos, endPos, true, IsAjax); } catch (System.Web.HttpException ex) { throw ex; } #endregion //开始获取截取内容 Regex re = new Regex(@strCut, RegexOptions.IgnoreCase | RegexOptions.Multiline); MatchCollection mc = re.Matches(this.WebpageSource); if (mc.Count == 0) { tempData = null; return(tempData); } DataRow drNew = null; i = 0; #region 开始输出截取字符,并拼成一个表 //开始根据采集的数据构造数据表进行输出 //在此需要处理采集数据有可能错行的问题 //下面被注释的代码是最初构建数据表的代码,但会出现错行现象 //Match ma; int rows = 0; //统计共采集了多少行 int m = 0; //计数使用 try { while (m < mc.Count) { //新建新行 drNew = tempData.NewRow(); rows++; for (i = 0; i < this.CutFlag.Count; i++) { if (m < mc.Count) { if (i == 0) { while (!mc[m].Value.StartsWith(this.CutFlag[i].StartPos, StringComparison.CurrentCultureIgnoreCase)) { m++; if (m >= mc.Count) { //退出所有循环 goto ExitWhile; } } drNew[i] = mc[m].Value.Substring(this.CutFlag[i].StartPos.Length, mc[m].Value.Length - this.CutFlag[i].StartPos.Length); m++; } else { if (mc[m].Value.StartsWith(this.CutFlag[i].StartPos, StringComparison.CurrentCultureIgnoreCase)) { drNew[i] = mc[m].Value.Substring(this.CutFlag[i].StartPos.Length, mc[m].Value.Length - this.CutFlag[i].StartPos.Length); m++; } else { if (mc[m].Value.StartsWith(this.CutFlag[i - 1].StartPos, StringComparison.CurrentCultureIgnoreCase)) { m++; i--; } else { if (i < this.CutFlag.Count - 1) { if (mc[m].Value.StartsWith(this.CutFlag[i + 1].StartPos, StringComparison.CurrentCultureIgnoreCase)) { } else { m++; i--; } } else { m++; i--; } //当采集时发生了缺少采集内容,采用此方法进行采集内容补空 //drNew[i] = ""; //continue; } } } } } tempData.Rows.Add(drNew); drNew = null; } } catch (System.Exception ex) { throw ex; } #endregion #region 开始进行输出控制,进行获取数据加工 ExitWhile: //在此判断是否需要在输出时进行数据的限制,根据任务版本1.2增加了数据输出的限制 for (i = 0; i < this.CutFlag.Count; i++) { switch (this.CutFlag[i].ExportLimit) { case (int)cGlobalParas.ExportLimit.ExportNoLimit: break; case (int)cGlobalParas.ExportLimit.ExportNoWebSign: for (int index = 0; index < tempData.Rows.Count; index++) { tempData.Rows[index][i] = getTxt(tempData.Rows[index][i].ToString()); } break; case (int)cGlobalParas.ExportLimit.ExportPrefix: for (int index = 0; index < tempData.Rows.Count; index++) { tempData.Rows[index][i] = this.CutFlag[i].ExportExpression + tempData.Rows[index][i].ToString(); } break; case (int)cGlobalParas.ExportLimit.ExportReplace: for (int index = 0; index < tempData.Rows.Count; index++) { string oStr = this.CutFlag[i].ExportExpression.Substring(1, this.CutFlag[i].ExportExpression.IndexOf(",") - 2); string nStr = this.CutFlag[i].ExportExpression.Substring(this.CutFlag[i].ExportExpression.IndexOf(",") + 2, this.CutFlag[i].ExportExpression.Length - this.CutFlag[i].ExportExpression.IndexOf(",") - 3); tempData.Rows[index][i] = tempData.Rows[index][i].ToString().Replace(oStr, nStr); } break; case (int)cGlobalParas.ExportLimit.ExportSuffix: for (int index = 0; index < tempData.Rows.Count; index++) { tempData.Rows[index][i] = tempData.Rows[index][i].ToString() + this.CutFlag[i].ExportExpression; } break; case (int)cGlobalParas.ExportLimit.ExportTrimLeft: for (int index = 0; index < tempData.Rows.Count; index++) { int len = tempData.Rows[index][i].ToString().Length; int lefti = int.Parse(this.CutFlag[i].ExportExpression.ToString()); if (tempData.Rows[index][i].ToString().Length > lefti) { tempData.Rows[index][i] = tempData.Rows[index][i].ToString().Substring(lefti, len - lefti); } } break; case (int)cGlobalParas.ExportLimit.ExportTrimRight: for (int index = 0; index < tempData.Rows.Count; index++) { int len = tempData.Rows[index][i].ToString().Length; int righti = int.Parse(this.CutFlag[i].ExportExpression.ToString()); if (tempData.Rows[index][i].ToString().Length > righti) { tempData.Rows[index][i] = tempData.Rows[index][i].ToString().Substring(0, len - righti); } } break; case (int)cGlobalParas.ExportLimit.ExportTrim: for (int index = 0; index < tempData.Rows.Count; index++) { tempData.Rows[index][i] = tempData.Rows[index][i].ToString().Trim(); } break; case (int)cGlobalParas.ExportLimit.ExportRegexReplace: for (int index = 0; index < tempData.Rows.Count; index++) { //string oStr=cTool.RegexReplaceTrans( this.CutFlag[i].ExportExpression .Substring (1,this.CutFlag[i].ExportExpression.IndexOf (",")-2)); //string nStr = this.CutFlag[i].ExportExpression.Substring(this.CutFlag[i].ExportExpression.IndexOf(",") + 2, this.CutFlag[i].ExportExpression.Length - this.CutFlag[i].ExportExpression.IndexOf(",") - 3); //tempData.Rows[index][i] = Regex.Replace(tempData.Rows[index][i].ToString(), oStr, nStr, RegexOptions.IgnoreCase | RegexOptions.Multiline); string oStr = this.CutFlag[i].ExportExpression.Substring(1, this.CutFlag[i].ExportExpression.IndexOf(",") - 2); string nStr = this.CutFlag[i].ExportExpression.Substring(this.CutFlag[i].ExportExpression.IndexOf(",") + 2, this.CutFlag[i].ExportExpression.Length - this.CutFlag[i].ExportExpression.IndexOf(",") - 3); tempData.Rows[index][i] = Regex.Replace(tempData.Rows[index][i].ToString(), oStr, nStr, RegexOptions.IgnoreCase | RegexOptions.Multiline); } break; default: break; } } #endregion #region 针对采集需要下载文件的字段进行文件下载处理 //判断是否存在有下载文件的任务,如果有,则开始下载,因为此功能设计最初是下载图片使用 //并非是专用的下载工具,所以对下载处理并没有单独进行线程处理 try { if (IsDownloadFile == true) { if (sPath == "") { sPath = Program.getPrjPath() + "data\\tem_file"; } if (!Directory.Exists(sPath)) { Directory.CreateDirectory(sPath); } string FileUrl = ""; string DownloadFileName = ""; for (i = 0; i < rows; i++) { for (j = 0; j < this.CutFlag.Count; j++) { if (this.CutFlag[j].DataType != (int)cGlobalParas.GDataType.Txt) { FileUrl = tempData.Rows[i][j].ToString(); //开始获取下载文件名称 Regex s = new Regex(@"(?<=/)[^/]*", RegexOptions.IgnoreCase | RegexOptions.Multiline); MatchCollection urlstr = s.Matches(FileUrl); if (urlstr.Count == 0) { DownloadFileName = FileUrl; } else { DownloadFileName = urlstr[urlstr.Count - 1].ToString(); } DownloadFileName = sPath + "\\" + DownloadFileName; if (string.Compare(FileUrl.Substring(0, 4), "http", true) == 0) { DownloadFile(FileUrl, DownloadFileName); } else { if (FileUrl.Substring(0, 1) == "/") { Url = Url.Substring(7, Url.Length - 7); Url = FileUrl.Substring(0, Url.IndexOf("/")); Url = "http://" + Url; FileUrl = Url + FileUrl; } else if (FileUrl.IndexOf("/") <= 0) { Url = Url.Substring(0, Url.LastIndexOf("/") + 1); FileUrl = Url + FileUrl; } else { Url = Url.Substring(0, Url.LastIndexOf("/") + 1); FileUrl = Url + FileUrl; } DownloadFile(FileUrl, DownloadFileName); } } } } } } catch (System.Exception ex) { throw ex; } #endregion return(tempData); }
///解析导航网页 ///判断是否为最后一个级别,在这里需要注意一个问题,因为有可能 ///存储的级别并不是按照顺序进行的,所以,要根据传入的级别Level进行 ///判断,否则会出现错误,导航网页的解析必须是按照顺序的,否则会 ///无法解析 public List <string> PUrlRule(List <string> pUrl, int Level, List <cNavigRule> nRules, cGlobalParas.WebCode webCode, string cookie) { List <string> tmpUrls; List <string> Urls = new List <string> (); if (nRules.Count == 0) { Urls.Add(pUrl[0].ToString()); return(Urls); } string UrlRule = ""; int i; //根据Level得到需要导航级别的导航规则 for (i = 0; i < nRules.Count; i++) { if (Level == nRules[i].Level) { UrlRule = nRules [i].NavigRule; break; } } for (i = 0; i < pUrl.Count; i++) { tmpUrls = new List <string>(); tmpUrls = GetUrlsByRule(pUrl[i].ToString(), UrlRule, webCode, cookie); if (tmpUrls != null) { Urls.AddRange(tmpUrls); } } //判断是否为最底级的导航,如果是则返回,如果不是则继续导航 if (Level == nRules.Count) { return(Urls); } else { List <string> rUrls = PUrlRule(Urls, Level + 1, nRules, webCode, cookie); return(rUrls); } }
//根据导航规则,获取网页地址,是一个集合 public List <string> GetUrlsByRule(string Url, string UrlRule, cGlobalParas.WebCode webCode, string cookie) { string Url1; List <string> Urls = new List <string> (); if (UrlRule.Trim() == "") { Urls.Add(Url); return(Urls); } //判断网址是否存在参数,如果存在参数则取出第一个可用网址 if (Regex.IsMatch(Url, "{.*}")) { List <string> Urls1 = SplitWebUrl(Url); //,IsUrlEncode ,UrlEncode Url1 = Urls1[0].ToString(); } else { Url1 = Url; } //返回网址的源码,并根据提取规则提取导航的网址 //string UrlSource= cTool.GetHtmlSource(Url1,true ); cGatherWeb gW = new cGatherWeb(); string UrlSource = gW.GetHtml(Url1, webCode, cookie, "", "", true, false); gW = null; if (UrlSource == "") { return(null); } //string Rule=@"(?<=href=[\W])" + cTool.RegexReplaceTrans(UrlRule) + @"(\S[^'"">]*)(?=[\s'""])"; string Rule = ""; if (UrlRule.StartsWith("<Regex:")) { Rule = @"(?<=[href=|src=|open(][\W])"; //处理前缀 string strPre = UrlRule.Substring(UrlRule.IndexOf("<Regex:") + 7, UrlRule.IndexOf(">") - 7); Rule += strPre; //处理中间内容 string cma = @"(?<=<Common:)\S+?(?=>)"; Regex cmas = new Regex(cma, RegexOptions.IgnoreCase | RegexOptions.Multiline); MatchCollection cs = cmas.Matches(UrlRule); foreach (Match ma in cs) { Rule += @"(\S*)" + ma.Value.ToString(); } //处理后缀 if (Regex.IsMatch(UrlRule, "<End:")) { string s = UrlRule.Substring(UrlRule.IndexOf("<End:") + 5, UrlRule.Length - UrlRule.IndexOf("<End:") - 6); Rule += @"(\S*)" + s; } else { Rule += @"(\S[^'"">]*)(?=[\s'""])"; } } else { Rule = @"(?<=[href=|src=|open(][\W])" + cTool.RegexReplaceTrans(UrlRule) + @"(\S[^'"">]*)(?=[\s'""])"; } Regex re = new Regex(Rule, RegexOptions.IgnoreCase | RegexOptions.Multiline); MatchCollection aa = re.Matches(UrlSource); DataTable d1 = new DataTable(); d1.Columns.Add("Name"); foreach (Match ma in aa) { //Urls.Add(ma.Value.ToString()); d1.Rows.Add(ma.Value.ToString()); } //导航时可能会获取重复网址的列表,所以需要去重 //去除重复行 string[] strComuns = new string[d1.Columns.Count]; for (int m = 0; m < d1.Columns.Count; m++) { strComuns[m] = d1.Columns[m].ColumnName; } DataView dv = new DataView(d1); DataTable d2 = dv.ToTable(true, strComuns); for (int i = 0; i < d2.Rows.Count; i++) { if (string.Compare(d2.Rows[i][0].ToString().Substring(0, 4), "http", true) != 0) { string PreUrl = Url; if (d2.Rows[i][0].ToString().Substring(0, 1) == "/") { PreUrl = PreUrl.Substring(7, PreUrl.Length - 7); PreUrl = PreUrl.Substring(0, PreUrl.IndexOf("/")); PreUrl = "http://" + PreUrl; } else { Match a = Regex.Match(PreUrl, ".*/"); PreUrl = a.Groups[0].Value.ToString(); } Urls.Add(PreUrl + d2.Rows[i][0].ToString()); } else { Urls.Add(d2.Rows[i][0].ToString()); } } return(Urls); }