/// <summary> /// 获取连接信息 /// </summary> /// <param name="sContent"></param> /// <param name="sUrl"></param> /// <param name="lisA"></param> private static void _GetLinks(string sContent, string sUrl, ref Dictionary <string, string> lisA) { Regex re = new Regex(@"<a\s+[^>]*href\s*=\s*[^>]+>[\s\S]*?</a>", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline); Regex re2 = new Regex(@"""|'", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline); MatchCollection mcs = re.Matches(sContent); //foreach (Match mc in mcs) for (int i = mcs.Count - 1; i >= 0; i--) { Match mc = mcs[i]; string strHref = GetLink(mc.Value).Trim(); strHref = strHref.Replace("\\\"", "");//针对JS输出链接 strHref = strHref.Replace("\\\'", ""); string strTemp = RemoveByReg(strHref, @"^http.*/$");//屏蔽以“http”开头“/”结尾的链接地址 if (strTemp.Length < 2) { continue; } //过滤广告或无意义的链接 string strText = CString.ClearTag(GetTextByLink(mc.Value)).Trim(); strTemp = RemoveByReg(strText, Const.sFilter); if (CString.GetLength(strTemp) < 9) { continue; } if (re2.IsMatch(strText)) { continue; } //换上绝对地址 strHref = CText.GetUrlByRelative(sUrl, strHref); if (strHref.Length <= 18)//例如,http://www.163.com = 18 { continue; } //计算#字符出现的位置,移除它后面的内容 //如果是域名地址,就跳过 int charIndex = strHref.IndexOf('#'); if (charIndex > -1) { strHref = strHref.Substring(0, charIndex); } strHref = strHref.Trim(new char[] { '/', '\\' }); string tmpDomainURL = CRegex.GetDomain(strHref); if (strHref.Equals(tmpDomainURL, StringComparison.OrdinalIgnoreCase)) { continue; } if (!lisA.ContainsKey(strHref) && !lisA.ContainsValue(strText)) { lisA.Add(strHref, strText); } } }
/// <summary> /// 获取连接信息 /// </summary> /// <param name="sContent">网页内容</param> /// <param name="sUrl">网页url</param> /// <param name="lisDes">连接列表</param> /// <returns></returns> public static Dictionary <string, string> GetLinks(string sContent, string sUrl, ref Dictionary <string, string> lisDes) { Dictionary <string, string> lisA = new Dictionary <string, string>(); _GetLinks(sContent, sUrl, ref lisA); string domain = CRegex.GetDomain(sUrl).ToLower(); //抓取脚本输出的链接 Regex re = new Regex(@"<script[^>]+src\s*=\s*(?:'(?<src>[^']+)'|""(?<src>[^""]+)""|(?<src>[^>\s]+))\s*[^>]*>", RegexOptions.Multiline | RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase); MatchCollection mcs = re.Matches(sContent); //foreach (Match mc in mcs) for (int i = mcs.Count - 1; i >= 0; i--) { Match mc = mcs[i]; string subUrl = CRegex.GetUrl(sUrl, mc.Groups["src"].Value); if (domain.CompareTo(CRegex.GetDomain(subUrl).ToLower()) != 0) { //同一域的才提炼 continue; } string subContent = CSocket.GetHtmlByUrl(subUrl); if (subContent.Length == 0) { continue; } _GetLinks(subContent, subUrl, ref lisA); } if (lisA.Count == 0) { return(GetLinksFromRss(sContent, sUrl, ref lisDes)); } return(lisA); }