public static Dictionary <string, string> GetLinks(string sContent, string sUrl, ref Dictionary <string, string> lisDes) { Dictionary <string, string> dictionary = new Dictionary <string, string>(); smethod_0(sContent, sUrl, ref dictionary); string str = CRegex.GetDomain(sUrl).ToLower(); MatchCollection matchs = new Regex("<script[^>]+src\\s*=\\s*(?:'(?<src>[^']+)'|\"(?<src>[^\"]+)\"|(?<src>[^>\\s]+))\\s*[^>]*>", RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline | RegexOptions.IgnoreCase).Matches(sContent); for (int i = matchs.Count - 1; i >= 0; i--) { Match match = matchs[i]; string url = CRegex.GetUrl(sUrl, match.Groups["src"].Value); if (str.CompareTo(CRegex.GetDomain(url).ToLower()) == 0) { string htmlByUrl = CSocket.GetHtmlByUrl(url); if (htmlByUrl.Length != 0) { smethod_0(htmlByUrl, url, ref dictionary); } } } if (dictionary.Count == 0) { return(GetLinksFromRss(sContent, sUrl, ref lisDes)); } return(dictionary); }
public static string UnicodeToString(string str) { string str2 = ""; str = CRegex.Replace(str, "[\r\n]", "", 0); if (!string.IsNullOrEmpty(str)) { string[] strArray = str.Replace(@"\u", "㊣").Split(new char[] { '㊣' }); try { str2 = str2 + strArray[0]; for (int i = 1; i < strArray.Length; i++) { string str3 = strArray[i]; if (!(string.IsNullOrEmpty(str3) || (str3.Length < 4))) { str3 = strArray[i].Substring(0, 4); str2 = str2 + ((char)int.Parse(str3, NumberStyles.HexNumber)); str2 = str2 + strArray[i].Substring(4); } } } catch (FormatException) { str2 = str2 + "Erorr"; } } return(str2); }
private static void smethod_0(string string_0, string string_1, ref Dictionary <string, string> dictionary_0) { Regex regex = new Regex(@"<a\s+[^>]*href\s*=\s*[^>]+>[\s\S]*?</a>", RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline | RegexOptions.IgnoreCase); Regex regex2 = new Regex("\"|'", RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline | RegexOptions.IgnoreCase); MatchCollection matchs = regex.Matches(string_0); for (int i = matchs.Count - 1; i >= 0; i--) { Match match = matchs[i]; string sContent = GetLink(match.Value).Trim().Replace("\\\"", "").Replace(@"\'", ""); if (RemoveByReg(sContent, "^http.*/$").Length >= 2) { string str = CString.ClearTag(GetTextByLink(match.Value)).Trim(); if ((CString.GetLength(RemoveByReg(str, "首页|下载|中文|English|反馈|讨论区|投诉|建议|联系|关于|about|诚邀|工作|简介|新闻|掠影|风采\r\n|登录|注销|注册|使用|体验|立即|收藏夹|收藏|添加|加入\r\n|更多|more|专题|精选|热卖|热销|推荐|精彩\r\n|加盟|联盟|友情|链接|相关\r\n|订阅|阅读器|RSS\r\n|免责|条款|声明|我的|我们|组织|概况|有限|免费|公司|法律|导航|广告|地图|隐私\r\n|〖|〗|【|】|(|)|[|]|『|』|\\.")) >= 9) && !regex2.IsMatch(str)) { sContent = GetUrlByRelative(string_1, sContent); if (sContent.Length > 0x12) { int index = sContent.IndexOf('#'); if (index > -1) { sContent = sContent.Substring(0, index); } sContent = sContent.Trim(new char[] { '/', '\\' }); string domain = CRegex.GetDomain(sContent); if (!sContent.Equals(domain, StringComparison.OrdinalIgnoreCase) && !(dictionary_0.ContainsKey(sContent) || dictionary_0.ContainsValue(str))) { dictionary_0.Add(sContent, str); } } } } } }
/// <summary> /// 将Unicode字串\u.\u.格式字串转换为原始字符串 /// </summary> /// <param name="str"></param> /// <returns></returns> public static string UnicodeToString(string str) { string outStr = ""; str = CRegex.Replace(str, "[\r\n]", "", 0); if (!string.IsNullOrEmpty(str)) { string[] strlist = str.Replace("\\u", "㊣").Split('㊣'); try { outStr += strlist[0]; for (int i = 1; i < strlist.Length; i++) { string strTemp = strlist[i]; if (!string.IsNullOrEmpty(strTemp) && strTemp.Length >= 4) { strTemp = strlist[i].Substring(0, 4); //将unicode字符转为10进制整数,然后转为char中文字符 outStr += (char)int.Parse(strTemp, System.Globalization.NumberStyles.HexNumber); outStr += strlist[i].Substring(4); } } } catch (FormatException ex) { outStr += "Erorr";//ex.Message; } } return(outStr); }
public static PageType GetPageType(string sUrl, ref string sHtml) { PageType pt = PageType.HTML; //看有没有RSS FEED string regRss = @"<link\s+[^>]*((type=""application/rss\+xml"")|(type=application/rss\+xml))[^>]*>"; Regex r = new Regex(regRss, RegexOptions.IgnoreCase); Match m = r.Match(sHtml); if (m.Captures.Count != 0) {//有,则转向从RSS FEED中抓取 string regHref = @"href=\s*(?:'(?<href>[^']+)'|""(?<href>[^""]+)""|(?<href>[^>\s]+))"; r = new Regex(regHref, RegexOptions.IgnoreCase); m = r.Match(m.Captures[0].Value); if (m.Captures.Count > 0) { //有可能是相对路径,加上绝对路径 string rssFile = CRegex.GetUrl(sUrl, m.Groups["href"].Value); sHtml = GetHtmlByUrl(rssFile); pt = PageType.RSS; } } else {//看这个地址本身是不是一个Rss feed r = new Regex(@"<rss\s+[^>]*>", RegexOptions.IgnoreCase); m = r.Match(sHtml); if (m.Captures.Count > 0) { pt = PageType.RSS; } } return(pt); }
public static PageType GetPageType(string sUrl, ref string sHtml) { PageType hTML = PageType.HTML; string pattern = "<link\\s+[^>]*((type=\"application/rss\\+xml\")|(type=application/rss\\+xml))[^>]*>"; Regex regex = new Regex(pattern, RegexOptions.IgnoreCase); Match match = regex.Match(sHtml); if (match.Captures.Count != 0) { string str2 = "href=\\s*(?:'(?<href>[^']+)'|\"(?<href>[^\"]+)\"|(?<href>[^>\\s]+))"; match = new Regex(str2, RegexOptions.IgnoreCase).Match(match.Captures[0].Value); if (match.Captures.Count > 0) { string url = CRegex.GetUrl(sUrl, match.Groups["href"].Value); sHtml = GetHtmlByUrl(url); hTML = PageType.RSS; } return(hTML); } regex = new Regex(@"<rss\s+[^>]*>", RegexOptions.IgnoreCase); if (regex.Match(sHtml).Captures.Count > 0) { hTML = PageType.RSS; } return(hTML); }
private static string[] smethod_2(string string_0, string string_1, string string_2) { ArrayList list = new ArrayList(); Regex regex = new Regex(string_0, RegexOptions.IgnoreCase); for (Match match = regex.Match(string_2); match.Success; match = match.NextMatch()) { list.Add(CRegex.GetUrl(string_1, match.Groups["src"].Value)); } return((string[])list.ToArray(Type.GetType("System.String"))); }
private static string[] DealWithFrame(string strReg, string url, string content) { ArrayList alFrame = new ArrayList(); Regex r = new Regex(strReg, RegexOptions.IgnoreCase); Match m = r.Match(content); while (m.Success) { alFrame.Add(CRegex.GetUrl(url, m.Groups["src"].Value)); m = m.NextMatch(); } return((string[])alFrame.ToArray(System.Type.GetType("System.String"))); }
/// <summary> /// 置换连接 /// </summary> private static string _ReplaceUrl(string strRe, string subMatch, string sFormartted, string sPageUrl) { Regex re = new Regex(strRe, RegexOptions.Multiline | RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase); MatchCollection mcs = re.Matches(sFormartted); string sOriStr = ""; string sSubMatch = ""; string sReplaceStr = ""; foreach (Match mc in mcs) { sOriStr = mc.Value; sSubMatch = mc.Groups[subMatch].Value; sReplaceStr = sOriStr.Replace(sSubMatch, CRegex.GetUrl(sPageUrl, sSubMatch)); sFormartted = sFormartted.Replace(sOriStr, sReplaceStr); } return(sFormartted); }
public static Dictionary <string, string> GetLinks(string sContent, string sUrl, ref Dictionary <string, string> lisDes) { Dictionary <string, string> lisA = new Dictionary <string, string>(); _GetLinks(sContent, sUrl, ref lisA); string domain = CRegex.GetDomain(sUrl).ToLower(); //抓取脚本输出的链接 Regex re = new Regex(@"<script[^>]+src\s*=\s*(?:'(?<src>[^']+)'|""(?<src>[^""]+)""|(?<src>[^>\s]+))\s*[^>]*>", RegexOptions.Multiline | RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase); MatchCollection mcs = re.Matches(sContent); //foreach (Match mc in mcs) for (int i = mcs.Count - 1; i >= 0; i--) { Match mc = mcs[i]; string subUrl = CRegex.GetUrl(sUrl, mc.Groups["src"].Value); if (domain.CompareTo(CRegex.GetDomain(subUrl).ToLower()) != 0) { //同一域的才提炼 continue; } string subContent = CSocket.GetHtmlByUrl(subUrl); if (subContent.Length == 0) { continue; } _GetLinks(subContent, subUrl, ref lisA); } if (lisA.Count == 0) { return(GetLinksFromRss(sContent, sUrl, ref lisDes)); } return(lisA); }
public static string GetContent(string sOriContent, string sOtherRemoveReg, string sPageUrl, DataTable dtAntiLink) { string sFormartted = sOriContent; //去掉有危险的标记 sFormartted = Regex.Replace(sFormartted, @"<script[\s\S]*?</script>", "", RegexOptions.Multiline | RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase); sFormartted = Regex.Replace(sFormartted, @"<iframe[^>]*>[\s\S]*?</iframe>", "", RegexOptions.Multiline | RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase); Regex r = new Regex(@"<input[\s\S]+?>|<form[\s\S]+?>|</form[\s\S]*?>|<select[\s\S]+?>?</select>|<textarea[\s\S]*?>?</textarea>|<file[\s\S]*?>|<noscript>|</noscript>", RegexOptions.IgnoreCase); sFormartted = r.Replace(sFormartted, ""); string[] sOtherReg = sOtherRemoveReg.Split(new string[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries); foreach (string sRemoveReg in sOtherReg) { sFormartted = CRegex.Replace(sFormartted, sRemoveReg, "", 0); } //图片路径 //sFormartted = _ReplaceUrl("<img[^>]+src\\s*=\\s*(?:'(?<src>[^']+)'|\"(?<src>[^\"]+)\"|(?<src>[^>\\s]+))\\s*[^>]*>", "src", sFormartted,sPageUrl); sFormartted = _ReplaceUrl("<img[\\s\\S]+?src\\s*=\\s*(?:'(?<src>[^']+)'|\"(?<src>[^\"]+)\"|(?<src>[^>\\s]+))\\s*[^>]*>", "src", sFormartted, sPageUrl); //反防盗链 string domain = GetDomain(sPageUrl); DataRow[] drs = dtAntiLink.Select("Domain='" + domain + "'"); if (drs.Length > 0) { foreach (DataRow dr in drs) { switch (Convert.ToInt32(dr["Type"])) { case 1: //置换 sFormartted = sFormartted.Replace(dr["imgUrl"].ToString(), "http://stat.580k.com/t.asp?url="); break; default: //附加 sFormartted = sFormartted.Replace(dr["imgUrl"].ToString(), "http://stat.580k.com/t.asp?url=" + dr["imgUrl"].ToString()); break; } } } //A链接 sFormartted = _ReplaceUrl(@"<a[^>]+href\s*=\s*(?:'(?<href>[^']+)'|""(?<href>[^""]+)""|(?<href>[^>\s]+))\s*[^>]*>", "href", sFormartted, sPageUrl); //CSS sFormartted = _ReplaceUrl(@"<link[^>]+href\s*=\s*(?:'(?<href>[^']+)'|""(?<href>[^""]+)""|(?<href>[^>\s]+))\s*[^>]*>", "href", sFormartted, sPageUrl); //BACKGROUND sFormartted = _ReplaceUrl(@"background\s*=\s*(?:'(?<img>[^']+)'|""(?<img>[^""]+)""|(?<img>[^>\s]+))", "img", sFormartted, sPageUrl); //style方式的背景:background-image:url(...) sFormartted = _ReplaceUrl(@"background-image\s*:\s*url\s*\x28(?<img>[^\x29]+)\x29", "img", sFormartted, sPageUrl); //FLASH sFormartted = _ReplaceUrl(@"<param\s[^>]+""movie""[^>]+value\s*=\s*""(?<flash>[^"">]+\x2eswf)""[^>]*>", "flash", sFormartted, sPageUrl); //XSL if (IsXml(sFormartted)) { sFormartted = _ReplaceUrl(@"<\x3fxml-stylesheet\s+[^\x3f>]+href=\s*(?:'(?<href>[^']+)'|""(?<href>[^""]+)"")\s*[^\x3f>]*\x3f>", "href", sFormartted, sPageUrl); } //script //sFormartted = _ReplaceUrl(@"<script[^>]+src\s*=\s*(?:'(?<src>[^']+)'|""(?<src>[^""]+)""|(?<src>[^>\s]+))\s*[^>]*>", "src", sFormartted,sPageUrl); return(sFormartted); }
/// <summary> /// 网页内容 /// </summary> /// <param name="sInput">输入内容</param> public static string GetHtml(string sInput) { return(CRegex.Replace(sInput, @"(?<Head>[^<]+)<", "", "Head")); }
private static void _GetLinks(string sContent, string sUrl, ref Dictionary <string, string> lisA) { const string sFilter = @"首页|下载|中文|English|反馈|讨论区|投诉|建议|联系|关于|about|诚邀|工作|简介|新闻|掠影|风采 |登录|注销|注册|使用|体验|立即|收藏夹|收藏|添加|加入 |更多|more|专题|精选|热卖|热销|推荐|精彩 |加盟|联盟|友情|链接|相关 |订阅|阅读器|RSS |免责|条款|声明|我的|我们|组织|概况|有限|免费|公司|法律|导航|广告|地图|隐私 |〖|〗|【|】|(|)|[|]|『|』|\."; Regex re = new Regex(@"<a\s+[^>]*href\s*=\s*[^>]+>[\s\S]*?</a>", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline); Regex re2 = new Regex(@"""|'", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline); MatchCollection mcs = re.Matches(sContent); //foreach (Match mc in mcs) for (int i = mcs.Count - 1; i >= 0; i--) { Match mc = mcs[i]; string strHref = GetLink(mc.Value).Trim(); strHref = strHref.Replace("\\\"", "");//针对JS输出链接 strHref = strHref.Replace("\\\'", ""); string strTemp = RemoveByReg(strHref, @"^http.*/$");//屏蔽以“http”开头“/”结尾的链接地址 if (strTemp.Length < 2) { continue; } //过滤广告或无意义的链接 string strText = CString.ClearTag(GetTextByLink(mc.Value)).Trim(); strTemp = RemoveByReg(strText, sFilter); if (CString.GetLength(strTemp) < 9) { continue; } if (re2.IsMatch(strText)) { continue; } //换上绝对地址 strHref = CText.GetUrlByRelative(sUrl, strHref); if (strHref.Length <= 18)//例如,http://www.163.com = 18 { continue; } //计算#字符出现的位置,移除它后面的内容 //如果是域名地址,就跳过 int charIndex = strHref.IndexOf('#'); if (charIndex > -1) { strHref = strHref.Substring(0, charIndex); } strHref = strHref.Trim(new char[] { '/', '\\' }); string tmpDomainURL = CRegex.GetDomain(strHref); if (strHref.Equals(tmpDomainURL, StringComparison.OrdinalIgnoreCase)) { continue; } if (!lisA.ContainsKey(strHref) && !lisA.ContainsValue(strText)) { lisA.Add(strHref, strText); } } }