public static Dictionary <string, string> GetLinks(string sContent, string sUrl, ref Dictionary <string, string> lisDes)
{
Dictionary <string, string> dictionary = new Dictionary <string, string>();
smethod_0(sContent, sUrl, ref dictionary);
string str = CRegex.GetDomain(sUrl).ToLower();
MatchCollection matchs = new Regex("<script[^>]+src\\s*=\\s*(?:'(?<src>[^']+)'|\"(?<src>[^\"]+)\"|(?<src>[^>\\s]+))\\s*[^>]*>", RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline | RegexOptions.IgnoreCase).Matches(sContent);
for (int i = matchs.Count - 1; i >= 0; i--)
{
Match match = matchs[i];
string url = CRegex.GetUrl(sUrl, match.Groups["src"].Value);
if (str.CompareTo(CRegex.GetDomain(url).ToLower()) == 0)
{
string htmlByUrl = CSocket.GetHtmlByUrl(url);
if (htmlByUrl.Length != 0)
{
smethod_0(htmlByUrl, url, ref dictionary);
}
}
}
if (dictionary.Count == 0)
{
return(GetLinksFromRss(sContent, sUrl, ref lisDes));
}
return(dictionary);
}
public static string UnicodeToString(string str)
{
string str2 = "";
str = CRegex.Replace(str, "[\r\n]", "", 0);
if (!string.IsNullOrEmpty(str))
{
string[] strArray = str.Replace(@"\u", "㊣").Split(new char[] { '㊣' });
try
{
str2 = str2 + strArray[0];
for (int i = 1; i < strArray.Length; i++)
{
string str3 = strArray[i];
if (!(string.IsNullOrEmpty(str3) || (str3.Length < 4)))
{
str3 = strArray[i].Substring(0, 4);
str2 = str2 + ((char)int.Parse(str3, NumberStyles.HexNumber));
str2 = str2 + strArray[i].Substring(4);
}
}
}
catch (FormatException)
{
str2 = str2 + "Erorr";
}
}
return(str2);
}
private static void smethod_0(string string_0, string string_1, ref Dictionary <string, string> dictionary_0)
{
Regex regex = new Regex(@"<a\s+[^>]*href\s*=\s*[^>]+>[\s\S]*?</a>", RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline | RegexOptions.IgnoreCase);
Regex regex2 = new Regex("\"|'", RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline | RegexOptions.IgnoreCase);
MatchCollection matchs = regex.Matches(string_0);
for (int i = matchs.Count - 1; i >= 0; i--)
{
Match match = matchs[i];
string sContent = GetLink(match.Value).Trim().Replace("\\\"", "").Replace(@"\'", "");
if (RemoveByReg(sContent, "^http.*/$").Length >= 2)
{
string str = CString.ClearTag(GetTextByLink(match.Value)).Trim();
if ((CString.GetLength(RemoveByReg(str, "首页|下载|中文|English|反馈|讨论区|投诉|建议|联系|关于|about|诚邀|工作|简介|新闻|掠影|风采\r\n|登录|注销|注册|使用|体验|立即|收藏夹|收藏|添加|加入\r\n|更多|more|专题|精选|热卖|热销|推荐|精彩\r\n|加盟|联盟|友情|链接|相关\r\n|订阅|阅读器|RSS\r\n|免责|条款|声明|我的|我们|组织|概况|有限|免费|公司|法律|导航|广告|地图|隐私\r\n|〖|〗|【|】|（|）|［|］|『|』|\\.")) >= 9) && !regex2.IsMatch(str))
{
sContent = GetUrlByRelative(string_1, sContent);
if (sContent.Length > 0x12)
{
int index = sContent.IndexOf('#');
if (index > -1)
{
sContent = sContent.Substring(0, index);
}
sContent = sContent.Trim(new char[] { '/', '\\' });
string domain = CRegex.GetDomain(sContent);
if (!sContent.Equals(domain, StringComparison.OrdinalIgnoreCase) && !(dictionary_0.ContainsKey(sContent) || dictionary_0.ContainsValue(str)))
{
dictionary_0.Add(sContent, str);
}
}
}
}
}
}
/// <summary>
/// 将Unicode字串\u.\u.格式字串转换为原始字符串
/// </summary>
/// <param name="str"></param>
/// <returns></returns>
public static string UnicodeToString(string str)
{
string outStr = "";
str = CRegex.Replace(str, "[\r\n]", "", 0);
if (!string.IsNullOrEmpty(str))
{
string[] strlist = str.Replace("\\u", "㊣").Split('㊣');
try
{
outStr += strlist[0];
for (int i = 1; i < strlist.Length; i++)
{
string strTemp = strlist[i];
if (!string.IsNullOrEmpty(strTemp) && strTemp.Length >= 4)
{
strTemp = strlist[i].Substring(0, 4);
//将unicode字符转为10进制整数，然后转为char中文字符
outStr += (char)int.Parse(strTemp, System.Globalization.NumberStyles.HexNumber);
outStr += strlist[i].Substring(4);
}
}
}
catch (FormatException ex)
{
outStr += "Erorr";//ex.Message;
}
}
return(outStr);
}
public static PageType GetPageType(string sUrl, ref string sHtml)
{
PageType pt = PageType.HTML;
//看有没有RSS FEED
string regRss = @"<link\s+[^>]*((type=""application/rss\+xml"")|(type=application/rss\+xml))[^>]*>";
Regex r = new Regex(regRss, RegexOptions.IgnoreCase);
Match m = r.Match(sHtml);
if (m.Captures.Count != 0)
{//有，则转向从RSS FEED中抓取
string regHref = @"href=\s*(?:'(?<href>[^']+)'|""(?<href>[^""]+)""|(?<href>[^>\s]+))";
r = new Regex(regHref, RegexOptions.IgnoreCase);
m = r.Match(m.Captures[0].Value);
if (m.Captures.Count > 0)
{
//有可能是相对路径，加上绝对路径
string rssFile = CRegex.GetUrl(sUrl, m.Groups["href"].Value);
sHtml = GetHtmlByUrl(rssFile);
pt = PageType.RSS;
}
}
else
{//看这个地址本身是不是一个Rss feed
r = new Regex(@"<rss\s+[^>]*>", RegexOptions.IgnoreCase);
m = r.Match(sHtml);
if (m.Captures.Count > 0)
{
pt = PageType.RSS;
}
}
return(pt);
}
public static PageType GetPageType(string sUrl, ref string sHtml)
{
PageType hTML = PageType.HTML;
string pattern = "<link\\s+[^>]*((type=\"application/rss\\+xml\")|(type=application/rss\\+xml))[^>]*>";
Regex regex = new Regex(pattern, RegexOptions.IgnoreCase);
Match match = regex.Match(sHtml);
if (match.Captures.Count != 0)
{
string str2 = "href=\\s*(?:'(?<href>[^']+)'|\"(?<href>[^\"]+)\"|(?<href>[^>\\s]+))";
match = new Regex(str2, RegexOptions.IgnoreCase).Match(match.Captures[0].Value);
if (match.Captures.Count > 0)
{
string url = CRegex.GetUrl(sUrl, match.Groups["href"].Value);
sHtml = GetHtmlByUrl(url);
hTML = PageType.RSS;
}
return(hTML);
}
regex = new Regex(@"<rss\s+[^>]*>", RegexOptions.IgnoreCase);
if (regex.Match(sHtml).Captures.Count > 0)
{
hTML = PageType.RSS;
}
return(hTML);
}
private static string[] smethod_2(string string_0, string string_1, string string_2)
{
ArrayList list = new ArrayList();
Regex regex = new Regex(string_0, RegexOptions.IgnoreCase);
for (Match match = regex.Match(string_2); match.Success; match = match.NextMatch())
{
list.Add(CRegex.GetUrl(string_1, match.Groups["src"].Value));
}
return((string[])list.ToArray(Type.GetType("System.String")));
}
private static string[] DealWithFrame(string strReg, string url, string content)
{
ArrayList alFrame = new ArrayList();
Regex r = new Regex(strReg, RegexOptions.IgnoreCase);
Match m = r.Match(content);
while (m.Success)
{
alFrame.Add(CRegex.GetUrl(url, m.Groups["src"].Value));
m = m.NextMatch();
}
return((string[])alFrame.ToArray(System.Type.GetType("System.String")));
}
/// <summary>
/// 置换连接
/// </summary>
private static string _ReplaceUrl(string strRe, string subMatch, string sFormartted, string sPageUrl)
{
Regex re = new Regex(strRe, RegexOptions.Multiline | RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase);
MatchCollection mcs = re.Matches(sFormartted);
string sOriStr = "";
string sSubMatch = "";
string sReplaceStr = "";
foreach (Match mc in mcs)
{
sOriStr = mc.Value;
sSubMatch = mc.Groups[subMatch].Value;
sReplaceStr = sOriStr.Replace(sSubMatch, CRegex.GetUrl(sPageUrl, sSubMatch));
sFormartted = sFormartted.Replace(sOriStr, sReplaceStr);
}
return(sFormartted);
}
public static Dictionary <string, string> GetLinks(string sContent, string sUrl, ref Dictionary <string, string> lisDes)
{
Dictionary <string, string> lisA = new Dictionary <string, string>();
_GetLinks(sContent, sUrl, ref lisA);
string domain = CRegex.GetDomain(sUrl).ToLower();
//抓取脚本输出的链接
Regex re = new Regex(@"<script[^>]+src\s*=\s*(?:'(?<src>[^']+)'|""(?<src>[^""]+)""|(?<src>[^>\s]+))\s*[^>]*>", RegexOptions.Multiline | RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase);
MatchCollection mcs = re.Matches(sContent);
//foreach (Match mc in mcs)
for (int i = mcs.Count - 1; i >= 0; i--)
{
Match mc = mcs[i];
string subUrl = CRegex.GetUrl(sUrl, mc.Groups["src"].Value);
if (domain.CompareTo(CRegex.GetDomain(subUrl).ToLower()) != 0)
{
//同一域的才提炼
continue;
}
string subContent = CSocket.GetHtmlByUrl(subUrl);
if (subContent.Length == 0)
{
continue;
}
_GetLinks(subContent, subUrl, ref lisA);
}
if (lisA.Count == 0)
{
return(GetLinksFromRss(sContent, sUrl, ref lisDes));
}
return(lisA);
}
public static string GetContent(string sOriContent, string sOtherRemoveReg, string sPageUrl, DataTable dtAntiLink)
{
string sFormartted = sOriContent;
//去掉有危险的标记
sFormartted = Regex.Replace(sFormartted, @"<script[\s\S]*?</script>", "", RegexOptions.Multiline | RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase);
sFormartted = Regex.Replace(sFormartted, @"<iframe[^>]*>[\s\S]*?</iframe>", "", RegexOptions.Multiline | RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase);
Regex r = new Regex(@"<input[\s\S]+?>|<form[\s\S]+?>|</form[\s\S]*?>|<select[\s\S]+?>?</select>|<textarea[\s\S]*?>?</textarea>|<file[\s\S]*?>|<noscript>|</noscript>", RegexOptions.IgnoreCase);
sFormartted = r.Replace(sFormartted, "");
string[] sOtherReg = sOtherRemoveReg.Split(new string[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
foreach (string sRemoveReg in sOtherReg)
{
sFormartted = CRegex.Replace(sFormartted, sRemoveReg, "", 0);
}
//图片路径
//sFormartted = _ReplaceUrl("<img[^>]+src\\s*=\\s*(?:'(?<src>[^']+)'|\"(?<src>[^\"]+)\"|(?<src>[^>\\s]+))\\s*[^>]*>", "src", sFormartted,sPageUrl);
sFormartted = _ReplaceUrl("<img[\\s\\S]+?src\\s*=\\s*(?:'(?<src>[^']+)'|\"(?<src>[^\"]+)\"|(?<src>[^>\\s]+))\\s*[^>]*>", "src", sFormartted, sPageUrl);
//反防盗链
string domain = GetDomain(sPageUrl);
DataRow[] drs = dtAntiLink.Select("Domain='" + domain + "'");
if (drs.Length > 0)
{
foreach (DataRow dr in drs)
{
switch (Convert.ToInt32(dr["Type"]))
{
case 1: //置换
sFormartted = sFormartted.Replace(dr["imgUrl"].ToString(), "http://stat.580k.com/t.asp?url=");
break;
default: //附加
sFormartted = sFormartted.Replace(dr["imgUrl"].ToString(), "http://stat.580k.com/t.asp?url=" + dr["imgUrl"].ToString());
break;
}
}
}
//A链接
sFormartted = _ReplaceUrl(@"<a[^>]+href\s*=\s*(?:'(?<href>[^']+)'|""(?<href>[^""]+)""|(?<href>[^>\s]+))\s*[^>]*>", "href", sFormartted, sPageUrl);
//CSS
sFormartted = _ReplaceUrl(@"<link[^>]+href\s*=\s*(?:'(?<href>[^']+)'|""(?<href>[^""]+)""|(?<href>[^>\s]+))\s*[^>]*>", "href", sFormartted, sPageUrl);
//BACKGROUND
sFormartted = _ReplaceUrl(@"background\s*=\s*(?:'(?<img>[^']+)'|""(?<img>[^""]+)""|(?<img>[^>\s]+))", "img", sFormartted, sPageUrl);
//style方式的背景：background-image:url(...)
sFormartted = _ReplaceUrl(@"background-image\s*:\s*url\s*\x28(?<img>[^\x29]+)\x29", "img", sFormartted, sPageUrl);
//FLASH
sFormartted = _ReplaceUrl(@"<param\s[^>]+""movie""[^>]+value\s*=\s*""(?<flash>[^"">]+\x2eswf)""[^>]*>", "flash", sFormartted, sPageUrl);
//XSL
if (IsXml(sFormartted))
{
sFormartted = _ReplaceUrl(@"<\x3fxml-stylesheet\s+[^\x3f>]+href=\s*(?:'(?<href>[^']+)'|""(?<href>[^""]+)"")\s*[^\x3f>]*\x3f>", "href", sFormartted, sPageUrl);
}
//script
//sFormartted = _ReplaceUrl(@"<script[^>]+src\s*=\s*(?:'(?<src>[^']+)'|""(?<src>[^""]+)""|(?<src>[^>\s]+))\s*[^>]*>", "src", sFormartted,sPageUrl);
return(sFormartted);
}
/// <summary>
/// 网页内容
/// </summary>
/// <param name="sInput">输入内容</param>
public static string GetHtml(string sInput)
{
return(CRegex.Replace(sInput, @"(?<Head>[^<]+)<", "", "Head"));
}
private static void _GetLinks(string sContent, string sUrl, ref Dictionary <string, string> lisA)
{
const string sFilter =
@"首页|下载|中文|English|反馈|讨论区|投诉|建议|联系|关于|about|诚邀|工作|简介|新闻|掠影|风采
|登录|注销|注册|使用|体验|立即|收藏夹|收藏|添加|加入
|更多|more|专题|精选|热卖|热销|推荐|精彩
|加盟|联盟|友情|链接|相关
|订阅|阅读器|RSS
|免责|条款|声明|我的|我们|组织|概况|有限|免费|公司|法律|导航|广告|地图|隐私
|〖|〗|【|】|（|）|［|］|『|』|\.";
Regex re = new Regex(@"<a\s+[^>]*href\s*=\s*[^>]+>[\s\S]*?</a>", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline);
Regex re2 = new Regex(@"""|'", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline);
MatchCollection mcs = re.Matches(sContent);
//foreach (Match mc in mcs)
for (int i = mcs.Count - 1; i >= 0; i--)
{
Match mc = mcs[i];
string strHref = GetLink(mc.Value).Trim();
strHref = strHref.Replace("\\\"", "");//针对JS输出链接
strHref = strHref.Replace("\\\'", "");
string strTemp = RemoveByReg(strHref, @"^http.*/$");//屏蔽以“http”开头“/”结尾的链接地址
if (strTemp.Length < 2)
{
continue;
}
//过滤广告或无意义的链接
string strText = CString.ClearTag(GetTextByLink(mc.Value)).Trim();
strTemp = RemoveByReg(strText, sFilter);
if (CString.GetLength(strTemp) < 9)
{
continue;
}
if (re2.IsMatch(strText))
{
continue;
}
//换上绝对地址
strHref = CText.GetUrlByRelative(sUrl, strHref);
if (strHref.Length <= 18)//例如，http://www.163.com = 18
{
continue;
}
//计算#字符出现的位置，移除它后面的内容
//如果是域名地址，就跳过
int charIndex = strHref.IndexOf('#');
if (charIndex > -1)
{
strHref = strHref.Substring(0, charIndex);
}
strHref = strHref.Trim(new char[] { '/', '\\' });
string tmpDomainURL = CRegex.GetDomain(strHref);
if (strHref.Equals(tmpDomainURL, StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (!lisA.ContainsKey(strHref) && !lisA.ContainsValue(strText))
{
lisA.Add(strHref, strText);
}
}
}