private static string GetGeneralContent(string strUrl)
{
_requestCount++;
string referer = "http://www.gdajie.com/";
var cookies = new List <Cookie>()
{
new Cookie("AJSTAT_ok_times", "5", "/", "verycd.gdajie.com"),
new Cookie("Hm_lpvt_9398e7331484620894c57216bea9225e", "1468695450", "/", ".gdajie.com"),
new Cookie("JSESSIONID", "7F0463C2223BFF2E8868FC7EFA7D0167", "/", "verycd.gdajie.com"),
new Cookie("CNZZDATA4616656",
"cnzz_eid%3D1953947045-1464628831-http%253A%252F%252Fwww.verycd.gdajie.com%252F%26ntime%3D1467845452",
"/", "verycd.gdajie.com"),
new Cookie("CNZZDATA1254524697", "78693709-1464629094-%7C1468690207", "/", "verycd.gdajie.com")
};
string html = CrawlerUtility.GetGeneralContent(strUrl, referer, cookies);
if (html.Contains("对不起，我们怀疑您正使用采集软件对我们的网站进行采集，所以我们采取了封您ip的决定"))
{
Console.WriteLine("url:" + strUrl + ",ip被封,_requestCount:" + _requestCount);
Info("url:" + strUrl + ",ip被封,requestCount:" + _requestCount);
html = "";
}
if (string.IsNullOrEmpty(html))
{
Thread.Sleep(10 * 1000);
}
else
{
Thread.Sleep(5 * 1000);
}
return(html);
}
private static string GetGeneralContent(string strUrl)
{
_requestCount++;
string referer = "http://www.mp4ba.com/";
var cookies = new List <Cookie>()
{
new Cookie("CNZZDATA5925857", "cnzz_eid%3D921376316-1462025230-%26ntime%3D1467044423", "/",
"www.mp4ba.com")
};
string html = CrawlerUtility.GetGeneralContent(strUrl, referer, cookies);
if (html.Contains("服务器维护中，请稍后访问"))
{
Info("服务器维护中，请稍后访问,60秒后继续操作");
html = "";
}
if (string.IsNullOrEmpty(html))
{
Thread.Sleep(60 * 1000);
}
else
{
Thread.Sleep(5 * 1000);
}
return(html);
}
private static string GetGeneralContent(string strUrl)
{
_requestCount++;
string referer = "http://www.gdajie.com/";
var cookies = new List <Cookie>()
{
new Cookie("AJSTAT_ok_pages", _requestCount.ToString(), "/", "www.ed2kers.com"),
new Cookie("AJSTAT_ok_times", "1", "/", "www.ed2kers.com"),
new Cookie("Hm_lpvt_9398e7331484620894c57216bea9225e", "1468695450", "/", ".gdajie.com"),
new Cookie("PHPSESSID", "0hbum6a055q5sfg5oi03a7ug35", "/", "www.ed2kers.com"),
new Cookie("Hm_lvt_9ad7d25789ce8adb7b225bc58a1b8525", "1470154733", "/", "ed2kers.com"),
new Cookie("Hm_lpvt_9ad7d25789ce8adb7b225bc58a1b8525", "1470156513", "/", "ed2kers.com")
};
string html = CrawlerUtility.GetGeneralContent(strUrl, referer, cookies);
if (string.IsNullOrEmpty(html))
{
//Thread.Sleep(10 * 1000);
}
else
{
//Thread.Sleep(5 * 1000);
}
return(html);
}
private static string GetGeneralContent(string strUrl)
{
string referer = "http://www.zimuzu.tv/";
var cookies = new List <Cookie>
{
new Cookie("CNZZDATA1254180690", "176448891-1468518959-%7C1468679132", "/", "www.zimuzu.tv"),
new Cookie("PHPSESSID", "ht8gtggpqjlth57ef41b7g03k0", "/", "www.zimuzu.tv"),
new Cookie("GINFO",
"uid%3D3667676%26nickname%3Dkinshine%26group_id%3D1%26avatar_t%3D%26main_group_id%3D0%26common_group_id%3D54",
"/", ".zimuzu.tv"),
new Cookie("GKEY", "e4b20dc7275fac6a7f984afcb5938ff3", "/", ".zimuzu.tv"),
new Cookie("mykeywords",
"%3A1%3A%7Bi%3A0%3Bs%3A21%3A%22%E8%82%A5%E7%91%9E%E7%9A%84%E7%96%AF%E7%8B%82%E6%97%A5%E8%AE%B0%22%3B%7D",
"/", ".zimuzu.tv")
};
string html = CrawlerUtility.GetGeneralContent(strUrl, referer, cookies);
if (string.IsNullOrEmpty(html))
{
Thread.Sleep(10 * 1000);
}
else
{
Thread.Sleep(5 * 1000);
}
return(html);
}
private static string GetGeneralContent(string strUrl)
{
_requestCount++;
string referer = "http://www.ed2000.com/";
var cookies = new List <Cookie>()
{
new Cookie("ASPSESSIONIDQCSQTDQD", "OPOIBEHBPMHCPNAMCBJHFPDD", "/", "www.ed2000.com"),
new Cookie("UserID", "30602", "/", "www.ed2000.com"),
new Cookie("UserPassword", "A9E485F632124349EDFCEC448E649702", "/", "www.ed2000.com"),
new Cookie("CNZZDATA947842", "cnzz_eid%3D1459071056-1467470818-null%26ntime%3D1469246688", "/", "www.ed2000.com"),
new Cookie("VisitsNumber", _requestCount.ToString(), "/", "www.ed2000.com")
};
string html = CrawlerUtility.GetGeneralContent(strUrl, referer, cookies);
if (string.IsNullOrEmpty(html))
{
Thread.Sleep(60 * 1000);
}
else
{
Thread.Sleep(2 * 1000);
}
return(html);
}