public static HtmlResponse GetHtmlLockedWebClient(string cookieHost, string url, CookieContainer cc = null)
{
HtmlResponse res = new HtmlResponse
{
Success = false
};
try
{
var wc = new WebClient();
if (cc != null)
{
wc.Headers.Add(HttpRequestHeader.Cookie, cc.GetCookieHeader(new Uri(cookieHost)));
}
wc.Headers.Add(HttpRequestHeader.UserAgent, string.Format(UserAgent, GetChromeVersion()));
var hltext = wc.DownloadData(url);
res.Content = (Encoding.GetEncoding("UTF-8").GetString(hltext));
res.Success = true;
}
catch (Exception ex)
{
res.Content = ex.ToString();
if (ex.ToString().Contains("503"))
{
cc = null;
}
}
return(res);
}
public static HtmlResponse GetHtmlWebClient(string cookieHost, string url, CookieContainer cc = null, bool userProxy = false)
{
HtmlResponse res = new HtmlResponse
{
Success = false
};
List <string> my_headers = new List <string> {
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)"
};
try
{
var wc = new WebClient();
if (cc != null)
{
var c = cc.GetCookieHeader(new Uri(cookieHost));
wc.Headers.Add(HttpRequestHeader.Cookie, c);
}
if (userProxy)
{
Random r = new Random();
var index = r.Next(my_headers.Count - 1);
wc.Headers.Add(HttpRequestHeader.UserAgent, my_headers[index]);
//WebProxy proxy = new WebProxy();
//proxy.UseDefaultCredentials = false;
//proxy.Address = new Uri("http://183.239.167.122:8080"); // new Uri("http://183.239.167.122:8080");
//wc.Proxy = proxy;
}
else
{
wc.Headers.Add(HttpRequestHeader.UserAgent, string.Format(UserAgent, GetChromeVersion()));
}
var hltext = wc.DownloadData(url);
res.Content = (Encoding.GetEncoding("UTF-8").GetString(hltext));
res.Success = true;
}
catch (Exception ex)
{
res.Content = ex.ToString();
}
return(res);
}
public static HtmlResponse GetHtmlContentViaUrl(string url, string end = "utf-8", bool isJav = false, CookieContainer cc = null)
{
HtmlResponse res = new HtmlResponse
{
Success = false
};
try
{
GC.Collect();
StringBuilder sb = new StringBuilder();
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Credentials = CredentialCache.DefaultCredentials;
request.Timeout = 90000;
request.UserAgent = string.Format(UserAgent, GetChromeVersion());
request.Method = "GET";
if (isJav)
{
request.CookieContainer = cc;
}
request.KeepAlive = true;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream dataStream = response.GetResponseStream();
StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding(end));
//while (!reader.EndOfStream)
//{
// sb.AppendLine(reader.ReadLine());
//}
res.Content = reader.ReadToEnd();
reader.Close();
dataStream.Close();
response.Close();
}
catch (Exception e)
{
res.Success = false;
if (e.Message == Error)
{
Console.WriteLine("123");
}
return(res);
}
res.Success = true;
return(res);
}
public static HtmlResponse GetHtmlWebClient(string url, CookieContainer cc = null, string cookieHost = "", string host = "", string refere = "")
{
HtmlResponse res = new HtmlResponse
{
Success = false
};
try
{
var wc = new WebClient();
if (cc != null)
{
wc.Headers.Add(HttpRequestHeader.Cookie, cc.GetCookieHeader(new Uri(cookieHost)));
}
wc.Headers.Add(HttpRequestHeader.UserAgent, string.Format(UserAgent, GetChromeVersion()));
if (!string.IsNullOrEmpty(host))
{
wc.Headers.Add(HttpRequestHeader.Host, host);
}
if (!string.IsNullOrEmpty(refere))
{
wc.Headers.Add(HttpRequestHeader.Referer, refere);
}
var hltext = wc.DownloadData(url);
res.Content = (Encoding.GetEncoding("UTF-8").GetString(hltext));
res.Success = true;
}
catch (Exception ex)
{
res.Content = ex.ToString();
}
return(res);
}
private static void ScanEachAvSingleThread()
{
List <ScanURL> urls = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false).ToList();
int index = 0;
foreach (var url in urls)
{
int retry = 1;
//二次确认
if (!JavDataBaseManager.HasAv(url.URL))
{
var htmlRes = new Utils.HtmlResponse();
//最多重试5次
while (retry <= 5)
{
htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", url.URL, cc);
if (htmlRes.IsExpire)
{
GetJavCookie();
retry++;
continue;
}
else
{
break;
}
}
if (htmlRes.Success)
{
index++;
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(htmlRes.Content);
var av = GenerateAVModel(htmlRes.Content, url.URL);
JavDataBaseManager.InsertAV(av);
Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 插入AV => " + av.ID + " - " + av.Name);
JavDataBaseManager.UpdateScanURL(url.URL);
string result = "";
if (!File.Exists(ImgFolder + av.ID + av.Name + ".jpg"))
{
result = DownloadHelper.DownloadHttps(av.PictureURL, ImgFolder + av.ID + av.Name + ".jpg", "");
if (string.IsNullOrEmpty(result))
{
Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 下载AV图片成功 => " + av.ID + " - " + av.Name);
}
else
{
Console.WriteLine(result);
}
}
else
{
Console.WriteLine("已存在图片不下载");
}
Console.WriteLine("完成" + index + " / " + urls.Count);
}
}
else
{
JavDataBaseManager.UpdateScanURL(url.URL);
Console.WriteLine("详情页已下载 => " + url.URL + " 完成" + index + " / " + urls.Count);
}
}
}
private static void ScanCategoryPageUrlSingleThread(Dictionary <string, string> urls)
{
int index = 1;
foreach (var url in urls)
{
int retry = 1;
var htmlRes = new Utils.HtmlResponse();
//如果取不到cookie最多重试5次
while (retry <= 5)
{
htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", url.Key, cc);
if (htmlRes.IsExpire)
{
GetJavCookie();
retry++;
continue;
}
else
{
break;
}
}
if (htmlRes.Success)
{
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(htmlRes.Content);
var videoPath = "//div[@class='video']";
var videoNodes = htmlDocument.DocumentNode.SelectNodes(videoPath);
if (videoNodes != null)
{
int unScanCount = 0;
foreach (var node in videoNodes)
{
var urlAndTitle = node.ChildNodes[0];
if (urlAndTitle != null && urlAndTitle.ChildNodes.Count >= 3)
{
var id = urlAndTitle.ChildNodes[0].InnerText.Trim();
var name = FileUtility.ReplaceInvalidChar(urlAndTitle.ChildNodes[2].InnerText.Trim());
var avUrl = urlAndTitle.Attributes["href"].Value.Trim().Replace("./", "http://www.javlibrary.com/cn/");
if (!string.IsNullOrEmpty(avUrl) && !string.IsNullOrEmpty(name) && !string.IsNullOrWhiteSpace(id))
{
ScanURL scan = new ScanURL
{
Category = url.Value,
ID = id,
IsDownload = false,
Title = name,
URL = avUrl
};
if (!JavDataBaseManager.HasScan(scan))
{
unScanCount++;
JavDataBaseManager.InsertScanURL(scan);
}
}
}
}
Console.WriteLine(url.Value + " 第 " + index + " / " + urls.Count + " 页, 加入" + unScanCount + " 条未扫描AV");
index++;
}
}
else
{
Console.WriteLine("获取列表页 " + url.Key + " 内容失败");
}
}
}
public static HtmlResponse GetHtmlContentViaUrl(string url, string end = "utf-8", bool isJav = false, CookieContainer cc = null, string agent = "")
{
HtmlResponse res = new HtmlResponse
{
Success = false
};
HttpWebResponse response = null;
try
{
GC.Collect();
StringBuilder sb = new StringBuilder();
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Credentials = CredentialCache.DefaultCredentials;
request.Timeout = 90000;
request.UserAgent = string.IsNullOrEmpty(agent) ? string.Format(UserAgent, GetChromeVersion()) : agent;
request.Method = "GET";
if (isJav)
{
request.CookieContainer = cc;
}
request.KeepAlive = true;
response = (HttpWebResponse)request.GetResponse();
Stream dataStream = response.GetResponseStream();
StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding(end));
//while (!reader.EndOfStream)
//{
// sb.AppendLine(reader.ReadLine());
//}
res.Content = reader.ReadToEnd();
reader.Close();
dataStream.Close();
response.Close();
}
catch (WebException ex)
{
response = (HttpWebResponse)ex.Response;
res.Success = false;
res.Content = "";
if (isJav)
{
Stream myResponseStream = response.GetResponseStream();
using (StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.UTF8))
{
res.Content = myStreamReader.ReadToEnd();
}
myResponseStream.Close();
foreach (Cookie cookie in response.Cookies)
{
cc.Add(cookie);
}
}
return(res);
}
res.Success = true;
return(res);
}