public static void DoDailyUpdate(int page = 200, bool showConsole = true)
{
Dictionary <string, string> updatePages = new Dictionary <string, string>();
List <string> urls = new List <string>();
GetJavCookie(showConsole);
var htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", "http://www.javlibrary.com/cn/vl_update.php?&mode=&page=1", cc);
for (int i = 1; i <= page; i++)
{
updatePages.Add("http://www.javlibrary.com/cn/vl_update.php?&mode=&page=" + i, "更新");
}
int index = 0;
Parallel.ForEach(updatePages, new ParallelOptions {
MaxDegreeOfParallelism = 100
}, url =>
{
index++;
ScanCategoryPageUrl(url.Key, url.Value, index, page, urls);
});
ScanDailyAvList(urls);
}
private static Utils.HtmlResponse JavCookieContanierHelper(string url)
{
var htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", url, cc);
return(htmlRes);
}
private static void ScanEachAvSingleThread()
{
List <ScanURL> urls = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false).ToList();
int index = 0;
foreach (var url in urls)
{
int retry = 1;
//二次确认
if (!JavDataBaseManager.HasAv(url.URL))
{
var htmlRes = new Utils.HtmlResponse();
//最多重试5次
while (retry <= 5)
{
htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", url.URL, cc);
if (htmlRes.IsExpire)
{
GetJavCookie();
retry++;
continue;
}
else
{
break;
}
}
if (htmlRes.Success)
{
index++;
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(htmlRes.Content);
var av = GenerateAVModel(htmlRes.Content, url.URL);
JavDataBaseManager.InsertAV(av);
Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 插入AV => " + av.ID + " - " + av.Name);
JavDataBaseManager.UpdateScanURL(url.URL);
string result = "";
if (!File.Exists(ImgFolder + av.ID + av.Name + ".jpg"))
{
result = DownloadHelper.DownloadHttps(av.PictureURL, ImgFolder + av.ID + av.Name + ".jpg", "");
if (string.IsNullOrEmpty(result))
{
Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 下载AV图片成功 => " + av.ID + " - " + av.Name);
}
else
{
Console.WriteLine(result);
}
}
else
{
Console.WriteLine("已存在图片不下载");
}
Console.WriteLine("完成" + index + " / " + urls.Count);
}
}
else
{
JavDataBaseManager.UpdateScanURL(url.URL);
Console.WriteLine("详情页已下载 => " + url.URL + " 完成" + index + " / " + urls.Count);
}
}
}
private static void ScanCategoryPageUrlSingleThread(Dictionary <string, string> urls)
{
int index = 1;
foreach (var url in urls)
{
int retry = 1;
var htmlRes = new Utils.HtmlResponse();
//如果取不到cookie最多重试5次
while (retry <= 5)
{
htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", url.Key, cc);
if (htmlRes.IsExpire)
{
GetJavCookie();
retry++;
continue;
}
else
{
break;
}
}
if (htmlRes.Success)
{
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(htmlRes.Content);
var videoPath = "//div[@class='video']";
var videoNodes = htmlDocument.DocumentNode.SelectNodes(videoPath);
if (videoNodes != null)
{
int unScanCount = 0;
foreach (var node in videoNodes)
{
var urlAndTitle = node.ChildNodes[0];
if (urlAndTitle != null && urlAndTitle.ChildNodes.Count >= 3)
{
var id = urlAndTitle.ChildNodes[0].InnerText.Trim();
var name = FileUtility.ReplaceInvalidChar(urlAndTitle.ChildNodes[2].InnerText.Trim());
var avUrl = urlAndTitle.Attributes["href"].Value.Trim().Replace("./", "http://www.javlibrary.com/cn/");
if (!string.IsNullOrEmpty(avUrl) && !string.IsNullOrEmpty(name) && !string.IsNullOrWhiteSpace(id))
{
ScanURL scan = new ScanURL
{
Category = url.Value,
ID = id,
IsDownload = false,
Title = name,
URL = avUrl
};
if (!JavDataBaseManager.HasScan(scan))
{
unScanCount++;
JavDataBaseManager.InsertScanURL(scan);
}
}
}
}
Console.WriteLine(url.Value + " 第 " + index + " / " + urls.Count + " 页, 加入" + unScanCount + " 条未扫描AV");
index++;
}
}
else
{
Console.WriteLine("获取列表页 " + url.Key + " 内容失败");
}
}
}