public static void DoDailyUpdate(int page = 200, bool showConsole = true) { Dictionary <string, string> updatePages = new Dictionary <string, string>(); List <string> urls = new List <string>(); GetJavCookie(showConsole); var htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", "http://www.javlibrary.com/cn/vl_update.php?&mode=&page=1", cc); for (int i = 1; i <= page; i++) { updatePages.Add("http://www.javlibrary.com/cn/vl_update.php?&mode=&page=" + i, "更新"); } int index = 0; Parallel.ForEach(updatePages, new ParallelOptions { MaxDegreeOfParallelism = 100 }, url => { index++; ScanCategoryPageUrl(url.Key, url.Value, index, page, urls); }); ScanDailyAvList(urls); }
private static Utils.HtmlResponse JavCookieContanierHelper(string url) { var htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", url, cc); return(htmlRes); }
private static void ScanEachAvSingleThread() { List <ScanURL> urls = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false).ToList(); int index = 0; foreach (var url in urls) { int retry = 1; //二次确认 if (!JavDataBaseManager.HasAv(url.URL)) { var htmlRes = new Utils.HtmlResponse(); //最多重试5次 while (retry <= 5) { htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", url.URL, cc); if (htmlRes.IsExpire) { GetJavCookie(); retry++; continue; } else { break; } } if (htmlRes.Success) { index++; HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument(); htmlDocument.LoadHtml(htmlRes.Content); var av = GenerateAVModel(htmlRes.Content, url.URL); JavDataBaseManager.InsertAV(av); Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 插入AV => " + av.ID + " - " + av.Name); JavDataBaseManager.UpdateScanURL(url.URL); string result = ""; if (!File.Exists(ImgFolder + av.ID + av.Name + ".jpg")) { result = DownloadHelper.DownloadHttps(av.PictureURL, ImgFolder + av.ID + av.Name + ".jpg", ""); if (string.IsNullOrEmpty(result)) { Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 下载AV图片成功 => " + av.ID + " - " + av.Name); } else { Console.WriteLine(result); } } else { Console.WriteLine("已存在图片不下载"); } Console.WriteLine("完成" + index + " / " + urls.Count); } } else { JavDataBaseManager.UpdateScanURL(url.URL); Console.WriteLine("详情页已下载 => " + url.URL + " 完成" + index + " / " + urls.Count); } } }
private static void ScanCategoryPageUrlSingleThread(Dictionary <string, string> urls) { int index = 1; foreach (var url in urls) { int retry = 1; var htmlRes = new Utils.HtmlResponse(); //如果取不到cookie最多重试5次 while (retry <= 5) { htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", url.Key, cc); if (htmlRes.IsExpire) { GetJavCookie(); retry++; continue; } else { break; } } if (htmlRes.Success) { HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument(); htmlDocument.LoadHtml(htmlRes.Content); var videoPath = "//div[@class='video']"; var videoNodes = htmlDocument.DocumentNode.SelectNodes(videoPath); if (videoNodes != null) { int unScanCount = 0; foreach (var node in videoNodes) { var urlAndTitle = node.ChildNodes[0]; if (urlAndTitle != null && urlAndTitle.ChildNodes.Count >= 3) { var id = urlAndTitle.ChildNodes[0].InnerText.Trim(); var name = FileUtility.ReplaceInvalidChar(urlAndTitle.ChildNodes[2].InnerText.Trim()); var avUrl = urlAndTitle.Attributes["href"].Value.Trim().Replace("./", "http://www.javlibrary.com/cn/"); if (!string.IsNullOrEmpty(avUrl) && !string.IsNullOrEmpty(name) && !string.IsNullOrWhiteSpace(id)) { ScanURL scan = new ScanURL { Category = url.Value, ID = id, IsDownload = false, Title = name, URL = avUrl }; if (!JavDataBaseManager.HasScan(scan)) { unScanCount++; JavDataBaseManager.InsertScanURL(scan); } } } } Console.WriteLine(url.Value + " 第 " + index + " / " + urls.Count + " 页, 加入" + unScanCount + " 条未扫描AV"); index++; } } else { Console.WriteLine("获取列表页 " + url.Key + " 内容失败"); } } }