public static void DoDownload(CookieContainer cc) { var noDownloads = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false); int currentItems = 1; foreach (var item in noDownloads) { cc = DownloadHelper.DownloadManager.Download(item.URL, currentItems, noDownloads.Count(), cc); } }
private static void ScanAvList() { List <ScanURL> avs = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false).ToList(); Parallel.ForEach(avs, new ParallelOptions { MaxDegreeOfParallelism = 100 }, av => { ScanEachAv(av, string.Format("进度 {0}/{1}", avs.IndexOf(av) + 1, avs.Count)); }); }
public static CookieContainer Scan(string url, string category, int currentCategory, int totalCategories, CookieContainer cc, bool isUpdate = false) { try { int page = 1; while (!string.IsNullOrEmpty(url)) { var ret = RecursiveHelper(url, category, currentCategory, totalCategories, page, cc, isUpdate); url = ret.Url; cc = ret.Cc; page++; } //var json = JsonConvert.SerializeObject(ForUpdate); var noDownloads = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false); int current = 1; if (isUpdate) { foreach (var update in ForUpdate) { if (!JavDataBaseManager.HasScan(update)) { cc = DownloadHelper.DownloadManager.Download(update.URL, current, ForUpdate.Count, cc); } current++; } } else { foreach (var download in noDownloads) { if (!JavDataBaseManager.HasScan(download)) { cc = DownloadHelper.DownloadManager.Download(download.URL, current, noDownloads.Count(), cc); } current++; } } } catch (Exception e) { _logger.WriteExceptionLog("", string.Format("Scan failed. {0}", e.ToString())); } return(cc); }
private static void ScanEachAvSingleThread() { List <ScanURL> urls = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false).ToList(); int index = 0; foreach (var url in urls) { int retry = 1; //二次确认 if (!JavDataBaseManager.HasAv(url.URL)) { var htmlRes = new Utils.HtmlResponse(); //最多重试5次 while (retry <= 5) { htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", url.URL, cc); if (htmlRes.IsExpire) { GetJavCookie(); retry++; continue; } else { break; } } if (htmlRes.Success) { index++; HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument(); htmlDocument.LoadHtml(htmlRes.Content); var av = GenerateAVModel(htmlRes.Content, url.URL); JavDataBaseManager.InsertAV(av); Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 插入AV => " + av.ID + " - " + av.Name); JavDataBaseManager.UpdateScanURL(url.URL); string result = ""; if (!File.Exists(ImgFolder + av.ID + av.Name + ".jpg")) { result = DownloadHelper.DownloadHttps(av.PictureURL, ImgFolder + av.ID + av.Name + ".jpg", ""); if (string.IsNullOrEmpty(result)) { Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 下载AV图片成功 => " + av.ID + " - " + av.Name); } else { Console.WriteLine(result); } } else { Console.WriteLine("已存在图片不下载"); } Console.WriteLine("完成" + index + " / " + urls.Count); } } else { JavDataBaseManager.UpdateScanURL(url.URL); Console.WriteLine("详情页已下载 => " + url.URL + " 完成" + index + " / " + urls.Count); } } }