public static void DoDownload(CookieContainer cc)
{
var noDownloads = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false);
int currentItems = 1;
foreach (var item in noDownloads)
{
cc = DownloadHelper.DownloadManager.Download(item.URL, currentItems, noDownloads.Count(), cc);
}
}
private static void ScanAvList()
{
List <ScanURL> avs = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false).ToList();
Parallel.ForEach(avs, new ParallelOptions {
MaxDegreeOfParallelism = 100
}, av =>
{
ScanEachAv(av, string.Format("进度 {0}/{1}", avs.IndexOf(av) + 1, avs.Count));
});
}
public static CookieContainer Scan(string url, string category, int currentCategory, int totalCategories, CookieContainer cc, bool isUpdate = false)
{
try
{
int page = 1;
while (!string.IsNullOrEmpty(url))
{
var ret = RecursiveHelper(url, category, currentCategory, totalCategories, page, cc, isUpdate);
url = ret.Url;
cc = ret.Cc;
page++;
}
//var json = JsonConvert.SerializeObject(ForUpdate);
var noDownloads = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false);
int current = 1;
if (isUpdate)
{
foreach (var update in ForUpdate)
{
if (!JavDataBaseManager.HasScan(update))
{
cc = DownloadHelper.DownloadManager.Download(update.URL, current, ForUpdate.Count, cc);
}
current++;
}
}
else
{
foreach (var download in noDownloads)
{
if (!JavDataBaseManager.HasScan(download))
{
cc = DownloadHelper.DownloadManager.Download(download.URL, current, noDownloads.Count(), cc);
}
current++;
}
}
}
catch (Exception e)
{
_logger.WriteExceptionLog("", string.Format("Scan failed. {0}", e.ToString()));
}
return(cc);
}
private static void ScanEachAvSingleThread()
{
List <ScanURL> urls = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false).ToList();
int index = 0;
foreach (var url in urls)
{
int retry = 1;
//二次确认
if (!JavDataBaseManager.HasAv(url.URL))
{
var htmlRes = new Utils.HtmlResponse();
//最多重试5次
while (retry <= 5)
{
htmlRes = HtmlManager.GetHtmlWebClientWithRenewCC("http://www.javlibrary.com/cn/", url.URL, cc);
if (htmlRes.IsExpire)
{
GetJavCookie();
retry++;
continue;
}
else
{
break;
}
}
if (htmlRes.Success)
{
index++;
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.LoadHtml(htmlRes.Content);
var av = GenerateAVModel(htmlRes.Content, url.URL);
JavDataBaseManager.InsertAV(av);
Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 插入AV => " + av.ID + " - " + av.Name);
JavDataBaseManager.UpdateScanURL(url.URL);
string result = "";
if (!File.Exists(ImgFolder + av.ID + av.Name + ".jpg"))
{
result = DownloadHelper.DownloadHttps(av.PictureURL, ImgFolder + av.ID + av.Name + ".jpg", "");
if (string.IsNullOrEmpty(result))
{
Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 下载AV图片成功 => " + av.ID + " - " + av.Name);
}
else
{
Console.WriteLine(result);
}
}
else
{
Console.WriteLine("已存在图片不下载");
}
Console.WriteLine("完成" + index + " / " + urls.Count);
}
}
else
{
JavDataBaseManager.UpdateScanURL(url.URL);
Console.WriteLine("详情页已下载 => " + url.URL + " 完成" + index + " / " + urls.Count);
}
}
}
