public static void DoScan(List <Category> categories, CookieContainer cc, int skip = 0) { bool isUpdate = false; if (categories.Count == 0) { categories = JavDataBaseManager.GetCategories(); } else { isUpdate = true; } int currentCategory = 1; if (skip > 0) { categories = categories.Skip(skip - 1).ToList(); } foreach (var category in categories) { cc = ScanHelper.ScanManager.Scan(category.Url, category.Name, currentCategory, categories.Count, cc, isUpdate); currentCategory++; } }
private void ProcessJSON() { if (json.Count > 0 && json[index] != null) { labelStatus.Text = string.Format(status, index + 1, json.Count); var pic = imgFolder + json[index].ID + FileUtility.ReplaceInvalidChar(json[index].Title) + ".jpg"; if (File.Exists(pic)) { pictureBox1.Image = Image.FromFile(pic); } var comments = JavDataBaseManager.GetComment(json[index].ID, FileUtility.ReplaceInvalidChar(json[index].Title)); if (ScanDataBaseManager.HasMatch(json[index].ID)) { labelMark.BackColor = Color.Green; } else { labelMark.BackColor = Color.Red; } ShowContent(comments); } }
private void btnSave_Click(object sender, EventArgs e) { if (currentAv != null) { currentAv.Actress = txtActress.Text.Replace("[", "").Replace("]", ""); currentAv.Category = txtCategory.Text.Replace("[", "").Replace("]", ""); currentAv.Company = txtCompany.Text.Replace("[", "").Replace("]", ""); currentAv.Publisher = txtPublisher.Text.Replace("[", "").Replace("]", ""); currentAv.ReleaseDate = DateTime.Parse(txtDate.Text); if (currentAv.ReleaseDate == DateTime.MinValue) { currentAv.ReleaseDate = new DateTime(2050, 1, 1); } if (!File.Exists(imgFolder + currentAv.ID + currentAv.Name + ".jpg")) { DownloadHelper.DownloadFile(currentAv.PictureURL, imgFolder + currentAv.ID + currentAv.Name + ".jpg"); } if (!JavDataBaseManager.HasAv(currentAv.ID, currentAv.Name)) { JavDataBaseManager.InsertAV(currentAv); } tx.Text = currentAv.ID.ToUpper(); this.DialogResult = DialogResult.Yes; this.Close(); } }
private static List <MissingCheckModel> DoAllRelatedJav(string table, string content) { List <MissingCheckModel> ret = new List <MissingCheckModel>(); var avs = JavDataBaseManager.GetAllAV(); switch (table) { case "category": avs = avs.Where(x => x.Category.Contains(content)).ToList(); break; case "actress": avs = avs.Where(x => x.Actress.Contains(content)).ToList(); break; case "prefix": avs = avs.Where(x => x.ID.StartsWith(content.ToUpper() + "-")).ToList(); break; } foreach (var av in avs) { MissingCheckModel mcm = new MissingCheckModel { Av = av, Fi = new List <FileInfo>(), IsMatch = false, Seeds = new List <SeedMagnetSearchModel>() }; ret.Add(mcm); } return(ret.OrderBy(x => x.Av.ID).ToList()); }
public static void GenerateReportDataOnly() { List <ReportItem> items = new List <ReportItem>(); ScanDataBaseManager.DeleteReportItem(); var allAv = JavDataBaseManager.GetAllAV(); var allMatch = GenerateExistingAVs(); Report report = new Report(); report.ExtensionModel = new Dictionary <string, int>(); report.TotalCount = allAv.Count; var reportId = ScanDataBaseManager.InsertReport(report); report.ReportId = reportId; int process = 0; foreach (var av in allAv) { ProcessReportType(av, allMatch, report, items); process++; Console.WriteLine(process + " / " + allAv.Count); } ; ScanDataBaseManager.BatchInserReportItem(items); ScanDataBaseManager.UpdateReport(report); ScanDataBaseManager.UpdateReportFinish(reportId); }
static void Main(string[] args) { Dictionary <string, AV> match = new Dictionary <string, AV>(); ReportModel report = new ReportModel(); report.Top = 10; Console.WriteLine("正在获取AV爬虫信息..."); var filesContainer = ReportHelper.GenerateExistingAVs(); Console.WriteLine("正在获取本地AV信息..."); var avs = JavDataBaseManager.GetAllAV(); Console.WriteLine("正在生成基本报表..."); ReportHelper.GenerateBasicReport(filesContainer, avs, match, report); Console.WriteLine("正在生成匹配报表..."); ReportHelper.GenerateMatchReport(filesContainer, avs, match, report); Console.WriteLine("正在生成报表其他部分..."); ReportHelper.GenerateOtherReport(filesContainer, avs, match, report); Console.WriteLine(ReportHelper.GenerateActuralReport(report)); //ReportHelper.Test(filesContainer, avs, match, report); Console.ReadKey(); }
private static bool ProcessCategory(string content) { try { JavDataBaseManager.DeleteCategory(); var m = Regex.Matches(content, categoryPattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); foreach (Match item in m) { Category c = new Category { Url = prefix + categoryPrefix + item.Groups[1].Value + postfix, Name = item.Groups[2].Value }; Console.WriteLine(string.Format("Get category {0}, URL {1}", c.Name, c.Url)); JavDataBaseManager.InsertCategory(c); } } catch (Exception e) { _logger.WriteExceptionLog("", string.Format("Process category error {0}", e.ToString())); return(false); } return(true); }
private static Dictionary <string, string> GetJavCategory() { Dictionary <string, string> genreDic = new Dictionary <string, string>(); int times = 1; int maxTimes = 3; //最大重试3次 while (times <= maxTimes && (cc == null || cc.Count < 3)) { GetJavCookie(); times++; if (cc != null && cc.Count >= 3) { break; } } if (times < 4 && cc != null) { //获取分类, 不会过期也不需要多线程 var htmlRes = HtmlManager.GetHtmlWebClient("http://www.javlibrary.com/cn/", "http://www.javlibrary.com/cn/genres.php", cc, false); if (htmlRes.Success) { HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument(); htmlDocument.LoadHtml(htmlRes.Content); var genrePath = "//div[@class='genreitem']"; var genreNodes = htmlDocument.DocumentNode.SelectNodes(genrePath); foreach (var node in genreNodes) { var aTagHref = "http://www.javlibrary.com/cn/" + node.ChildNodes[0].Attributes["href"].Value.Trim(); var aTagTitle = node.ChildNodes[0].InnerText.Trim(); if (!JavDataBaseManager.HasCategoryByName(aTagTitle)) { JavDataBaseManager.InsertCategory(new Category { Name = aTagTitle, Url = aTagHref }); } if (!genreDic.ContainsKey(aTagHref)) { genreDic.Add(aTagHref, aTagTitle); Console.WriteLine("加入分类 " + aTagHref); } } } } return(genreDic); }
public static void WriteLog(JavLibraryLog log) { JavDataBaseManager.WriteLog(log); if (!string.IsNullOrEmpty(log.URL)) { JavDataBaseManager.WriteSecondTry(log); } }
private static void GetAVFromIdAndName() { var folder = "E:\\New folder\\Fin"; var files = Directory.GetFiles(folder); int total = files.Length; int match = 0; List <string> unmatched = new List <string>(); foreach (var file in files) { FileInfo fi = new FileInfo(file); var split = fi.Name.Split('-'); if (split.Length >= 3) { var id = split[0] + "-" + split[1]; var nameAfter = fi.Name.Replace(id, "").Split('-'); string name = ""; if (nameAfter.Length == 0) { name = fi.Name.Replace(id, "").Replace(fi.Extension, ""); } else { name = nameAfter[0]; foreach (var slice in nameAfter) { if (slice.Length > 1) { name += "-" + slice; } } } var avs = JavDataBaseManager.GetAllAV(id, name); if (avs != null && avs.Count == 1) { match++; } else { unmatched.Add(fi.FullName); } } } foreach (var unmatch in unmatched) { Console.WriteLine(unmatch); } Console.WriteLine("Total: " + total + " Matched: " + match); }
public static void UpdateJavBusCategory(CookieContainer cc) { Dictionary <string, string> ret = new Dictionary <string, string>(); List <string> busCategory = new List <string>(); var url = "https://www.javbus.com/genre"; var mapping = JavBusDownloadHelper.GetJavCategoryMapping(MappingFile); List <string> javLibrary = GetJavLibraryCategory(); var content = HtmlManager.GetHtmlContentViaUrl(url, "utf-8", false, cc); if (content.Success) { HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.LoadHtml(content.Content); string xpath = "//a[@class='col-lg-2 col-md-2 col-sm-3 col-xs-6 text-center']"; HtmlNodeCollection nodes = htmlDocument.DocumentNode.SelectNodes(xpath); foreach (var node in nodes) { busCategory.Add(FileUtility.GetJianTiStr(node.InnerText.Trim())); } } var copyBus = string.Join("\r\n", busCategory); var copyLib = string.Join("\r\n", javLibrary); for (int i = 0; i < busCategory.Count; i++) { if (javLibrary.Contains(busCategory[i])) { if (!ret.ContainsKey(busCategory[i])) { ret.Add(busCategory[i], javLibrary.FirstOrDefault(x => x == busCategory[i])); } } else { if (mapping.ContainsKey(busCategory[i])) { ret.Add(busCategory[i], mapping[busCategory[i]]); } } } if (ret.Count > 0) { JavDataBaseManager.DeleteJavBusCategory(); foreach (var map in ret) { JavDataBaseManager.InsertJavBusCategory(map.Key, map.Value); } } }
private void FindAv(string id) { var avs = JavDataBaseManager.GetAllAV(id); if (avs != null && avs.Count > 0) { ShowAv(avs); } }
private static void ScanEachAv(ScanURL url, string status, bool force = false) { AV av = new AV(); if (!JavDataBaseManager.HasAv(url.URL) || force) { var htmlRes = JavCookieContanierHelper(url.URL); if (htmlRes.Success) { HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument(); htmlDocument.LoadHtml(htmlRes.Content); av = GenerateAVModel(htmlRes.Content, url.URL); if (!force) { JavDataBaseManager.InsertAV(av); Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 插入AV => " + av.ID + " - " + av.Name + " " + status); JavDataBaseManager.UpdateScanURL(url.URL); Console.WriteLine("AV:" + JsonConvert.SerializeObject(new RefreshModel { Id = av.ID, Name = av.Name, Url = av.PictureURL }) ); } string result = ""; if (!File.Exists(ImgFolder + av.ID + av.Name + ".jpg")) { result = DownloadHelper.DownloadHttps(av.PictureURL, ImgFolder + av.ID + av.Name + ".jpg", ""); if (string.IsNullOrEmpty(result)) { Console.WriteLine("线程 " + Thread.CurrentThread.ManagedThreadId.ToString() + " => 下载AV图片成功 => " + av.ID + " - " + av.Name); } else { Console.WriteLine(result); } } else { Console.WriteLine("已存在图片不下载"); } } } else { Console.WriteLine("已存在 => " + url.URL + " " + status); JavDataBaseManager.UpdateScanURL(url.URL); } }
private void btnPublisher_Click(object sender, EventArgs e) { if (!string.IsNullOrEmpty(txtPossible.Text)) { var list = JavDataBaseManager.GetSimilarContent("publisher", txtPossible.Text); txtResult.Text = string.Join(",", list.Select(x => x.Name)); } }
private void ShowRenameDetail() { RenameInitPartial(); var file = renameFi[indexOfRename]; currentFi = file; var scan = new Scan() { FileName = file.FullName, Location = file.DirectoryName, Size = Utils.FileSize.GetAutoSizeString(file.Length, 1) }; txtRenameFinal.Text = ""; pbRename.Maximum = renameFi.Count; pbRename.Value = indexOfRename; txtRenameOri.Text = currentFi.FullName; txtRenameFinal.Text = currentFi.FullName; if (file != null) { var ids = FileUtility.GetPossibleID(scan, prefixs); if (ids.Count > 0) { currentAVS = new List <AV>(); currentAVS.AddRange(JavDataBaseManager.GetAllAV(ids)); if (currentAVS.Count > 0) { indexOfCurrentAV = 0; ShowRenameInfo(); if (currentAVS.Count > 1) { btRenameLeft.Show(); btRenameRight.Show(); } else { btRenameLeft.Hide(); btRenameRight.Hide(); } } else { //MessageBox.Show("Not found, please add manually."); } } else { //MessageBox.Show("Not found, please add manually."); } } }
public static List <CommonModel> GetNames(string type) { List <CommonModel> ret = new List <CommonModel>(); JavDataBaseManager.GetCommonMode(type).ForEach(x => ret.Add(new CommonModel { Name = x.Name, Type = type, Url = "http://www.cainqs.com:8087/avapi/getav?" + type + "=" + x.Name })); return(ret); }
public static void DoDownload(CookieContainer cc) { var noDownloads = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false); int currentItems = 1; foreach (var item in noDownloads) { cc = DownloadHelper.DownloadManager.Download(item.URL, currentItems, noDownloads.Count(), cc); } }
private static void ScanCategoryPageUrl(string url, string cate, int current, int total, List <string> scans = null) { var htmlRes = JavCookieContanierHelper(url); if (htmlRes.Success) { HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument(); htmlDocument.LoadHtml(htmlRes.Content); var videoPath = "//div[@class='video']"; var videoNodes = htmlDocument.DocumentNode.SelectNodes(videoPath); if (videoNodes != null) { int unScanCount = 0; foreach (var node in videoNodes) { var urlAndTitle = node.ChildNodes[0]; if (urlAndTitle != null && urlAndTitle.ChildNodes.Count >= 3) { var id = urlAndTitle.ChildNodes[0].InnerText.Trim(); var name = FileUtility.ReplaceInvalidChar(urlAndTitle.ChildNodes[2].InnerText.Trim()); var avUrl = urlAndTitle.Attributes["href"].Value.Trim().Replace("./", "http://www.javlibrary.com/cn/"); if (!string.IsNullOrEmpty(avUrl) && !string.IsNullOrEmpty(name) && !string.IsNullOrWhiteSpace(id)) { ScanURL scan = new ScanURL { Category = url, ID = id, IsDownload = false, Title = name, URL = avUrl }; if (!JavDataBaseManager.HasScan(scan)) { unScanCount++; JavDataBaseManager.InsertScanURL(scan); if (scans != null) { scans.Add(avUrl); } } } } } Console.WriteLine(cate + " " + url + " 扫描了 " + unScanCount + " 未扫描, 进度" + current + " / " + total); } } }
public ActionResult ScanJav() { #region 页面 List <ScanMap> page = new List <ScanMap>(); page.Add(new ScanMap() { Title = "新话题", Url = "http://www.javlibrary.com/cn/vl_update.php?mode=" }); page.Add(new ScanMap() { Title = "新加入", Url = "http://www.javlibrary.com/cn/vl_newentries.php?mode=" }); page.Add(new ScanMap() { Title = "最想要", Url = "http://www.javlibrary.com/cn/vl_mostwanted.php?mode=" }); page.Add(new ScanMap() { Title = "高评价", Url = "http://www.javlibrary.com/cn/vl_bestrated.php?mode=" }); page.Add(new ScanMap() { Title = "新发行", Url = "http://www.javlibrary.com/cn/vl_newrelease.php?mode=" }); ViewData.Add("page", page); #endregion #region 演员 var actress = JavDataBaseManager.GetActress(); ViewData.Add("actress", actress); #endregion #region 类型 var cate = JavDataBaseManager.GetCategories(); ViewData.Add("cate", cate); #endregion #region 收藏 var faviModel = ScanDataBaseManager.GetFaviScan(); var favi = faviModel.GroupBy(x => x.Category).ToDictionary(x => x.Key, x => x.ToList()); ViewData.Add("favi", favi); #endregion return(View()); }
private static void ScanAvList() { List <ScanURL> avs = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false).ToList(); Parallel.ForEach(avs, new ParallelOptions { MaxDegreeOfParallelism = 100 }, av => { ScanEachAv(av, string.Format("进度 {0}/{1}", avs.IndexOf(av) + 1, avs.Count)); }); }
public static List <AV> GetClosetMatch(string inputName) { List <AV> ret = new List <AV>(); List <string> allPossibleMatch = new List <string>(); inputName = inputName.ToUpper(); if (!CacheTools.HasCache("cache-prefix")) { UpdateCache(); } var allPrefix = CacheTools.GetCache <List <string> >("cache-prefix"); foreach (var prefix in allPrefix) { if (inputName.Contains(prefix)) { var pattern = prefix + "{1}-?\\d{1,5}"; var possibleId = Regex.Match(inputName, pattern).Value; if (possibleId.Contains("-")) { allPossibleMatch.Add(possibleId); } else { bool isFirst = true; StringBuilder sb = new StringBuilder(); foreach (var c in possibleId) { if (c >= '0' && c <= '9') { if (isFirst) { sb.Append("-"); isFirst = false; } } sb.Append(c); } allPossibleMatch.Add(sb.ToString()); } } } ret = JavDataBaseManager.GetAllAV(allPossibleMatch); ret = ret.OrderByDescending(x => x.AvLength).ToList(); return(ret); }
private void RefreshCache() { foreach (var name in JavDataBaseManager.GetAllAV().Select(x => x.ID).ToList()) { if (!allPrefix.Contains(name.Split('-')[0])) { allPrefix.Add(name.Split('-')[0]); } } allPrefix = allPrefix.OrderByDescending(x => x.Length).ToList(); }
private void InitDropDown() { var actress = JavDataBaseManager.GetDropDownList("Actress"); var company = JavDataBaseManager.GetDropDownList("Company"); var director = JavDataBaseManager.GetDropDownList("Director"); var publisher = JavDataBaseManager.GetDropDownList("Publisher"); var category = JavDataBaseManager.GetDropDownList("Category"); var actressSource = new AutoCompleteStringCollection(); actressSource.AddRange(actress); var companySource = new AutoCompleteStringCollection(); companySource.AddRange(company); var directorSource = new AutoCompleteStringCollection(); directorSource.AddRange(director); var publisherSource = new AutoCompleteStringCollection(); publisherSource.AddRange(publisher); var categorySource = new AutoCompleteStringCollection(); categorySource.AddRange(category); cbActress.AutoCompleteCustomSource = actressSource; cbActress.AutoCompleteMode = AutoCompleteMode.SuggestAppend; cbActress.AutoCompleteSource = AutoCompleteSource.CustomSource; cbCompany.AutoCompleteCustomSource = companySource; cbCompany.AutoCompleteMode = AutoCompleteMode.SuggestAppend; cbCompany.AutoCompleteSource = AutoCompleteSource.CustomSource; cbDirector.AutoCompleteCustomSource = directorSource; cbDirector.AutoCompleteMode = AutoCompleteMode.SuggestAppend; cbDirector.AutoCompleteSource = AutoCompleteSource.CustomSource; cbPublisher.AutoCompleteCustomSource = publisherSource; cbPublisher.AutoCompleteMode = AutoCompleteMode.SuggestAppend; cbPublisher.AutoCompleteSource = AutoCompleteSource.CustomSource; cbCategory.AutoCompleteCustomSource = categorySource; cbCategory.AutoCompleteMode = AutoCompleteMode.SuggestAppend; cbCategory.AutoCompleteSource = AutoCompleteSource.CustomSource; pageModel = new PageModel(); lbCurrent.Text = "0"; lbTotal.Text = "0"; }
private static void UpdateCache() { List <string> allPrefix = new List <string>(); foreach (var name in JavDataBaseManager.GetAllAVId().Select(x => x.ID).ToList()) { if (!allPrefix.Contains(name.Split('-')[0])) { allPrefix.Add(name.Split('-')[0]); ScanDataBaseManager.InsertPrefix(name.Split('-')[0]); } } }
public static CookieContainer Scan(string url, string category, int currentCategory, int totalCategories, CookieContainer cc, bool isUpdate = false) { try { int page = 1; while (!string.IsNullOrEmpty(url)) { var ret = RecursiveHelper(url, category, currentCategory, totalCategories, page, cc, isUpdate); url = ret.Url; cc = ret.Cc; page++; } //var json = JsonConvert.SerializeObject(ForUpdate); var noDownloads = JavDataBaseManager.GetScanURL().Where(x => x.IsDownload == false); int current = 1; if (isUpdate) { foreach (var update in ForUpdate) { if (!JavDataBaseManager.HasScan(update)) { cc = DownloadHelper.DownloadManager.Download(update.URL, current, ForUpdate.Count, cc); } current++; } } else { foreach (var download in noDownloads) { if (!JavDataBaseManager.HasScan(download)) { cc = DownloadHelper.DownloadManager.Download(download.URL, current, noDownloads.Count(), cc); } current++; } } } catch (Exception e) { _logger.WriteExceptionLog("", string.Format("Scan failed. {0}", e.ToString())); } return(cc); }
private void btnRenameSearch_Click(object sender, EventArgs e) { currentAVS = new List <AV>(); currentAVS.AddRange(JavDataBaseManager.GetAllAV(txtKeyword.Text)); if (currentAVS.Count > 0) { indexOfCurrentAV = 0; ShowRenameInfo(); } else { //MessageBox.Show("Not found, please add manually."); } }
private static void RestoreJavFromJson(string jsonFile) { if (File.Exists(jsonFile)) { StreamReader sr = new StreamReader(jsonFile); var json = sr.ReadToEnd(); sr.Close(); List <AV> data = JsonConvert.DeserializeObject <List <AV> >(json); foreach (var av in data) { JavDataBaseManager.InsertAV(av); } } }
private static void AddTemp(Scan scan, List <string> possibleIDs, List <Match> temp) { foreach (var id in possibleIDs) { var avs = JavDataBaseManager.GetAllAV(id); foreach (var av in avs) { temp.Add(new Match { AvID = av.ID.ToLower(), Location = scan.Location.ToLower(), Name = scan.FileName.ToLower() }); } } }
private static void ClearJav() { var all = JavDataBaseManager.GetAllAV(); var idDIc = all.GroupBy(x => x.ID).ToDictionary(x => x.Key, x => x.ToList()).Where(x => x.Value.Count > 1).ToDictionary(x => x.Key, x => x); var urlDic = all.GroupBy(x => x.URL).ToDictionary(x => x.Key, x => x.ToList()).Where(x => x.Value.Count > 1).ToDictionary(x => x.Key, x => x); var fileName = "G:\\JavAvBak.json"; File.Create(fileName).Close(); StreamWriter sw = new StreamWriter(fileName); sw.WriteLine(JsonConvert.SerializeObject(all)); sw.Close(); }
private void saveBtn_Click(object sender, EventArgs e) { if (av != null) { if (!JavDataBaseManager.HasAv(av.URL)) { JavDataBaseManager.InsertAV(av); } string result = ""; if (!File.Exists(imgFolder + av.ID + av.Name + ".jpg")) { result = DownloadHelper.DownloadFile(av.PictureURL, imgFolder + av.ID + av.Name + ".jpg"); } this.DialogResult = DialogResult.Yes; this.Close(); } }