public static CookieContainer StartDownload(string url, string oriURL, int currentItem, int totalItem, CookieContainer cc) { var ret = InitHelper.InitManager.UpdateCookie(cc, url); cc = ret.CC; var res = ret.Content; try { if (res.Success) { _logger.WriteLog(url, string.IsNullOrWhiteSpace(res.Content) ? "çİş" : res.Content); Console.WriteLine(string.Format("Start to download {0}, {1}/{2}", oriURL, currentItem, totalItem)); AV av = new AV(); var m = Regex.Matches(res.Content, detailIDPattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); foreach (Match item in m) { var data = item.Groups[1].Value; Console.WriteLine(string.Format("Get AV {0}, ID -> {1}", url, data)); av.ID = data; } m = Regex.Matches(res.Content, detailTitlePattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); foreach (Match item in m) { var data = item.Groups[2].Value.Replace(av.ID + " ", ""); Console.WriteLine(string.Format("Get AV {0}, Title -> {1}", url, data)); av.Name = FileUtility.ReplaceInvalidChar(data); } m = Regex.Matches(res.Content, detailImgPattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); foreach (Match item in m) { var data = item.Groups[1].Value.StartsWith("http") ? item.Groups[1].Value : "http:" + item.Groups[1].Value; Console.WriteLine(string.Format("Get AV {0}, IMG -> {1}", url, data)); av.PictureURL = data; } m = Regex.Matches(res.Content, detailDatePattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); foreach (Match item in m) { var data = item.Groups[1].Value; Console.WriteLine(string.Format("Get AV {0}, Date -> {1}", url, data)); av.ReleaseDate = DateTime.Parse(data); } m = Regex.Matches(res.Content, detailLengthPattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); foreach (Match item in m) { var data = item.Groups[1].Value; Console.WriteLine(string.Format("Get AV {0}, Length -> {1}", url, data)); av.AvLength = int.Parse(data); } m = Regex.Matches(res.Content, detailDirectorPattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); foreach (Match item in m) { var u = item.Groups[1].Value; var data = item.Groups[2].Value; Console.WriteLine(string.Format("Get AV {0}, Director -> {1}", url, data)); av.Director += data + ","; Director d = new Director { CreateTime = DateTime.Now, Name = data, URL = prefix + directorPrefix + u }; if (!JavDataBaseManager.HasDirector(d.URL)) { JavDataBaseManager.InsertDirector(d); } } m = Regex.Matches(res.Content, detailCompanyPattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); foreach (Match item in m) { var u = item.Groups[1].Value; var data = item.Groups[2].Value; Console.WriteLine(string.Format("Get AV {0}, Company -> {1}", url, data)); av.Company += data + ","; Company c = new Company { CreateTime = DateTime.Now, Name = data, URL = prefix + companyPrefix + u }; if (!JavDataBaseManager.HasCompany(c.URL)) { JavDataBaseManager.InsertCompany(c); } } m = Regex.Matches(res.Content, detailPublisherPattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); foreach (Match item in m) { var u = item.Groups[1].Value; var data = item.Groups[2].Value; Console.WriteLine(string.Format("Get AV {0}, Publisher -> {1}", url, data)); av.Publisher += data + ","; Publisher p = new Publisher { CreateTime = DateTime.Now, Name = data, URL = prefix + publisherPrefix + u }; if (!JavDataBaseManager.HasPublisher(p.URL)) { JavDataBaseManager.InsertPublisher(p); } } m = Regex.Matches(res.Content, detailCategoryPattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); foreach (Match item in m) { var data = item.Groups[2].Value; Console.WriteLine(string.Format("Get AV {0}, Category -> {1}", url, data)); av.Category += data + ","; } m = Regex.Matches(res.Content, detailActressPattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); foreach (Match item in m) { var u = item.Groups[1].Value; var data = item.Groups[2].Value; Console.WriteLine(string.Format("Get AV {0}, Actress -> {1}", url, data)); av.Actress += data + ","; Actress a = new Actress { CreateTime = DateTime.Now, Name = data, URL = prefix + actressPrefix + u }; if (!JavDataBaseManager.HasActress(a.URL)) { JavDataBaseManager.InsertActress(a); } } //m = Regex.Matches(res.Content, detailCommentPattern, RegexOptions.Multiline | RegexOptions.IgnoreCase); //foreach (Match item in m) //{ // var data = item.Groups[1].Value; // Console.WriteLine(string.Format("Get AV {0}, Comments -> {1}", url, data)); // Comments c = new Comments // { // Comment = data, // AvID = av.ID, // AvTitle = av.Name, // CreateTime = DateTime.Now // }; // if (!JavDataBaseManager.HasComment(c)) // { // JavDataBaseManager.InsertComment(c); // } //} av.URL = url; if (!JavDataBaseManager.HasAv(av.URL)) { JavDataBaseManager.InsertAV(av); } string result = ""; if (!File.Exists(imgFolder + av.ID + av.Name + ".jpg")) { result = Utils.DownloadHelper.DownloadFile(av.PictureURL, imgFolder + av.ID + av.Name + ".jpg"); } JavDataBaseManager.UpdateScanURL(oriURL); if (!string.IsNullOrEmpty(result)) { _logger.WriteExceptionLog(url, string.Format("Download picture failed {0}", imgFolder + av.ID + av.Name + ".jpg")); } } else { _logger.WriteExceptionLog(url, string.Format("Download failed {0}", url)); } } catch (Exception e) { _logger.WriteExceptionLog(url, string.Format("Download failed {0}", e.ToString())); } return(cc); }
private static AV GenerateAVModel(string html, string avUrl) { AV av = new AV(); HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument(); htmlDocument.LoadHtml(html); var titlePath = "//h3[@class='post-title text']"; var picPath = "//img[@id='video_jacket_img']"; var releasdPath = "//div[@id='video_date']//td[@class='text']"; var lengthPath = "//div[@id='video_length']//span[@class='text']"; var dirPath = "//span[@class='director']//a"; var comPath = "//span[@class='maker']//a"; var pubPath = "//span[@class='label']//a"; var catPath = "//span[@class='genre']//a"; var staPath = "//span[@class='star']//a"; var titleNode = htmlDocument.DocumentNode.SelectSingleNode(titlePath); var title = titleNode.InnerText.Trim(); var id = title.Substring(0, title.IndexOf(" ")); title = FileUtility.ReplaceInvalidChar(title.Substring(title.IndexOf(" ") + 1)); var picUrl = htmlDocument.DocumentNode.SelectSingleNode(picPath); av.URL = avUrl; av.PictureURL = picUrl.Attributes["src"].Value; av.PictureURL = av.PictureURL.StartsWith("http") ? av.PictureURL : "http:" + av.PictureURL; av.Name = title; av.ID = id; var release = htmlDocument.DocumentNode.SelectSingleNode(releasdPath); DateTime rDate = new DateTime(2050, 1, 1); if (release != null && !string.IsNullOrEmpty(release.InnerText)) { DateTime.TryParse(release.InnerText.Trim(), out rDate); if (rDate <= DateTime.MinValue) { rDate = new DateTime(2050, 1, 1); } } av.ReleaseDate = rDate; var length = htmlDocument.DocumentNode.SelectSingleNode(lengthPath); if (length != null && !string.IsNullOrEmpty(length.InnerText)) { av.AvLength = int.Parse(length.InnerText.Trim()); } var dirNode = htmlDocument.DocumentNode.SelectNodes(dirPath); if (dirNode != null) { foreach (var dir in dirNode) { var name = dir.InnerHtml.Trim(); var url = "http://www.javlibrary.com/cn/" + dir.Attributes["href"].Value; Director d = new Director { CreateTime = DateTime.Now, Name = name, URL = url }; if (!JavDataBaseManager.HasDirector(d.URL)) { JavDataBaseManager.InsertDirector(d); } av.Director += name + ","; } } var comNode = htmlDocument.DocumentNode.SelectNodes(comPath); if (comNode != null) { foreach (var com in comNode) { var name = com.InnerHtml.Trim(); var url = "http://www.javlibrary.com/cn/" + com.Attributes["href"].Value; Company c = new Company { CreateTime = DateTime.Now, Name = name, URL = url }; if (!JavDataBaseManager.HasCompany(c.URL)) { JavDataBaseManager.InsertCompany(c); } av.Company += name + ","; } } var pubNode = htmlDocument.DocumentNode.SelectNodes(pubPath); if (pubNode != null) { foreach (var pub in pubNode) { var name = pub.InnerHtml.Trim(); var url = "http://www.javlibrary.com/cn/" + pub.Attributes["href"].Value; Publisher p = new Publisher { CreateTime = DateTime.Now, Name = name, URL = url }; if (!JavDataBaseManager.HasPublisher(p.URL)) { JavDataBaseManager.InsertPublisher(p); } av.Publisher += name + ","; } } var catNodes = htmlDocument.DocumentNode.SelectNodes(catPath); if (catNodes != null) { foreach (var cat in catNodes) { var name = cat.InnerHtml.Trim(); var url = "http://www.javlibrary.com/cn/" + cat.Attributes["href"].Value; av.Category += name + ","; } } var starNodes = htmlDocument.DocumentNode.SelectNodes(staPath); if (starNodes != null) { foreach (var star in starNodes) { var name = star.InnerHtml.Trim(); var url = "http://www.javlibrary.com/cn/" + star.Attributes["href"].Value; Actress a = new Actress { CreateTime = DateTime.Now, Name = name, URL = url }; if (!JavDataBaseManager.HasActress(a.URL)) { JavDataBaseManager.InsertActress(a); } av.Actress += name + ","; } } return(av); }