public void CollectioniQiYiHtmlMovie(int movieID, string name, int timeout = 30000) { var res_str = ""; CookieCollection cookie = new CookieCollection(); var url = "http://so.iqiyi.com/so/q_" + name; try { var request = hwru.SendRequest(url, null, null, null, null, "GET", out res_str, out cookie, Encoding.UTF8, timeout, true); if (request == null || request.StatusCode != HttpStatusCode.OK) { if (request.StatusCode == HttpStatusCode.NotFound) { throw new Exception(); } //return CollectionYouKuHtmlMovie(name); } else { var list = AnalysisPlayer(res_str); list.ForEach(m => { if (name == m.Name) { m.MovieID = movieID; } if (player_bll.Save(m) > 0) { Console.WriteLine(m.Name + "\t写入数据库成功!"); } }); } } catch (Exception ex) { throw ex; } }
public void CollectionRARGBHtmlMovieList(int page, out int lastpage, string url = "https://rarbg.is/torrents.php?category=movies", int timeout = 30000) { var res_str = ""; CookieCollection cookie = new CookieCollection(); url += ("&page=" + page); lastpage = 999999999; try { //var list = new List<Model.RARGB.Torrent>(); var request = hwru.SendRequest(url, null, null, null, null, "GET", out res_str, out cookie, Encoding.UTF8, timeout, true); if (request == null || request.StatusCode != HttpStatusCode.OK || res_str.Contains("Please wait while we try to verify your browser...") || res_str.Contains("We have too many requests from your ip in the past 24h")) { if (request.StatusCode == HttpStatusCode.NotFound) { throw new Exception(); } if (res_str.Contains("Please wait while we try to verify your browser...") || res_str.Contains("We have too many requests from your ip in the past 24h")) { ChangeIP(); } CollectionRARGBHtmlMovieList(page, out lastpage); } else { HtmlDocument hd = new HtmlDocument(); hd.LoadHtml(res_str); var tr_nodes = hd.DocumentNode.SelectNodes("//tr[@class='lista2']"); if (tr_nodes != null) { foreach (var item in tr_nodes) { var a_node = item.SelectSingleNode("./td[2]/a[1]"); if (a_node != null) { var title = a_node.InnerText.Trim(); var href = a_node.GetAttributeValue("href", "").Trim(); var token = href.Substring(href.LastIndexOf("/") + 1); if (!string.IsNullOrWhiteSpace(token)) { if (new BLL.RARGB.Torrent_BLL().InsertToken(new Model.RARGB.RAGGBToken() { Token = token, ActionStatus = 0 }) != null) { Console.WriteLine("Token:" + token + "\t写入成功!"); } else { Console.WriteLine("Token:" + token + "\t写入失败!"); } ////先判断Token存不存在 //var obj = rargb_torrent_bll.SingleTorrent(token); //if (obj==null) //{ // var thread = new Thread(new ParameterizedThreadStart(Run_CollectionRARGBHtmlMovieTorrent)); // thread.Name = token; // //OnCollectionTorrentClear += new EventHandler(Thread_OnCollectionTorrentClear); // thread.Start(token); // //Console.WriteLine("正在采集:"+token); // //var model = CollectionRARGBHtmlMovieTorrent(token); // //if (model != null) // //{ // // list.Add(model); // //} //} } } } } var lastpage_node = hd.DocumentNode.SelectSingleNode("//div[@id='pager_links']/*[last()]"); if (lastpage_node != null) { lastpage = Utils.StrToInt(lastpage_node.InnerText.Trim(), lastpage); } } //return list; } catch (Exception ex) { throw ex; } }
public Model.DouBan.Movie CollectionDouBanHtmlMovie(string doubanID, int timeout = 30000) { var res_str = ""; CookieCollection cookie = new CookieCollection(); var url = "https://movie.douban.com/subject/" + doubanID + "/"; try { var request = hwru.SendRequest(url, null, null, null, null, "GET", out res_str, out cookie, Encoding.UTF8, timeout, true); if (request == null || request.StatusCode != HttpStatusCode.OK || res_str.Contains("https://sec.douban.com/a")) { if (request.StatusCode == HttpStatusCode.NotFound) { throw new Exception(); } if (res_str.Contains("https://sec.douban.com/a")) { ChangeIP(); } return(CollectionDouBanHtmlMovie(doubanID)); } else { Model.DouBan.Movie movie = new Model.DouBan.Movie(); HtmlDocument hd = new HtmlDocument(); hd.LoadHtml(res_str); //标题 var title_node = hd.DocumentNode.SelectSingleNode("//span[@property='v:itemreviewed']"); movie.Title = title_node == null ? "" : title_node.InnerText.Trim(); Console.WriteLine("获取到影片:" + movie.Title); //导演 movie.Director = new List <Model.DouBan.Celebrity>(); var directed_nodes = hd.DocumentNode.SelectNodes("//a[@rel='v:directedBy']"); if (directed_nodes != null) { foreach (var item in directed_nodes) { var directed_name = item.InnerText.Trim(); var directed_id = item.GetAttributeValue("href", "").Trim(); directed_id = directed_id.Contains("/celebrity") ? directed_id.Replace("/celebrity/", "").Replace("/", "") : null; if (string.IsNullOrWhiteSpace(directed_id)) { continue; } var directed_obj = douban_movie_bll.SingleCelebrity(directed_id); if (directed_obj == null) { directed_obj = CollectionDouBanHtmlCelebrity(directed_id); } movie.Director.Add(directed_obj); } } //编剧 movie.Screenwriter = new List <Model.DouBan.Celebrity>(); var screenwriter_sapn_node = hd.DocumentNode.SelectSingleNode("//span[text()='编剧']"); if (screenwriter_sapn_node != null) { var screenwriter_nodes = screenwriter_sapn_node.ParentNode.SelectNodes("./span[@class='attrs']/a"); if (screenwriter_nodes != null) { foreach (var item in screenwriter_nodes) { var screenwriter_name = item.InnerText.Trim(); var screenwriter_id = item.GetAttributeValue("href", "").Trim(); screenwriter_id = screenwriter_id.Contains("/celebrity") ? screenwriter_id.Replace("/celebrity/", "").Replace("/", "") : null; if (string.IsNullOrWhiteSpace(screenwriter_id)) { continue; } var screenwriter_obj = douban_movie_bll.SingleCelebrity(screenwriter_id); if (screenwriter_obj == null) { screenwriter_obj = CollectionDouBanHtmlCelebrity(screenwriter_id); } movie.Screenwriter.Add(screenwriter_obj); } } } //明星 movie.Performer = new List <Celebrity>(); var performer_nodes = hd.DocumentNode.SelectNodes("//a[@rel='v:starring']"); if (performer_nodes != null) { foreach (var item in performer_nodes) { var performer_name = item.InnerText.Trim(); var performer_id = item.GetAttributeValue("href", "").Trim(); performer_id = performer_id.Contains("/celebrity") ? performer_id.Replace("/celebrity/", "").Replace("/", "") : null; if (string.IsNullOrWhiteSpace(performer_id)) { continue; } var performer_obj = douban_movie_bll.SingleCelebrity(performer_id); if (performer_obj == null) { performer_obj = CollectionDouBanHtmlCelebrity(performer_id); } movie.Performer.Add(performer_obj); } } //类型 movie.FilmGenres = new List <FilmGenre>(); var genre_nodes = hd.DocumentNode.SelectNodes("//span[@property='v:genre']"); if (genre_nodes != null) { foreach (var item in genre_nodes) { var genre_name = item.InnerText.Trim(); if (string.IsNullOrWhiteSpace(genre_name)) { continue; } movie.FilmGenres.Add(new FilmGenre() { FilmGenreName = genre_name }); } } //制片国家 movie.Countrys = new List <Country>(); var country_span_node = hd.DocumentNode.SelectSingleNode("//span[text()='制片国家/地区:']"); if (country_span_node != null) { var country_span_node_ParentHtml = country_span_node.ParentNode.InnerHtml; var sub_country_span_node_ParentHtml = country_span_node_ParentHtml.Substring(country_span_node_ParentHtml.IndexOf(country_span_node.OuterHtml) + country_span_node.OuterHtml.Length); var countrys_str = sub_country_span_node_ParentHtml.Substring(0, sub_country_span_node_ParentHtml.IndexOf("<br>")).Trim(); if (!string.IsNullOrWhiteSpace(countrys_str)) { var countrys = countrys_str.Split('/'); foreach (var item in countrys) { if (!string.IsNullOrWhiteSpace(item)) { var country_name = item.Trim(); if (string.IsNullOrWhiteSpace(country_name)) { continue; } movie.Countrys.Add(new Country() { CountryName = country_name }); } } } } //语言 movie.Languages = new List <Language>(); var language_span_node = hd.DocumentNode.SelectSingleNode("//span[text()='语言:']"); if (language_span_node != null) { var language_span_node_ParentHtml = language_span_node.ParentNode.InnerHtml; var sub_language_span_node_ParentHtml = language_span_node_ParentHtml.Substring(language_span_node_ParentHtml.IndexOf(language_span_node.OuterHtml) + language_span_node.OuterHtml.Length); var languages_str = sub_language_span_node_ParentHtml.Substring(0, sub_language_span_node_ParentHtml.IndexOf("<br>")).Trim(); if (!string.IsNullOrWhiteSpace(languages_str)) { var languages = languages_str.Split('/'); foreach (var item in languages) { if (!string.IsNullOrWhiteSpace(item)) { var language_name = item.Trim(); if (string.IsNullOrWhiteSpace(language_name)) { continue; } movie.Languages.Add(new Language() { LanguageName = language_name }); } } } } //上映时间 movie.ReleaseTimes = new List <ReleaseTime>(); var releasetime_nodes = hd.DocumentNode.SelectNodes("//span[@property='v:initialReleaseDate']"); if (releasetime_nodes != null) { foreach (var item in releasetime_nodes) { var releasetime_text = item.InnerText.Trim(); var releasetime_datestr = releasetime_text.Contains("(") ? releasetime_text.Substring(0, releasetime_text.IndexOf("(")) : releasetime_text; var releasetime_address = releasetime_text.Replace(releasetime_datestr, "").Replace("(", "").Replace(")", "").Trim(); //对releasetime_date深度加工 var dates = releasetime_datestr.Split('-'); DateTime?releasetime_date = null; try { if (dates.Length > 0) { if (dates[0].Length == 4) { releasetime_date = new DateTime(Utils.StrToInt(dates[0], 0), dates.Length > 1 ? Utils.StrToInt(dates[1], 1) : 1, dates.Length > 2 ? Utils.StrToInt(dates[2], 1) : 1); } } } catch (Exception) { releasetime_date = null; } if (releasetime_date != null) { movie.ReleaseTimes.Add(new ReleaseTime() { DouBanID = doubanID, Place = releasetime_address, ReleaseDate = releasetime_date }); } } } //片长 movie.LongTime = null; var runtime_node = hd.DocumentNode.SelectSingleNode("//span[@property='v:runtime']"); if (runtime_node != null) { movie.LongTime = Utils.StrToInt(runtime_node.GetAttributeValue("content", "0"), 0); movie.LongTime = movie.LongTime == 0 ? null : movie.LongTime; } //更多名称 movie.MoreName = new List <MovieName>(); var morename_span_node = hd.DocumentNode.SelectSingleNode("//span[text()='又名:']"); if (morename_span_node != null) { var morename_span_node_ParentHtml = morename_span_node.ParentNode.InnerHtml; var sub_morename_span_node_ParentHtml = morename_span_node_ParentHtml.Substring(morename_span_node_ParentHtml.IndexOf(morename_span_node.OuterHtml) + morename_span_node.OuterHtml.Length); var morenames_str = sub_morename_span_node_ParentHtml.Substring(0, sub_morename_span_node_ParentHtml.IndexOf("<br>")).Trim(); if (!string.IsNullOrWhiteSpace(morenames_str)) { var morenames = morenames_str.Split('/'); foreach (var item in morenames) { var morename = item.Trim(); if (string.IsNullOrWhiteSpace(morename)) { continue; } movie.MoreName.Add(new MovieName() { DouBanID = doubanID, MovieTitle = morename }); } } } //IMDbID movie.IMDbID = null; var imdb_node = hd.DocumentNode.SelectSingleNode("//div[@id='info']/a[@target='_blank'][@rel='nofollow'][last()]"); if (imdb_node != null) { movie.IMDbID = string.IsNullOrWhiteSpace(imdb_node.InnerText.Trim()) ? null : imdb_node.InnerText.Trim(); movie.IMDbID = movie.IMDbID.Length > 10 ? null : movie.IMDbID; } //评分 movie.Score = null; var score_node = hd.DocumentNode.SelectSingleNode("//strong[@property='v:average']"); if (score_node != null) { movie.Score = Utils.StrToFloat(score_node.InnerText.Trim(), 0); movie.Score = movie.Score == 0 ? null : movie.Score; } //简介 movie.Describe = null; var summary_node = hd.DocumentNode.SelectSingleNode("//span[@property='v:summary']"); if (summary_node != null) { movie.Describe = summary_node.InnerText.Trim(); } //封面 movie.Poster = null; movie.PosterID = null; var poster_node = hd.DocumentNode.SelectSingleNode("//div[@id='mainpic']/a[@class='nbgnbg']/img"); if (poster_node != null) { //https://img3.doubanio.com/view/movie_poster_cover/lpst/public/p2457983084.webp var img_lpst = poster_node.GetAttributeValue("src", ""); if (!string.IsNullOrWhiteSpace(img_lpst) && img_lpst.Contains("lpst")) { movie.Poster = new Image() { Large = img_lpst, Medium = img_lpst.Replace("lpst", "spst"), Small = img_lpst.Replace("lpst", "ipst"), }; } } //DouBanID movie.DouBanID = doubanID; return(movie); } } catch (Exception ex) { throw ex; } }