public void CollectioniQiYiHtmlMovie(int movieID, string name, int timeout = 30000)
{
var res_str = "";
CookieCollection cookie = new CookieCollection();
var url = "http://so.iqiyi.com/so/q_" + name;
try
{
var request = hwru.SendRequest(url, null, null, null, null, "GET", out res_str, out cookie, Encoding.UTF8, timeout, true);
if (request == null || request.StatusCode != HttpStatusCode.OK)
{
if (request.StatusCode == HttpStatusCode.NotFound)
{
throw new Exception();
}
//return CollectionYouKuHtmlMovie(name);
}
else
{
var list = AnalysisPlayer(res_str);
list.ForEach(m =>
{
if (name == m.Name)
{
m.MovieID = movieID;
}
if (player_bll.Save(m) > 0)
{
Console.WriteLine(m.Name + "\t写入数据库成功!");
}
});
}
}
catch (Exception ex)
{
throw ex;
}
}
public void CollectionRARGBHtmlMovieList(int page, out int lastpage, string url = "https://rarbg.is/torrents.php?category=movies", int timeout = 30000)
{
var res_str = "";
CookieCollection cookie = new CookieCollection();
url += ("&page=" + page);
lastpage = 999999999;
try
{
//var list = new List<Model.RARGB.Torrent>();
var request = hwru.SendRequest(url, null, null, null, null, "GET", out res_str, out cookie, Encoding.UTF8, timeout, true);
if (request == null || request.StatusCode != HttpStatusCode.OK || res_str.Contains("Please wait while we try to verify your browser...") || res_str.Contains("We have too many requests from your ip in the past 24h"))
{
if (request.StatusCode == HttpStatusCode.NotFound)
{
throw new Exception();
}
if (res_str.Contains("Please wait while we try to verify your browser...") || res_str.Contains("We have too many requests from your ip in the past 24h"))
{
ChangeIP();
}
CollectionRARGBHtmlMovieList(page, out lastpage);
}
else
{
HtmlDocument hd = new HtmlDocument();
hd.LoadHtml(res_str);
var tr_nodes = hd.DocumentNode.SelectNodes("//tr[@class='lista2']");
if (tr_nodes != null)
{
foreach (var item in tr_nodes)
{
var a_node = item.SelectSingleNode("./td[2]/a[1]");
if (a_node != null)
{
var title = a_node.InnerText.Trim();
var href = a_node.GetAttributeValue("href", "").Trim();
var token = href.Substring(href.LastIndexOf("/") + 1);
if (!string.IsNullOrWhiteSpace(token))
{
if (new BLL.RARGB.Torrent_BLL().InsertToken(new Model.RARGB.RAGGBToken()
{
Token = token,
ActionStatus = 0
}) != null)
{
Console.WriteLine("Token:" + token + "\t写入成功！");
}
else
{
Console.WriteLine("Token:" + token + "\t写入失败！");
}
////先判断Token存不存在
//var obj = rargb_torrent_bll.SingleTorrent(token);
//if (obj==null)
//{
// var thread = new Thread(new ParameterizedThreadStart(Run_CollectionRARGBHtmlMovieTorrent));
// thread.Name = token;
// //OnCollectionTorrentClear += new EventHandler(Thread_OnCollectionTorrentClear);
// thread.Start(token);
// //Console.WriteLine("正在采集："+token);
// //var model = CollectionRARGBHtmlMovieTorrent(token);
// //if (model != null)
// //{
// // list.Add(model);
// //}
//}
}
}
}
}
var lastpage_node = hd.DocumentNode.SelectSingleNode("//div[@id='pager_links']/*[last()]");
if (lastpage_node != null)
{
lastpage = Utils.StrToInt(lastpage_node.InnerText.Trim(), lastpage);
}
}
//return list;
}
catch (Exception ex)
{
throw ex;
}
}
public Model.DouBan.Movie CollectionDouBanHtmlMovie(string doubanID, int timeout = 30000)
{
var res_str = "";
CookieCollection cookie = new CookieCollection();
var url = "https://movie.douban.com/subject/" + doubanID + "/";
try
{
var request = hwru.SendRequest(url, null, null, null, null, "GET", out res_str, out cookie, Encoding.UTF8, timeout, true);
if (request == null || request.StatusCode != HttpStatusCode.OK || res_str.Contains("https://sec.douban.com/a"))
{
if (request.StatusCode == HttpStatusCode.NotFound)
{
throw new Exception();
}
if (res_str.Contains("https://sec.douban.com/a"))
{
ChangeIP();
}
return(CollectionDouBanHtmlMovie(doubanID));
}
else
{
Model.DouBan.Movie movie = new Model.DouBan.Movie();
HtmlDocument hd = new HtmlDocument();
hd.LoadHtml(res_str);
//标题
var title_node = hd.DocumentNode.SelectSingleNode("//span[@property='v:itemreviewed']");
movie.Title = title_node == null ? "" : title_node.InnerText.Trim();
Console.WriteLine("获取到影片：" + movie.Title);
//导演
movie.Director = new List <Model.DouBan.Celebrity>();
var directed_nodes = hd.DocumentNode.SelectNodes("//a[@rel='v:directedBy']");
if (directed_nodes != null)
{
foreach (var item in directed_nodes)
{
var directed_name = item.InnerText.Trim();
var directed_id = item.GetAttributeValue("href", "").Trim();
directed_id = directed_id.Contains("/celebrity") ? directed_id.Replace("/celebrity/", "").Replace("/", "") : null;
if (string.IsNullOrWhiteSpace(directed_id))
{
continue;
}
var directed_obj = douban_movie_bll.SingleCelebrity(directed_id);
if (directed_obj == null)
{
directed_obj = CollectionDouBanHtmlCelebrity(directed_id);
}
movie.Director.Add(directed_obj);
}
}
//编剧
movie.Screenwriter = new List <Model.DouBan.Celebrity>();
var screenwriter_sapn_node = hd.DocumentNode.SelectSingleNode("//span[text()='编剧']");
if (screenwriter_sapn_node != null)
{
var screenwriter_nodes = screenwriter_sapn_node.ParentNode.SelectNodes("./span[@class='attrs']/a");
if (screenwriter_nodes != null)
{
foreach (var item in screenwriter_nodes)
{
var screenwriter_name = item.InnerText.Trim();
var screenwriter_id = item.GetAttributeValue("href", "").Trim();
screenwriter_id = screenwriter_id.Contains("/celebrity") ? screenwriter_id.Replace("/celebrity/", "").Replace("/", "") : null;
if (string.IsNullOrWhiteSpace(screenwriter_id))
{
continue;
}
var screenwriter_obj = douban_movie_bll.SingleCelebrity(screenwriter_id);
if (screenwriter_obj == null)
{
screenwriter_obj = CollectionDouBanHtmlCelebrity(screenwriter_id);
}
movie.Screenwriter.Add(screenwriter_obj);
}
}
}
//明星
movie.Performer = new List <Celebrity>();
var performer_nodes = hd.DocumentNode.SelectNodes("//a[@rel='v:starring']");
if (performer_nodes != null)
{
foreach (var item in performer_nodes)
{
var performer_name = item.InnerText.Trim();
var performer_id = item.GetAttributeValue("href", "").Trim();
performer_id = performer_id.Contains("/celebrity") ? performer_id.Replace("/celebrity/", "").Replace("/", "") : null;
if (string.IsNullOrWhiteSpace(performer_id))
{
continue;
}
var performer_obj = douban_movie_bll.SingleCelebrity(performer_id);
if (performer_obj == null)
{
performer_obj = CollectionDouBanHtmlCelebrity(performer_id);
}
movie.Performer.Add(performer_obj);
}
}
//类型
movie.FilmGenres = new List <FilmGenre>();
var genre_nodes = hd.DocumentNode.SelectNodes("//span[@property='v:genre']");
if (genre_nodes != null)
{
foreach (var item in genre_nodes)
{
var genre_name = item.InnerText.Trim();
if (string.IsNullOrWhiteSpace(genre_name))
{
continue;
}
movie.FilmGenres.Add(new FilmGenre()
{
FilmGenreName = genre_name
});
}
}
//制片国家
movie.Countrys = new List <Country>();
var country_span_node = hd.DocumentNode.SelectSingleNode("//span[text()='制片国家/地区:']");
if (country_span_node != null)
{
var country_span_node_ParentHtml = country_span_node.ParentNode.InnerHtml;
var sub_country_span_node_ParentHtml = country_span_node_ParentHtml.Substring(country_span_node_ParentHtml.IndexOf(country_span_node.OuterHtml) + country_span_node.OuterHtml.Length);
var countrys_str = sub_country_span_node_ParentHtml.Substring(0, sub_country_span_node_ParentHtml.IndexOf("<br>")).Trim();
if (!string.IsNullOrWhiteSpace(countrys_str))
{
var countrys = countrys_str.Split('/');
foreach (var item in countrys)
{
if (!string.IsNullOrWhiteSpace(item))
{
var country_name = item.Trim();
if (string.IsNullOrWhiteSpace(country_name))
{
continue;
}
movie.Countrys.Add(new Country()
{
CountryName = country_name
});
}
}
}
}
//语言
movie.Languages = new List <Language>();
var language_span_node = hd.DocumentNode.SelectSingleNode("//span[text()='语言:']");
if (language_span_node != null)
{
var language_span_node_ParentHtml = language_span_node.ParentNode.InnerHtml;
var sub_language_span_node_ParentHtml = language_span_node_ParentHtml.Substring(language_span_node_ParentHtml.IndexOf(language_span_node.OuterHtml) + language_span_node.OuterHtml.Length);
var languages_str = sub_language_span_node_ParentHtml.Substring(0, sub_language_span_node_ParentHtml.IndexOf("<br>")).Trim();
if (!string.IsNullOrWhiteSpace(languages_str))
{
var languages = languages_str.Split('/');
foreach (var item in languages)
{
if (!string.IsNullOrWhiteSpace(item))
{
var language_name = item.Trim();
if (string.IsNullOrWhiteSpace(language_name))
{
continue;
}
movie.Languages.Add(new Language()
{
LanguageName = language_name
});
}
}
}
}
//上映时间
movie.ReleaseTimes = new List <ReleaseTime>();
var releasetime_nodes = hd.DocumentNode.SelectNodes("//span[@property='v:initialReleaseDate']");
if (releasetime_nodes != null)
{
foreach (var item in releasetime_nodes)
{
var releasetime_text = item.InnerText.Trim();
var releasetime_datestr = releasetime_text.Contains("(") ? releasetime_text.Substring(0, releasetime_text.IndexOf("(")) : releasetime_text;
var releasetime_address = releasetime_text.Replace(releasetime_datestr, "").Replace("(", "").Replace(")", "").Trim();
//对releasetime_date深度加工
var dates = releasetime_datestr.Split('-');
DateTime?releasetime_date = null;
try
{
if (dates.Length > 0)
{
if (dates[0].Length == 4)
{
releasetime_date = new DateTime(Utils.StrToInt(dates[0], 0), dates.Length > 1 ? Utils.StrToInt(dates[1], 1) : 1, dates.Length > 2 ? Utils.StrToInt(dates[2], 1) : 1);
}
}
}
catch (Exception)
{
releasetime_date = null;
}
if (releasetime_date != null)
{
movie.ReleaseTimes.Add(new ReleaseTime()
{
DouBanID = doubanID,
Place = releasetime_address,
ReleaseDate = releasetime_date
});
}
}
}
//片长
movie.LongTime = null;
var runtime_node = hd.DocumentNode.SelectSingleNode("//span[@property='v:runtime']");
if (runtime_node != null)
{
movie.LongTime = Utils.StrToInt(runtime_node.GetAttributeValue("content", "0"), 0);
movie.LongTime = movie.LongTime == 0 ? null : movie.LongTime;
}
//更多名称
movie.MoreName = new List <MovieName>();
var morename_span_node = hd.DocumentNode.SelectSingleNode("//span[text()='又名:']");
if (morename_span_node != null)
{
var morename_span_node_ParentHtml = morename_span_node.ParentNode.InnerHtml;
var sub_morename_span_node_ParentHtml = morename_span_node_ParentHtml.Substring(morename_span_node_ParentHtml.IndexOf(morename_span_node.OuterHtml) + morename_span_node.OuterHtml.Length);
var morenames_str = sub_morename_span_node_ParentHtml.Substring(0, sub_morename_span_node_ParentHtml.IndexOf("<br>")).Trim();
if (!string.IsNullOrWhiteSpace(morenames_str))
{
var morenames = morenames_str.Split('/');
foreach (var item in morenames)
{
var morename = item.Trim();
if (string.IsNullOrWhiteSpace(morename))
{
continue;
}
movie.MoreName.Add(new MovieName()
{
DouBanID = doubanID,
MovieTitle = morename
});
}
}
}
//IMDbID
movie.IMDbID = null;
var imdb_node = hd.DocumentNode.SelectSingleNode("//div[@id='info']/a[@target='_blank'][@rel='nofollow'][last()]");
if (imdb_node != null)
{
movie.IMDbID = string.IsNullOrWhiteSpace(imdb_node.InnerText.Trim()) ? null : imdb_node.InnerText.Trim();
movie.IMDbID = movie.IMDbID.Length > 10 ? null : movie.IMDbID;
}
//评分
movie.Score = null;
var score_node = hd.DocumentNode.SelectSingleNode("//strong[@property='v:average']");
if (score_node != null)
{
movie.Score = Utils.StrToFloat(score_node.InnerText.Trim(), 0);
movie.Score = movie.Score == 0 ? null : movie.Score;
}
//简介
movie.Describe = null;
var summary_node = hd.DocumentNode.SelectSingleNode("//span[@property='v:summary']");
if (summary_node != null)
{
movie.Describe = summary_node.InnerText.Trim();
}
//封面
movie.Poster = null;
movie.PosterID = null;
var poster_node = hd.DocumentNode.SelectSingleNode("//div[@id='mainpic']/a[@class='nbgnbg']/img");
if (poster_node != null)
{
//https://img3.doubanio.com/view/movie_poster_cover/lpst/public/p2457983084.webp
var img_lpst = poster_node.GetAttributeValue("src", "");
if (!string.IsNullOrWhiteSpace(img_lpst) && img_lpst.Contains("lpst"))
{
movie.Poster = new Image()
{
Large = img_lpst,
Medium = img_lpst.Replace("lpst", "spst"),
Small = img_lpst.Replace("lpst", "ipst"),
};
}
}
//DouBanID
movie.DouBanID = doubanID;
return(movie);
}
}
catch (Exception ex)
{
throw ex;
}
}
