/// <summary> /// 获取影片详情 /// </summary> /// <param name="url"></param> void GetMovieInfo(string url, string name, Family family) { //发送请求 var httpResult = http.GetHtml(new HttpItem() { URL = url, Accept = "text/html", ReadWriteTimeout = 1000 * 60, Timeout = 1000 * 60 }); //判断请求是否成功 if (httpResult.StatusCode == System.Net.HttpStatusCode.OK) { //处理json if (httpResult.Html.Contains("var PageConfig = ")) { var json = httpResult.Html.GetSubstr_("var PageConfig = ", ";"); if (!string.IsNullOrWhiteSpace(json)) { var info = json.ParseJSON <Models.YouKu.Rootobject>(); if (info != null) { if (!string.IsNullOrWhiteSpace(info.doubanId)) { ////判断该豆瓣ID是否已经存在 //int? movieId = movie_bll.GetMovieIDByDouBanID(info.doubanId.ToString()); //if (!movieId.HasValue) //{ // //根据豆瓣ID获取详细信息 // movieId = douBanCollection.CollectionDouBanInfo(info.doubanId.ToString()); //} //else // Console.WriteLine(family.ToString_() + ":\t" + name + "\t已经被收录!"); int?movieId = douBanCollection.CollectionDouBanInfo(info.doubanId.ToString()); if (movieId.HasValue) { Console.WriteLine("正在采集" + name + "的播放地址..."); //获取下载地址 CollectionMoviePlayer(movieId.Value, info.showid, family); } } } } } } else { Console.WriteLine("请求发生错误\tStatusCode:" + httpResult.StatusCode); } }
/// <summary> /// 获取影片详情 /// </summary> /// <param name="url"></param> void GetMovieInfo(string vid, string name, Family family) { var url = "https://v.qq.com/detail/" + vid.Substring(0, 1) + "/" + vid + ".html"; //发送请求 var httpResult = http.GetHtml(new HttpItem() { URL = url, Accept = "text/html", ReadWriteTimeout = 1000 * 60, Timeout = 1000 * 60 }); //判断请求是否成功 if (httpResult.StatusCode == System.Net.HttpStatusCode.OK) { //将html先解析 HtmlDocument hd = new HtmlDocument(); hd.LoadHtml(httpResult.Html); var score_dbNode = hd.DocumentNode.SelectSingleNode("//a[@class='score_db']"); if (score_dbNode != null) { var doubanUrl = score_dbNode.GetAttributeValue("href", ""); var doubanID = ""; if (!string.IsNullOrWhiteSpace(doubanUrl)) { doubanID = doubanUrl.Replace("https://movie.douban.com/subject/", "").TrimEnd('/').Trim(); } if (!string.IsNullOrWhiteSpace(doubanID)) { ////判断该豆瓣ID是否已经存在 //int? movieId = movie_bll.GetMovieIDByDouBanID(doubanID); //if (!movieId.HasValue) //{ // //根据豆瓣ID获取详细信息 // movieId = douBanCollection.CollectionDouBanInfo(doubanID); //} //else // Console.WriteLine(family.ToString_() + ":\t" + name + "\t已经被收录!"); int?movieId = douBanCollection.CollectionDouBanInfo(doubanID); if (movieId.HasValue) { Console.WriteLine("正在采集" + name + "的播放地址..."); CollectionMoviePlayer(movieId.Value, vid, name, family); } } } } else { Console.WriteLine("请求发生错误\tStatusCode:" + httpResult.StatusCode); } }
/// <summary> /// 爬取电影列表 /// </summary> /// <param name="family"></param> public void CollectionMovies(Family family) { //发送请求 var httpResult = http.GetHtml(new HttpItem() { URL = (family == Family.电视剧 ? "https://gaoqing.fm/api/listtv?limit=500000&page=" : "https://gaoqing.fm/api/listfilm?limit=500000&page=") + 1, Accept = "application/json", ReadWriteTimeout = 1000 * 60, Timeout = 1000 * 60 }); //判断请求是否成功 if (httpResult.StatusCode == System.Net.HttpStatusCode.OK) { //将结果处理成实体集合 var list = httpResult.Html.ParseJSON <List <Models.GaoQing.GaoQing> >().OrderBy(o => o.nd).ToList(); Console.WriteLine("本次共采集到" + family.ToString_() + ":\t" + list.Count + "条..."); list.ForEach_(m => { ////判断该豆瓣ID是否已经存在 //int? movieId = movie_bll.GetMovieIDByDouBanID(m.subject); //if (!movieId.HasValue) //{ // //根据豆瓣ID获取详细信息 // movieId = douBanCollection.CollectionDouBanInfo(m.subject); //} //else // Console.WriteLine(family.ToString_() + ":\t" + m.name + "\t已经被收录!"); int?movieId = douBanCollection.CollectionDouBanInfo(m.subject); if (movieId.HasValue) { Console.WriteLine("正在采集" + m.name + "的资源..."); //获取下载地址 CollectionMovieLib(movieId.Value, m.hash, m.name); //new Task(() => //{ // Console.WriteLine("正在采集" + m.name + "的资源..."); // //获取下载地址 // CollectionMovieLib(movieId.Value, m.hash, m.name); //}).Start(); } }); } else { Console.WriteLine("请求发生错误\tStatusCode:" + httpResult.StatusCode); } }
/// <summary> /// 爬取电影列表 /// </summary> /// <param name="family"></param> public void CollectionMovies(Family family) { //初始参数设定 int pageSize = 800; int nowYear = DateTime.Now.Year; int startYear = 1900; //累加年份加载数据 for (int year = startYear; year <= nowYear; year++) { int maxPage = 10; for (int page = 1; page <= maxPage; page++) { //发送请求 var httpResult = http.GetHtml(new HttpItem() { URL = "http://search.video.iqiyi.com/o?type=list&pos=1&if=html5&site=iqiyi&mode=4&ctgName=" + family.ToString_() + "&pageSize=" + pageSize + "&market_release_date_level=" + year + "&pageNum=" + page, Accept = "application/json", ReadWriteTimeout = 1000 * 60, Timeout = 1000 * 60 }); //判断请求是否成功 if (httpResult.StatusCode == System.Net.HttpStatusCode.OK) { //将结果处理成实体集合 if (!httpResult.Html.Contains("search result is empty")) { var res_obj = httpResult.Html.ParseJSON <Models.iQiYi.Rootobject>(); if (res_obj.code == "A00000") { //加载数据成功 Console.WriteLine("本次共采集到" + family.ToString_() + ":\t" + res_obj.data.docinfos.Length + "条..."); res_obj.data.docinfos.ForEach_(m => { if (m.albumDocInfo != null) { if (m.albumDocInfo.video_lib_meta != null) { if (m.albumDocInfo.video_lib_meta.douban_id.HasValue) { //判断该豆瓣ID是否已经存在 //int? movieId = movie_bll.GetMovieIDByDouBanID(m.albumDocInfo.video_lib_meta.douban_id.Value.ToString()); //if (!movieId.HasValue) //{ // //根据豆瓣ID获取详细信息 // movieId = douBanCollection.CollectionDouBanInfo(m.albumDocInfo.video_lib_meta.douban_id.Value.ToString()); //} //else // Console.WriteLine(family.ToString_() + ":\t" + m.albumDocInfo.albumTitle + "\t已经被收录!"); int?movieId = douBanCollection.CollectionDouBanInfo(m.albumDocInfo.video_lib_meta.douban_id.Value.ToString()); if (movieId.HasValue && !string.IsNullOrWhiteSpace(m.albumDocInfo.albumId)) { Console.WriteLine("正在采集" + m.albumDocInfo.albumTitle + "的播放地址..."); //获取下载地址 CollectionMoviePlayer(movieId.Value, m.albumDocInfo.albumId, family); } } } } }); //最大页码 maxPage = (res_obj.data.result_num.ToInt_() / pageSize) + ((res_obj.data.result_num.ToInt_() % pageSize) > 0 ? 1 : 0); } else { Console.WriteLine("加载数据失败\tcode:" + res_obj.code); } } } else { Console.WriteLine("请求发生错误\tStatusCode:" + httpResult.StatusCode); } } } }