示例#1
0
 public async Task Execute(IJobExecutionContext context)
 {
     await Task.Run(async() =>
     {
         var HttpRequestDomain = RelitClass.HttpRequestDomain;
         var jobData           = context.JobDetail.JobDataMap;//获取Job中的参数
         string Url            = jobData.GetString("Url");
         string Message        = "";
         var DocumentHtml      = await HttpRequestDomain.HttpRequest(Url);
         HtmlDocument html     = new HtmlDocument();
         html.LoadHtml(DocumentHtml);
         var DataNode   = html.DocumentNode.SelectSingleNode("//div[@class='box-model-cont fn-clear']");
         var InsertData = new List <CarouselMap>();
         var FlgCount   = 0;
         foreach (var item in DataNode.ChildNodes)
         {
             //只取五条数据
             if (FlgCount == 5)
             {
                 break;
             }
             InsertData.Add(new CarouselMap
             {
                 ImgUrl = item.ChildNodes[0].ChildNodes[0].Attributes["src"].Value.ToString(),
                 Link   = Url + item.Attributes["href"].Value.ToString(),
                 Sort   = FlgCount + 1
             });
             FlgCount++;
         }
         //验证是否一致数据
         DbContextFace <CarouselMap> Server = new DbContextFace <CarouselMap>();
         var mapList = Server.All();
         if (mapList == null || mapList.Count == 0)
         {
             Server.AddRange(InsertData);
             Message = "最新影视--数据库数据为空 -- 插入成功";
         }
         else
         {
             foreach (var item in mapList)
             {
                 //清空数据
                 Server.Delete(item.Id);
                 Message += "最新影视--数据清楚成功";
             }
             //插入数据
             Server.AddRange(InsertData);
             Message += "最新影视--数据添加成功";
         }
         //写入日志
         LogStreamWrite.WriteLineLog(Message);
     }).ConfigureAwait(false);
 }
示例#2
0
        public async Task Execute(IJobExecutionContext context)
        {
            Console.WriteLine(context.ToString());
            await Task.Run(async() =>
            {
                var HttpRequestDomain = RelitClass.HttpRequestDomain;
                var jobData           = context.JobDetail.JobDataMap;//获取Job中的参数
                string Url            = jobData.GetString("Url");
                string Message        = "开始请求数据";
                var DocumentHtml      = await HttpRequestDomain.HttpRequest(Url);
                HtmlDocument html     = new HtmlDocument();
                html.LoadHtml(DocumentHtml);
                var DataNode   = html.DocumentNode.SelectNodes("//div[@class='box-model-cont fn-clear']");
                var InsertData = new List <Movie>();
                //读取链接
                foreach (var item in DataNode)
                {
                    foreach (var itemSon in item.ChildNodes)
                    {
                        InsertData.Add(new Movie
                        {
                            MovieName     = itemSon.ChildNodes[1].ChildNodes[0].InnerText,
                            MovieCategory = Movie.MOVIE_CATEGORY_INDEX,
                            MovieDesc     = itemSon.ChildNodes[1].ChildNodes[1].InnerText,
                            MovieImgUrl   = itemSon.ChildNodes[0].ChildNodes[0].Attributes["src"].Value.ToString(),
                            MovieLink     = Url + itemSon.Attributes["href"].Value.ToString(),
                            MovieTitle    = this.JudgeMovieDefinition(itemSon.ChildNodes[0].ChildNodes[2].InnerText.ToString()),
                        });
                    }
                }

                //读取详情内容
                for (int i = 0; i < InsertData.Count; i++)
                {
                    var TempInsert = InsertData[i];

                    var DesctionHTML             = await HttpRequestDomain.HttpRequest(TempInsert.MovieLink);
                    HtmlDocument DectionDocument = new HtmlDocument();
                    DectionDocument.LoadHtml(DesctionHTML);
                    //拆解信息
                    var DesctionNode = DectionDocument.DocumentNode.SelectSingleNode("//div[@class='fd-box']");
                    var Index        = DesctionNode.ChildNodes[4].ChildNodes[2].InnerText.IndexOf(":") + 1;
                    //读取信息
                    InsertData[i].MovieDesction = new MovieDesc
                    {
                        Director = DesctionNode.ChildNodes[2].ChildNodes[1].FirstChild.NextSibling == null
                        ? DesctionNode.ChildNodes[2].ChildNodes[1].FirstChild.InnerText :
                                   DesctionNode.ChildNodes[2].ChildNodes[1].FirstChild.NextSibling.InnerText,
                        Describe   = DesctionNode.ChildNodes[6].ChildNodes[0].FirstChild.InnerText,
                        Address    = DesctionNode.ChildNodes[4].ChildNodes[0].FirstChild.NextSibling.InnerText,
                        Year       = DesctionNode.ChildNodes[4].ChildNodes[1].FirstChild.NextSibling.InnerText,
                        UpdateTime = DateTime.Parse(DesctionNode.ChildNodes[4].ChildNodes[2].InnerText.Substring(
                                                        Index, DesctionNode.ChildNodes[4].ChildNodes[2].InnerText.Length - Index) ?? ""),
                        MovieId = InsertData[i].Id.ToString(),
                    };

                    //读取主演
                    InsertData[i].MovieDesction.MainDirector = DesctionNode.ChildNodes[3].FirstChild.InnerText;

                    //读取影片集数
                    var Collections = DectionDocument.DocumentNode.SelectNodes("//div[@class='lv-bf-list']//a");

                    InsertData[i].MovieDesction.MovieCollections = new List <MovieCollections>();
                    foreach (var item in Collections)
                    {
                        InsertData[i].MovieDesction.MovieCollections.Add(new MovieCollections
                        {
                            ColletionName = this.JudgeMovieDefinition(item.Attributes["title"].Value.ToString()),
                            Link          = Url + item.Attributes["href"].Value.ToString(),
                            MovieId       = InsertData[i].Id.ToString()
                        });
                    }
                }


                #region 注释
                //////验证是否一致数据  -- 存入 推荐电影
                ////DbContextFace<NewestMovie> NewestServer = new DbContextFace<NewestMovie>();
                ////var mapList = NewestServer.All();
                ////if (mapList == null || mapList.Count == 0)
                ////{
                ////    NewestServer.AddRange(InsertData.Take(10).ToList());
                ////    Message += "旧数据库数据为空 -- 插入成功 -- 推荐 /n";
                ////}
                ////else
                ////{
                ////    foreach (var item in mapList)
                ////    {
                ////        //清空数据
                ////        NewestServer.Delete(item.Id);
                ////        Message += "数据清楚成功 -- 推荐!/n";
                ////    }
                ////    //插入数据
                ////    NewestServer.AddRange(InsertData.Take(10).ToList());
                ////    Message += "数据添加成功 -- 推荐 \n";
                ////}
                //////存入 电影
                ////DbContextFace<FilmOpera> FilmService = new DbContextFace<FilmOpera>();
                ////var FilmOperaList = FilmService.All();
                ////List<FilmOpera> FilmOpra = InsertData.Skip(10).Take(20).ToList().Select(d => new FilmOpera
                ////{
                ////    MovieName = d.MovieName,
                ////    MovieDesc = d.MovieDesc,
                ////    MovieImgUrl = d.MovieImgUrl,
                ////    MovieLink = d.MovieLink,
                ////    MovieTitle = this.JudgeMovieDefinition(d.MovieTitle)
                ////}).ToList();
                ////if (FilmOperaList == null || FilmOperaList.Count == 0)
                ////{
                ////    FilmService.AddRange(FilmOpra);
                ////    Message += "旧数据库数据为空 -- 插入成功 -- 电影/n";
                ////}
                ////else
                ////{
                ////    foreach (var item in FilmOperaList)
                ////    {
                ////        //清空数据
                ////        FilmService.Delete(item.Id);
                ////        Message += "数据清楚成功 -- 电影!/n";
                ////    }
                ////    //插入数据
                ////    FilmService.AddRange(FilmOpra);
                ////    Message += "数据添加成功 -- 电影 /n";
                ////}



                ////// 存入 大陆剧
                ////DbContextFace<MainlandOpera> MainLandServer = new DbContextFace<MainlandOpera>();
                ////var MainLandList = MainLandServer.All();
                ////List<MainlandOpera> MainLandOpera = InsertData.Skip(20).Take(30).ToList().Select(d => new MainlandOpera
                ////{
                ////    MovieName = d.MovieName,
                ////    MovieDesc = d.MovieDesc,
                ////    MovieImgUrl = d.MovieImgUrl,
                ////    MovieLink = d.MovieLink,
                ////    MovieTitle = this.JudgeMovieDefinition(d.MovieTitle)
                ////}).ToList();
                ////if (MainLandList == null || MainLandList.Count == 0)
                ////{
                ////    MainLandServer.AddRange(MainLandOpera);
                ////    Message += "旧数据库数据为空 -- 插入成功 -- 大陆剧/n";
                ////}
                ////else
                ////{
                ////    foreach (var item in MainLandList)
                ////    {
                ////        //清空数据
                ////        MainLandServer.Delete(item.Id);
                ////        Message += "数据清楚成功 -- 大陆剧!/n";
                ////    }
                ////    //插入数据
                ////    MainLandServer.AddRange(MainLandOpera);
                ////    Message += "数据添加成功 -- 大陆剧 /n";
                ////}


                ////// 存入 港剧
                ////DbContextFace<HongKongOpera> HongKongService = new DbContextFace<HongKongOpera>();
                ////var HongKongList = HongKongService.All();
                ////List<HongKongOpera> HongKongOpera = InsertData.Skip(30).Take(40).ToList().Select(d => new HongKongOpera
                ////{
                ////    MovieName = d.MovieName,
                ////    MovieDesc = d.MovieDesc,
                ////    MovieImgUrl = d.MovieImgUrl,
                ////    MovieLink = d.MovieLink,
                ////    MovieTitle = this.JudgeMovieDefinition(d.MovieTitle)
                ////}).ToList();
                ////if (HongKongList == null || HongKongList.Count == 0)
                ////{
                ////    HongKongService.AddRange(HongKongOpera);
                ////    Message += "旧数据库数据为空 -- 插入成功 -- 港剧/n";
                ////}
                ////else
                ////{
                ////    foreach (var item in HongKongList)
                ////    {
                ////        //清空数据
                ////        HongKongService.Delete(item.Id);
                ////        Message += "数据清楚成功 -- 港剧!/n";
                ////    }
                ////    //插入数据
                ////    HongKongService.AddRange(HongKongOpera);
                ////    Message += "数据添加成功 -- 港剧 /n";
                ////}



                ////// 存入 韩剧
                ////DbContextFace<KoreanDramaOpera> KoreanDramaService = new DbContextFace<KoreanDramaOpera>();
                ////var KoreanDramaList = KoreanDramaService.All();
                ////List<KoreanDramaOpera> KoreanDramaOpera = InsertData.Skip(40).Take(50).ToList().Select(d => new KoreanDramaOpera
                ////{
                ////    MovieName = d.MovieName,
                ////    MovieDesc = d.MovieDesc,
                ////    MovieImgUrl = d.MovieImgUrl,
                ////    MovieLink = d.MovieLink,
                ////    MovieTitle = this.JudgeMovieDefinition(d.MovieTitle)
                ////}).ToList();
                ////if (KoreanDramaList == null || KoreanDramaList.Count == 0)
                ////{
                ////    KoreanDramaService.AddRange(KoreanDramaOpera);
                ////    Message += "旧数据库数据为空 -- 插入成功 -- 韩剧/n";
                ////}
                ////else
                ////{
                ////    foreach (var item in HongKongList)
                ////    {
                ////        //清空数据
                ////        KoreanDramaService.Delete(item.Id);
                ////        Message += "数据清楚成功 -- 韩剧!/n";
                ////    }
                ////    //插入数据
                ////    KoreanDramaService.AddRange(KoreanDramaOpera);
                ////    Message += "数据添加成功 -- 韩剧 /n";
                ////}



                ////// 存入 美剧
                ////DbContextFace<AmericanOpera> AmericanOperaService = new DbContextFace<AmericanOpera>();
                ////var AmericanOperaList = AmericanOperaService.All();
                ////List<AmericanOpera> AmericanOpera = InsertData.Skip(50).Take(60).ToList().Select(d => new AmericanOpera
                ////{
                ////    MovieName = d.MovieName,
                ////    MovieDesc = d.MovieDesc,
                ////    MovieImgUrl = d.MovieImgUrl,
                ////    MovieLink = d.MovieLink,
                ////    MovieTitle = this.JudgeMovieDefinition(d.MovieTitle)
                ////}).ToList();
                ////if (AmericanOperaList == null || AmericanOperaList.Count == 0)
                ////{
                ////    AmericanOperaService.AddRange(AmericanOpera);
                ////    Message += "旧数据库数据为空 -- 插入成功 -- 美剧/n";
                ////}
                ////else
                ////{
                ////    foreach (var item in AmericanOperaList)
                ////    {
                ////        //清空数据
                ////        AmericanOperaService.Delete(item.Id);
                ////        Message += "数据清楚成功 -- 美剧!/n";
                ////    }
                ////    //插入数据
                ////    AmericanOperaService.AddRange(AmericanOpera);
                ////    Message += "数据添加成功 -- 美剧 /n";
                ////}


                ////// 存入 动漫
                ////DbContextFace<AnimationOpera> AnimationOperaService = new DbContextFace<AnimationOpera>();
                ////var AnimationOperaList = AnimationOperaService.All();
                ////List<AnimationOpera> AnimationOpera = InsertData.Skip(60).Take(70).ToList().Select(d => new AnimationOpera
                ////{
                ////    MovieName = d.MovieName,
                ////    MovieDesc = d.MovieDesc,
                ////    MovieImgUrl = d.MovieImgUrl,
                ////    MovieLink = d.MovieLink,
                ////    MovieTitle = this.JudgeMovieDefinition(d.MovieTitle)
                ////}).ToList();
                ////if (AnimationOperaList == null || AnimationOperaList.Count == 0)
                ////{
                ////    AnimationOperaService.AddRange(AnimationOpera);
                ////    Message += "旧数据库数据为空 -- 插入成功 -- 动漫/n";
                ////}
                ////else
                ////{
                ////    foreach (var item in AnimationOperaList)
                ////    {
                ////        //清空数据
                ////        AnimationOperaService.Delete(item.Id);
                ////        Message += "数据清楚成功 -- 动漫!/n";
                ////    }
                ////    //插入数据
                ////    AnimationOperaService.AddRange(AnimationOpera);
                ////    Message += "数据添加成功 -- 动漫 /n";
                ////}

                #endregion


                // 将数据全部导入 Movie
                DbContextFace <Movie> MovieService = new DbContextFace <Movie>();
                //处理名称
                var HttpWebRequestArray = InsertData.Select(d => d.MovieName).ToArray();

                var FindMovieService = MovieService.Find(d => HttpWebRequestArray.Contains(d.MovieName));

                if (FindMovieService == null || FindMovieService.Count == 0)
                {
                    //插入数据
                    MovieService.AddRange(InsertData);
                    Message += "\n数据库数据为空,添加数据成功!";
                    LogStreamWrite.WriteLineLog(Message);
                }
                else
                {
                    //筛选不重复数据
                    var DisctinDataMovieList = InsertData.Where(d => !FindMovieService.
                                                                Select(d => d.MovieName).ToArray().Contains(d.MovieName)).ToList();
                    //添加数据
                    MovieService.AddRange(DisctinDataMovieList);
                    Message += "\n重新添加数据成功!";
                }

                //日志写入
                LogStreamWrite.WriteLineLog(Message);
            }).ConfigureAwait(false);
        }
        /// <summary>
        /// 发送请求
        /// </summary>
        /// <param name="Url"></param>
        /// <returns></returns>
        public async Task <string> HttpRequest(string Url)
        {
            //var httpWebRequest = (HttpWebRequest)WebRequest.Create(Url);
            //httpWebRequest.ContentType = "application/json";
            //httpWebRequest.Method = "GET";
            //string respContent = "";
            //var httpResponse = (HttpWebResponse)httpWebRequest.GetResponse();
            //using (var streamReader = new StreamReader(httpResponse.GetResponseStream()))
            //{
            //    if (httpResponse.StatusCode == HttpStatusCode.OK)
            //    {
            //        respContent = await streamReader.ReadToEndAsync();
            //    }
            //}
            //return respContent;

            WebRequest      Request  = null;
            HttpWebResponse Response = null;
            Stream          _Stream  = null;
            StreamReader    Reader   = null;
            string          HTML     = "";

            try
            {
                Request             = WebRequest.Create(Url);
                Request.Method      = "get";
                Request.Proxy       = null;
                Request.ContentType = "application/json";
                Request.Timeout     = 80000;
                Response            = (HttpWebResponse)Request.GetResponse();
                _Stream             = Response.GetResponseStream();
                Reader = new StreamReader(_Stream, Encoding.UTF8);
                HTML   = await Reader.ReadToEndAsync();

                //Request.Abort();
                //Response.Close();
                //_Stream.Close();
                //Reader.Close();
            }
            catch (Exception ex)
            {
                LogStreamWrite.WriteLineLog("HttpRequest请求错误:" + ex.Message);
            }
            finally
            {
                if (Request != null)
                {
                    Request.Abort();
                }
                if (Response != null)
                {
                    Response.Close();
                }
                if (_Stream != null)
                {
                    _Stream.Close();
                }
                if (Reader != null)
                {
                    Reader.Close();
                }
            }
            return(HTML);
        }