/// <summary>
        ///  获取知乎热点数据
        /// </summary>
        /// <returns></returns>
        public static List <HotRepo> GetHotData()
        {
            ATLog.Info("获取知乎热点数据");
            string html = GetHTMLContent();

            return(GetHotList(html));
        }
        /// <summary>
        /// 解析HTML内容,获取关键数据
        /// </summary>
        /// <param name="htmlContent"></param>
        /// <returns></returns>
        private static List <HotRepo> GetHotList(string htmlContent)
        {
            ATLog.Info("解析知乎HTML内容");
            List <HotRepo> hotRepos = new List <HotRepo>(); //存入数据的数据类型
            List <HotData> hotList  = new List <HotData>(); //分析的完整数据类型

            HtmlDocument doc = new HtmlDocument();

            doc.LoadHtml(htmlContent);

            HtmlNode      node        = doc.GetElementbyId("js-initialData");
            StringBuilder jsonContent = new StringBuilder(node.InnerText);
            JObject       jsonObj     = JObject.Parse(jsonContent.ToString());
            JToken        tokens      = jsonObj.SelectToken("initialState.topstory.hotList", true);

            if (tokens.Type == JTokenType.Array)
            {
                foreach (var token in tokens)
                {
                    HotData item = JsonConvert.DeserializeObject <HotData>(token.ToString());
                    hotList.Add(item);
                }
            }

            hotRepos = HotData.Convert(hotList);
            return(hotRepos);
        }
Example #3
0
        /// <summary>
        /// 与Github数据相关的行为
        /// </summary>
        private static void GithubAction()
        {
            if ((githubConf.isDebug && !githubConf.isDebugSend) || UpdateNoticeCondition(githubConf.noticeRate))
            {
                ATLog.Info("正在进行与Github数据相关的行为");
                string mailContens = "";
                //
                foreach (string key in githubConf.LanguagesDict.Keys)
                {
                    if (githubConf.LanguagesDict[key].Count != 0)
                    {//该语言有关注的人
                        mailContens = JsonParserGithub.GetFollowContents(key, githubConf.mailType);
                        Console.WriteLine("send language mail...");
                        //File.WriteAllText(System.Environment.CurrentDirectory + "/C.html", mailContens);
                        MailManager.SendMail(githubConf, SubscriptionSubject.Trending, mailContens, githubConf.LanguagesDict[key]);
                    }
                }

                //
                foreach (string key in githubConf.ThemesDict.Keys)
                {
                    if (githubConf.ThemesDict[key].Count != 0)
                    {
                        mailContens = JsonParserGithub.GetThemeContents(key, githubConf.mailType);
                        Console.WriteLine("send topic mail...");
                        MailManager.SendMail(githubConf, SubscriptionSubject.Topics, mailContens, githubConf.ThemesDict[key]);
                    }
                }

                if (githubConf.isDebug)
                {
                    githubConf.isDebugSend = true;
                }
                SendStatus.SetSendSatus(githubConf.noticeRate);
            }

            if (!githubConf.isDebug)
            {
                CheckSendStatusForReset(githubConf.noticeRate);
            }
        }
Example #4
0
        /// <summary>
        /// 与知乎数据相关的行为
        /// </summary>
        private static void ZhihuAction()
        {
            if ((zhihuConf.isDebug && !zhihuConf.isDebugSend) || UpdateNoticeCondition(zhihuConf.noticeRate))
            {
                List <HotRepo> repos = HTMLParserZhihu.GetHotData();
                if (zhihuConf.database)
                {
                    ZhihuOp.Instance.SaveRangeData(repos);
                }

                List <HotRepo> dataList    = ZhihuOp.Instance.GetRangeHotData(DateTime.Now);
                string         mailContent = MailTextTemplate.CreateMailTemplate(dataList);

                ATLog.Info("查找热点的订阅者");
                //查找热点订阅者
                List <string> users = new List <string>();
                foreach (var mailAddress in zhihuConf.SubscribeHot.Keys)
                {
                    if (zhihuConf.SubscribeHot[mailAddress])
                    {
                        users.Add(mailAddress);
                    }
                }

                ATLog.Info("向订阅者发送邮件");
                MailManager.SendMail(zhihuConf, SubscriptionSubject.Hots, mailContent, users, false);

                if (zhihuConf.isDebug)
                {
                    zhihuConf.isDebugSend = true;
                }
                SendStatus.SetSendSatus(zhihuConf.noticeRate);
            }

            if (!zhihuConf.isDebug)
            {
                CheckSendStatusForReset(zhihuConf.noticeRate);
            }
        }
        /// <summary>
        /// 获取HTML内容
        /// </summary>
        private static string GetHTMLContent()
        {
            ATLog.Info("获取知乎热点HTML内容");
            Stopwatch watch = new Stopwatch();

            watch.Start();
            string         url     = "https://www.zhihu.com/hot";
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

            request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36";
            request.Headers.Add("Cookie", "_zap=ffcb80ae-726b-4a59-a615-410e7eb27c0a; d_c0=\"AHBks-FZGQ6PTglKpzz049qTZPR5e0aOGBQ=|1534999452\"; _xsrf=EDNW11FHbkn920HvNkAg70GjVV6oC1uP; q_c1=c069cf71863f4712957982039d03e8db|1539078018000|1534999452000; tst=h; _ga=GA1.2.97907782.1536630653; __utmz=155987696.1541512098.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); capsion_ticket=\"2|1:0|10:1541658653|14:capsion_ticket|44:NmZkOTkwZDU2MmIxNDZkODk4ZWQxMmVkZWJiZGMwYWE=|b74c9551d614e6e38d07ff820a5dcdf5013ccb9da1f3fe755431f3835c6bcb54\"; z_c0=\"2|1:0|10:1541658655|4:z_c0|92:Mi4xSS1HS0JRQUFBQUFBY0dTejRWa1pEaVlBQUFCZ0FsVk5IeWJSWEFBb1hCN3EwclVSd0VqZGkydkhERm0yUExGdkRB|d89fe04f809cd2110fa8969101cc21fc0576cd48ee6021516fefbafa8b309687\"; tgw_l7_route=3072ae0b421aa02514eac064fb2d64b5");
            request.Headers.Add("Upgrade-Insecure-Requests", "1");
            request.Headers.Add("Cache-Control", "max-age=0");
            request.Accept  = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
            request.Method  = "GET";
            request.Referer = "https://www.zhihu.com/";
            request.Headers.Add("Accept-Encoding", " gzip, deflate");
            request.KeepAlive = true;//启用长连接

            string Content = String.Empty;

            using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
            {
                using (Stream dataStream = response.GetResponseStream())
                {
                    if (response.ContentEncoding.ToLower().Contains("gzip"))//解压
                    {
                        using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
                        {
                            using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                            {
                                Content = reader.ReadToEnd();
                            }
                        }
                    }
                    else if (response.ContentEncoding.ToLower().Contains("deflate"))//解压
                    {
                        using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
                        {
                            using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                            {
                                Content = reader.ReadToEnd();
                            }
                        }
                    }
                    else
                    {
                        using (StreamReader reader = new StreamReader(dataStream, Encoding.UTF8))
                        {
                            Content = reader.ReadToEnd();
                        }
                    }
                }
            }
            request.Abort();
            watch.Stop();
            ATLog.Info(string.Format("请求网页用了{0}毫秒", watch.ElapsedMilliseconds.ToString()));
            Console.WriteLine("请求网页用了{0}毫秒", watch.ElapsedMilliseconds.ToString());

            return(Content);
        }