/// <summary>
/// 发送邮件
/// </summary>
/// <param name="subject">邮件标题</param>
/// <param name="body">邮件内容</param>
/// <param name="mails">关注者</param>
/// <param name="mailContentType">邮件内容是否采用HTML</param>
public static void SendMail <T>(SpiderWebConfig <T> webConfig, string subject, string body, List <string> mails, bool mailContentType = true) where T : Receiver
{
ATLog.Info("正在发送邮件,详细信息:\n" +
"发件人: " + webConfig.sender + "\n" +
"邮件主题: " + subject + "\n" +
"收件人: " + mails.ToArray().ToString() + "\n");
MailMessage mailMessage = new MailMessage();
mailMessage.IsBodyHtml = mailContentType;
mailMessage.BodyEncoding = Encoding.GetEncoding(936);
mailMessage.From = new MailAddress(webConfig.sender);
for (int i = 0; i < mails.Count; i++)
{
mailMessage.To.Add(new MailAddress(mails[i]));
}
mailMessage.Subject = subject;
mailMessage.Body = body;
SmtpClient client = new SmtpClient();
client.Host = "14.17.57.241";//"smtp.qq.com";
client.EnableSsl = true;
client.UseDefaultCredentials = false;
client.Credentials = new NetworkCredential(webConfig.sender, webConfig.license);
ServicePointManager.ServerCertificateValidationCallback += RemoteCertificateValidate;
client.Send(mailMessage);
}
/// <summary>
/// 获取知乎热点数据
/// </summary>
/// <returns></returns>
public static List <HotRepo> GetHotData()
{
ATLog.Info("获取知乎热点数据");
string html = GetHTMLContent();
return(GetHotList(html));
}
/// <summary>
/// 解析HTML内容，获取关键数据
/// </summary>
/// <param name="htmlContent"></param>
/// <returns></returns>
private static List <HotRepo> GetHotList(string htmlContent)
{
ATLog.Info("解析知乎HTML内容");
List <HotRepo> hotRepos = new List <HotRepo>(); //存入数据的数据类型
List <HotData> hotList = new List <HotData>(); //分析的完整数据类型
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(htmlContent);
HtmlNode node = doc.GetElementbyId("js-initialData");
StringBuilder jsonContent = new StringBuilder(node.InnerText);
JObject jsonObj = JObject.Parse(jsonContent.ToString());
JToken tokens = jsonObj.SelectToken("initialState.topstory.hotList", true);
if (tokens.Type == JTokenType.Array)
{
foreach (var token in tokens)
{
HotData item = JsonConvert.DeserializeObject <HotData>(token.ToString());
hotList.Add(item);
}
}
hotRepos = HotData.Convert(hotList);
return(hotRepos);
}
/// <summary>
/// 批量保存数据
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="datas"></param>
public void SaveRangeData<T>(IEnumerable<T> datas) where T : IRepo
{
ATLog.Info("存储知乎的热点数据到数据库");
using (ZhiHuContext context = new ZhiHuContext())
{
if (typeof(T) == typeof(HotRepo))
context.HotEntities.AddRange(datas as IEnumerable<HotRepo>);
context.SaveChanges();
}
}
/// <summary>
/// 读取Github的配置
/// </summary>
public static GithubWebConfig <GithubReceiver> ReadSettingForGithub()
{
ATLog.Info("读取Github的配置");
GithubWebConfig <GithubReceiver> config = new GithubWebConfig <GithubReceiver>();
string settingContent = File.ReadAllText(githubConfPath);
Configure <GithubReceiver> githubConfigure = Newtonsoft.Json.JsonConvert.DeserializeObject <Configure <GithubReceiver> >(settingContent);
config.Convert(githubConfigure);
return(config);
}
/// <summary>
/// 读取知乎的配置
/// </summary>
public static ZhihuWebConfig <ZhihuReceiver> ReadSettingForZhihu()
{
ATLog.Info("读取知乎的配置");
ZhihuWebConfig <ZhihuReceiver> config = new ZhihuWebConfig <ZhihuReceiver>();
string settingContent = File.ReadAllText(zhihuConfPath);
Configure <ZhihuReceiver> zhihuConfigure = Newtonsoft.Json.JsonConvert.DeserializeObject <Configure <ZhihuReceiver> >(settingContent);
config.Convert(zhihuConfigure);
return(config);
}
public static async Task <List <TrendingRepo> > Trending(String Period = "daily", String Language = "")
{
ATLog.Info(" =>正在进行网络请求,解析HTML文件");
List <TrendingRepo> Repos = new List <TrendingRepo>();
//解决远程身份验证问题
ServicePointManager.ServerCertificateValidationCallback += RemoteCertificateValidate;
using (var web = new HttpClient())
{
var result = await web.GetAsync(GetUrl(Period, Language));
if (result.IsSuccessStatusCode)
{
var htmlData = await result.Content.ReadAsStringAsync();
var doc = new HtmlDocument();
doc.LoadHtml(htmlData);
var baseRepos = doc.DocumentNode.Descendants().Where(x => x.Name == "ol").Select(x => x.ChildNodes).First().Where(x => x.Name == "li").Select(x => x);
foreach (HtmlNode repo in baseRepos)
{
var tempTrendingRepo = new TrendingRepo();
var childDivs = repo.ChildNodes.Where(x => x.Name == "div").Where(x => x.HasAttributes).Where(x => x.GetAttributeValue("class", "") != "float-right");
var repoOwnerData = childDivs.ElementAt(0).ChildNodes.Where(x => x.Name == "h3").First().ChildNodes.Where(x => x.Name == "a").First().ChildNodes;
var repoURLData = childDivs.ElementAt(0).ChildNodes.Where(x => x.Name == "h3").First().ChildNodes.Where(x => x.Name == "a");
tempTrendingRepo.RepoOwner = GetRepoOwner(repoOwnerData);
tempTrendingRepo.RepoTitle = GetRepoTitle(repoOwnerData);
tempTrendingRepo.Url = GetRepoURL(repoURLData);
tempTrendingRepo.RepoDescription = GetRepoDescription(childDivs.ElementAt(1));
tempTrendingRepo.Language = GetRepoLanguage(childDivs.ElementAt(2));
tempTrendingRepo.Stars = GetRepoStars(childDivs.ElementAt(2));
tempTrendingRepo.Forks = GetRepoForks(childDivs.ElementAt(2));
tempTrendingRepo.StarsToday = GetRepoStarsToday(childDivs.ElementAt(2));
Repos.Add(tempTrendingRepo);
}
}
}
if (Repos.Count != 0)
{
GithubOp.Instance.SaveRangeData(Repos);
}
return(Repos);
}
/// <summary>
/// 查询满足时间条件的热点数据
/// </summary>
/// <typeparam name="T1"></typeparam>
/// <param name="context"></param>
/// <param name="conditionDate">时间精确到年月日时</param>
/// <returns></returns>
/// ///////////////////////////////
/// ZhihuOp.Instance.GetRangeHotData(new DateTime(2018, 11, 14, 16, 0, 0));
/// //////////////////////////////
public List<HotRepo> GetRangeHotData(DateTime conditionDate)
{
ATLog.Info("从数据库查询符合条件的热点数据");
List<HotRepo> finals = new List<HotRepo>();
using (ZhiHuContext context = new ZhiHuContext())
{
var results = context.Set<HotRepo>().AsQueryable().AsNoTracking();
finals = results.Where(c => conditionDate.Year == c.Date.Year
&& conditionDate.Month == c.Date.Month
&& conditionDate.Day == c.Date.Day
&& conditionDate.Hour == c.Date.Hour).ToList();
}
return finals;
}
/// <summary>
/// 获取话题数据
/// </summary>
/// <param name="theme"></param>
/// <param name="type"></param>
/// <returns></returns>
public static List <ThemeRepo> GetThemeRepos(string theme, SearchType type = SearchType.Repositories)
{
ATLog.Info("获取Github上关注的话题");
List <ThemeRepo> repos = new List <ThemeRepo>();
StringBuilder conditions = new StringBuilder();
conditions.Append("?");
conditions.Append(Conditions.SetQueryKeyWord(theme));
string requestURL = Conditions.SetRequestRootURL(type.ToString()) + conditions.ToString();
Console.WriteLine(requestURL);
HttpWebRequest httpWebRequest = WebRequest.CreateHttp(requestURL);
httpWebRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
httpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36";
//httpWebRequest.ContentType = "application/json";
httpWebRequest.Method = "GET";
HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
if (httpWebResponse.StatusCode == HttpStatusCode.OK)
{
//Console.WriteLine("状态返回值: success");
Stream stream = (Stream)httpWebResponse.GetResponseStream();
StreamReader sr = new StreamReader(stream);
string resultJson = sr.ReadToEnd();
resultJson = resultJson.Replace("private", "_private");
RepositoriesEntity repositoriesResult = JsonConvert.DeserializeObject <RepositoriesEntity>(resultJson);
RepositoriesResultItem[] items = repositoriesResult.items;
for (int i = 0; i < items.Length; i++)
{
ThemeRepo themeRepo = new ThemeRepo(items[i].full_name, items[i].html_url, items[i].description, items[i].stargazers_count, items[i].language, items[i].score, items[i].owner.login);
repos.Add(themeRepo);
}
//Console.WriteLine(repositoriesResult.total_count);
}
if (repos.Count != 0)
{
GithubOp.Instance.SaveRangeData(repos);
}
return(repos);
}
/// <summary>
/// 创建邮件的模板(无布局)
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="repos"></param>
/// <returns></returns>
public static string CreateMailTemplate <T>(IEnumerable <T> repos) where T : IRepo
{
ATLog.Info("正在创建邮件模板");
StringBuilder content = new StringBuilder();
foreach (T item in repos)
{
Type type = item.GetType();
PropertyInfo[] piArr = type.GetProperties();
foreach (var pi in piArr)
{
content.Append(pi.Name + ":" + pi.GetValue(item) + "\n");
}
content.Append("\n");
}
return(content.ToString());
}
/// <summary>
/// 删除文件夹及其所有子文件
/// </summary>
/// <param name="path">文件夹路径</param>
public static bool RemoveFullFolder(string path)
{
path = path.Replace("\\", "/");
if (!Directory.Exists(path))
{
ATLog.Error(path + "Not Found");
return(false);
}
List <string> files = new List <string>(Directory.GetFiles(path));
files.ForEach(c =>
{
string tempFileName = Path.Combine(path, Path.GetFileName(c));
FileInfo fileInfo = new FileInfo(tempFileName);
if (fileInfo.Attributes != FileAttributes.Normal)
{
fileInfo.Attributes = FileAttributes.Normal;
}
fileInfo.Delete();
ATLog.Info(fileInfo.Name + "已经删除!");
});
List <string> folders = new List <string>(Directory.GetDirectories(path));
folders.ForEach(c =>
{
string tempFolderName = Path.Combine(path, Path.GetFileName(c));
RemoveFullFolder(tempFolderName);
});
DirectoryInfo directoryInfo = new DirectoryInfo(path);
if (directoryInfo.Attributes != FileAttributes.Normal)
{
directoryInfo.Attributes = FileAttributes.Normal;
}
directoryInfo.Delete();
ATLog.Info(directoryInfo.Name + "已经删除!");
return(true);
}
/// <summary>
/// 获取Github主题邮件内容
/// </summary>
/// <param name="theme">关注的Github主题</param>
/// <param name="type">邮件内容格式</param>
/// <returns></returns>
public static string GetThemeContents(string theme, MailContentType type)
{
ATLog.Info("获取Github关注的话题");
List <ThemeRepo> repos = JsonParserGithub.GetThemeRepos(theme);
string content = "";
switch (type)
{
case MailContentType.TEXT:
content = MailTextTemplate.CreateMailByThemeTemplate(repos);
break;
case MailContentType.HTML:
content = MailHTMLTemplate.GetHTMLContentByTheme(repos);
break;
}
//File.WriteAllText(Path.Combine(System.Environment.CurrentDirectory, "B.html"), content);
return(content);
}
/// <summary>
/// 与Github数据相关的行为
/// </summary>
private static void GithubAction()
{
if ((githubConf.isDebug && !githubConf.isDebugSend) || UpdateNoticeCondition(githubConf.noticeRate))
{
ATLog.Info("正在进行与Github数据相关的行为");
string mailContens = "";
//
foreach (string key in githubConf.LanguagesDict.Keys)
{
if (githubConf.LanguagesDict[key].Count != 0)
{//该语言有关注的人
mailContens = JsonParserGithub.GetFollowContents(key, githubConf.mailType);
Console.WriteLine("send language mail...");
//File.WriteAllText(System.Environment.CurrentDirectory + "/C.html", mailContens);
MailManager.SendMail(githubConf, SubscriptionSubject.Trending, mailContens, githubConf.LanguagesDict[key]);
}
}
//
foreach (string key in githubConf.ThemesDict.Keys)
{
if (githubConf.ThemesDict[key].Count != 0)
{
mailContens = JsonParserGithub.GetThemeContents(key, githubConf.mailType);
Console.WriteLine("send topic mail...");
MailManager.SendMail(githubConf, SubscriptionSubject.Topics, mailContens, githubConf.ThemesDict[key]);
}
}
if (githubConf.isDebug)
{
githubConf.isDebugSend = true;
}
SendStatus.SetSendSatus(githubConf.noticeRate);
}
if (!githubConf.isDebug)
{
CheckSendStatusForReset(githubConf.noticeRate);
}
}
/// <summary>
/// 获取Github趋势邮件内容
/// </summary>
/// <param name="fllowLanguage">关注何种语言趋势</param>
/// <param name="type">邮件内容格式</param>
/// <returns></returns>
public static string GetFollowContents(string fllowLanguage, MailContentType type)
{
ATLog.Info("获取Github每日趋势");
List <TrendingRepo> repos = HTMLParserGitHub.Trending("daily", fllowLanguage).Result;
string content = "";
switch (type)
{
case MailContentType.TEXT:
ATLog.Info("创建Text格式的邮件模板");
content = MailTextTemplate.CreateMailTemplate(repos);
break;
case MailContentType.HTML:
ATLog.Info("创建HTML格式的邮件模板");
content = MailHTMLTemplate.GetHTMLContentByLanguage(repos);
break;
}
//File.WriteAllText(Path.Combine(System.Environment.CurrentDirectory, "A.html"),content);
return(content);
}
/// <summary>
/// 与知乎数据相关的行为
/// </summary>
private static void ZhihuAction()
{
if ((zhihuConf.isDebug && !zhihuConf.isDebugSend) || UpdateNoticeCondition(zhihuConf.noticeRate))
{
List <HotRepo> repos = HTMLParserZhihu.GetHotData();
if (zhihuConf.database)
{
ZhihuOp.Instance.SaveRangeData(repos);
}
List <HotRepo> dataList = ZhihuOp.Instance.GetRangeHotData(DateTime.Now);
string mailContent = MailTextTemplate.CreateMailTemplate(dataList);
ATLog.Info("查找热点的订阅者");
//查找热点订阅者
List <string> users = new List <string>();
foreach (var mailAddress in zhihuConf.SubscribeHot.Keys)
{
if (zhihuConf.SubscribeHot[mailAddress])
{
users.Add(mailAddress);
}
}
ATLog.Info("向订阅者发送邮件");
MailManager.SendMail(zhihuConf, SubscriptionSubject.Hots, mailContent, users, false);
if (zhihuConf.isDebug)
{
zhihuConf.isDebugSend = true;
}
SendStatus.SetSendSatus(zhihuConf.noticeRate);
}
if (!zhihuConf.isDebug)
{
CheckSendStatusForReset(zhihuConf.noticeRate);
}
}
/// <summary>
/// 获取HTML内容
/// </summary>
private static string GetHTMLContent()
{
ATLog.Info("获取知乎热点HTML内容");
Stopwatch watch = new Stopwatch();
watch.Start();
string url = "https://www.zhihu.com/hot";
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36";
request.Headers.Add("Cookie", "_zap=ffcb80ae-726b-4a59-a615-410e7eb27c0a; d_c0=\"AHBks-FZGQ6PTglKpzz049qTZPR5e0aOGBQ=|1534999452\"; _xsrf=EDNW11FHbkn920HvNkAg70GjVV6oC1uP; q_c1=c069cf71863f4712957982039d03e8db|1539078018000|1534999452000; tst=h; _ga=GA1.2.97907782.1536630653; __utmz=155987696.1541512098.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); capsion_ticket=\"2|1:0|10:1541658653|14:capsion_ticket|44:NmZkOTkwZDU2MmIxNDZkODk4ZWQxMmVkZWJiZGMwYWE=|b74c9551d614e6e38d07ff820a5dcdf5013ccb9da1f3fe755431f3835c6bcb54\"; z_c0=\"2|1:0|10:1541658655|4:z_c0|92:Mi4xSS1HS0JRQUFBQUFBY0dTejRWa1pEaVlBQUFCZ0FsVk5IeWJSWEFBb1hCN3EwclVSd0VqZGkydkhERm0yUExGdkRB|d89fe04f809cd2110fa8969101cc21fc0576cd48ee6021516fefbafa8b309687\"; tgw_l7_route=3072ae0b421aa02514eac064fb2d64b5");
request.Headers.Add("Upgrade-Insecure-Requests", "1");
request.Headers.Add("Cache-Control", "max-age=0");
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
request.Method = "GET";
request.Referer = "https://www.zhihu.com/";
request.Headers.Add("Accept-Encoding", " gzip, deflate");
request.KeepAlive = true;//启用长连接
string Content = String.Empty;
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
using (Stream dataStream = response.GetResponseStream())
{
if (response.ContentEncoding.ToLower().Contains("gzip"))//解压
{
using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
Content = reader.ReadToEnd();
}
}
}
else if (response.ContentEncoding.ToLower().Contains("deflate"))//解压
{
using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
Content = reader.ReadToEnd();
}
}
}
else
{
using (StreamReader reader = new StreamReader(dataStream, Encoding.UTF8))
{
Content = reader.ReadToEnd();
}
}
}
}
request.Abort();
watch.Stop();
ATLog.Info(string.Format("请求网页用了{0}毫秒", watch.ElapsedMilliseconds.ToString()));
Console.WriteLine("请求网页用了{0}毫秒", watch.ElapsedMilliseconds.ToString());
return(Content);
}