public static void Main(string[] args)
{
var logger = NLog.Web.NLogBuilder.ConfigureNLog("nlog.config").GetCurrentClassLogger();
var host = CreateHostBuilder(args).Build();//.Run();
try
{
using (IServiceScope scope = host.Services.CreateScope())
{
IConfiguration configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
string sqlString = configuration.GetSection("ConnectionStrings:MySqlConnection").Value;
NLogUtil.EnsureNlogConfig("Nlog.config", sqlString);
}
logger.Info("网站启动成功");
host.Run();
}
catch (Exception ex)
{
string errorMessage = "网站启动初始化数据异常";
NLogUtil.WriteFileLog(NLog.LogLevel.Error, LogType.Web, errorMessage, new Exception(errorMessage, ex));
NLogUtil.WriteDBLog(NLog.LogLevel.Error, LogType.Web, errorMessage, new Exception(errorMessage, ex));
throw;
}
finally
{
//确保在应用程序退出之前停止内部计时器/线程(避免在Linux上出现分段错误)
NLog.LogManager.Shutdown();
}
}
public List <BookBatch> CrawlerUrls(string entryUrl)
{
if (string.IsNullOrEmpty(entryUrl))
{
NLogUtil.ErrorTxt("TagListCrawler 没有入口Url");
}
List <BookBatch> result = new List <BookBatch>();
var doc = getDocbyEntryUrl(entryUrl);
var nodes = doc.DocumentNode.SelectNodes("//div[@class='article']//ul[@class='subject-list']/li");
if (nodes != null)
{
foreach (var n in nodes)
{
var url = n.SelectSingleNode("./div/a").Attributes["href"].Value;
result.Add(new BookBatch
{
DetailUrl = url
});
}
}
return(result);
}
/// <summary>
/// 书简介
/// </summary>
/// <param name="doc"></param>
public void AnalyContent(HtmlDocument doc)
{
EBookInfo bi = _bookDetailData.DouBanBookInfo;
string summery = "";
try
{
var hiddenNode = doc.DocumentNode.SelectSingleNode("//div[@class='related_info']//span[@class='all hidden']//div[@class='intro']");
if (hiddenNode != null)
{
summery = hiddenNode.InnerHtml.Trim();
}
else
{
var shortNode = doc.DocumentNode.SelectSingleNode("//div[@class='related_info']//div[@class='intro']");
if (shortNode != null)
{
summery = shortNode.InnerHtml.Trim();
}
}
bi.Summery = summery;
}
catch (Exception ex)
{
NLogUtil.ErrorTxt("BookDetailCrawler AnalyContent:" + ex.Message);
}
}
public void deleteComment_Res(long commentId)
{
try
{
_commentResRepository.Db.Ado.BeginTran();
//删除评论
_commentResRepository.DeleteByKey(commentId);
//删除对应的所有回复[ccCommentReply_Res]
_commentReplyResRepository.DeleteAllReplyByCommentId(commentId);
//删除评论点赞
_praizeRepository.DeletePraized_Comment_Res(commentId, null);
//删除评论对应的所有回复 点赞
_praizeRepository.DeletePraized_AllReplyBelowComment(commentId);
_commentResRepository.Db.Ado.CommitTran();
}
catch (Exception ex)
{
_commentResRepository.Db.Ado.RollbackTran();
NLogUtil.cc_ErrorTxt("CommentServices deleteComment_Res:" + ex.Message);
throw new Exception("删除评论失败");
}
}
public ResultEntity <ResponseToken> RefreshToken(string refToken)
{
ResultEntity <ResponseToken> result = new ResultEntity <ResponseToken>();
string url = _configuration["Id4Config:Id4Url"] + "connect/token";
requireUserPwdToken upToken = new requireUserPwdToken
{
grant_type = "refresh_token",
client_id = _configuration["Id4Config:client_id"],
client_secret = _configuration["Id4Config:client_secret"],
refresh_token = refToken,
};
try
{
var data = $"grant_type={upToken.grant_type}&client_id={upToken.client_id}&client_secret={upToken.client_secret}&refresh_token={upToken.refresh_token}";
var res = HttpUtil.Send(url, HttpUtil.HttpMethod.Post, data, HttpUtil.application_form);
result.Entity = JsonConvert.DeserializeObject <ResponseToken>(res);
}
catch (Exception ex)
{
NLogUtil.cc_ErrorTxt($"【UserController】GetUserPwdToken:{ex.Message}");
result.ErrorMsg = CCWebMsg.Token_GetUserPwdToken;
}
return(result);
}
public ResultEntity <panAccessToken> panCallback()
{
ResultEntity <panAccessToken> result = new ResultEntity <panAccessToken>();
try
{
NLogUtil.cc_InfoTxt($"pancallback comming");
string code = HttpContext.Request.Query["code"];
if (!string.IsNullOrEmpty(code))
{
// NLogUtil.cc_InfoTxt($"pancallback:{code}");
string url = @$ "https://openapi.baidu.com/oauth/2.0/token?grant_type=authorization_code&code={code}&client_id=YunvBnAcNTqOr3SbAGGoFNx9TAYwfX2k&client_secret=irhor1R0COYhjm8x2CyfUBGGb1Ggt07T&redirect_uri=http://ccapi.iqianba.cn/baidu/pancallback";
NLogUtil.cc_InfoTxt($"pancallback:{url}");
panAccessToken accessToken = HttpUtil.Get <panAccessToken>(url);
result.Entity = accessToken;
_baiduPanService.SaveAccessToken(accessToken);
}
}
catch (Exception ex)
{
result.ErrorMsg = ex.Message;
}
return(result);
}
/// <summary>
/// 根据Section Code ，覆盖所有相关 DataSecion.
///
/// </summary>
public async Task CoverNewSectionCodeAsync(List <EDataSection> newList)
{
var rAll = await base.Db.Ado.UseTranAsync(() =>
{
foreach (var es in newList)
{
int n = base.IsExist(a => new CountResult {
Count = SqlFunc.AggregateCount(a.Id)
}, a => a.SectionCode == es.SectionCode && a.ItemCode == es.ItemCode);
if (n == 0)
{
base.Add(es);
}
else
{
int r = base.DelAll(a => a.SectionCode == es.SectionCode && a.ItemCode == es.ItemCode && a.CreateDateTime.AddDays(30) < DateTime.Today);
if (r > 0)
{
base.Add(es);
}
}
}
// base.AddRange(newList);
});
if (!rAll.IsSuccess)
{
NLogUtil.ErrorTxt($"[CoverNewSectionCode]建立Book和 Section 关系:{rAll.ErrorMessage}");
}
}
public HtmlDocument getDocbyEntryUrl(string entryUrl)
{
HtmlDocument htmlDoc = null;
try
{
htmlDoc = newLoadWeb(entryUrl);
}
catch (ExceptionProxyConnect epc)
{
throw epc;
}
catch
{
NLogUtil.InfoTxt("Connect Error,Auto Try Next Proxy Connect");
try
{
ProxyManager.RemoveProxyHostCache();
htmlDoc = newLoadWeb(entryUrl);
}
catch (Exception ex)
{
throw new ExceptionProxyConnect("Connect Error while LoadDoc");
}
}
return(htmlDoc);
}
public ResultNormal ossMove(string fromPath, string toPath)
{
ResultNormal result = new ResultNormal();
try
{
if (OssClient.DoesObjectExist(_OssConfig.bucketName, fromPath))
{
var metadata = new ObjectMetadata();
//metadata.AddHeader("mk1", "mv1");
//metadata.AddHeader("mk2", "mv2");
var req = new CopyObjectRequest(_OssConfig.bucketName, fromPath, _OssConfig.bucketName, toPath)
{
NewObjectMetadata = metadata
};
// 拷贝文件。
OssClient.CopyObject(req);
OssClient.DeleteObject(_OssConfig.bucketName, fromPath);
}
}
catch (Exception ex)
{
NLogUtil.cc_ErrorTxt("Oss Services-moveOss:" + ex.Message);
result.ErrorMsg = ex.Message;
}
return(result);
}
public JsonResult Login(Sys_User entity)
{
try
{
int a = 10;
int b = 0;
int c = a / b;
}
catch (Exception ex)
{
NLogUtil.Error(ex.ToString());
//throw;
}
Sys_User user = _sysUserService.LoadEntities(c => c.UserName.Equals(entity.UserName)).SingleOrDefault();
if (user == null)
{
return(Json(new { state = 201, message = "用户名或密码错误" }, JsonRequestBehavior.AllowGet));
}
if (user.PassWord.Equals(EncryptUtil.Md5(entity.PassWord)))
{
Session["YD_USER"] = user;
string ip = Request.UserHostAddress;
Sys_Log log = _logService.AddEntity(new Sys_Log {
OccurTime = DateTime.Now, UserId = user.Id, Type = 1, LoginIp = ip
});
return(Json(new { state = 200, messgae = "登录成功" }, JsonRequestBehavior.AllowGet));
}
return(Json(new { state = 201, message = "用户名或密码错误" }, JsonRequestBehavior.AllowGet));
}
public async Task <IActionResult> Refund()
{
try
{
var notify = await _client.ExecuteAsync <WeChatPayRefundNotify>(Request, _optionsAccessor.Value);
NLogUtil.WriteDbLog(NLog.LogLevel.Trace, LogType.Refund, "退款结果通知" + JsonConvert.SerializeObject(notify));
NLogUtil.WriteFileLog(NLog.LogLevel.Trace, LogType.Refund, "退款结果通知" + JsonConvert.SerializeObject(notify));
if (notify.ReturnCode == WeChatPayCode.Success)
{
if (notify.RefundStatus == WeChatPayCode.Success)
{
//Console.WriteLine("OutTradeNo: " + notify.OutTradeNo);
var memo = JsonConvert.SerializeObject(notify);
return(WeChatPayNotifyResult.Success);
}
}
return(NoContent());
}
catch
{
return(NoContent());
}
}
public async void Run()
{
NLogUtil.InfoTxt("开始批量【豆瓣新书】", true);
var list = _LatestCrawler.CrawlerUrls("");
List <EDataSection> secList = new List <EDataSection>();
foreach (var url in list)
{
// _DetailCrawler = new BookDetailCrawler();
var midData = _DetailCrawler.Crawler(url.DetailUrl);
if (midData != null)
{
// 添加虚拟非虚拟
midData.DouBanBookInfo.FictionType = url.FictionType;
// 添加到最新专栏
var dataSection = _DouBanBookRepository.GetSection_NewExpress();
secList.Add(DataSectionRepository.newModelInstance(dataSection.Code, midData.DouBanBookInfo.Code));
await _DouBanBookRepository.HandleBookMiddleAsync(midData);
}
}
await _DouBanBookRepository.CoverSection(secList);
NLogUtil.InfoTxt("结束批量【豆瓣新书】", true);
}
private static void RunPlan()
{
int tryNum = 1;
while (tryNum < 3)
{
try
{
_DouBanBookRepository = _ServiceProvider.GetService <DouBanBookService>();
var list = _ServiceProvider.GetService <IEnumerable <ICrawlerBatchBook> >();
_TagList = list.FirstOrDefault(a => a.GetType().Name == "TagListCrawler");
_DetailBook = _ServiceProvider.GetService <ICrawlerBook>();
_CrawlerTag = _ServiceProvider.GetService <ICrawlerTag>();
PlanFromTagsTask planFromTagsTask = new PlanFromTagsTask(
_CrawlerTag,
_TagList,
_DetailBook,
_DouBanBookRepository);
planFromTagsTask.run();
}
catch (ExceptionProxyConnect epc)
{
NLogUtil.ErrorTxt($"代理连接错误:{epc.Message}");
NLogUtil.InfoTxt($"开始尝试第{tryNum++}次运行计划");
}
}
NLogUtil.InfoTxt($"第{tryNum}次运行计划后结束");
}
/// <summary>
/// 启动配置
/// </summary>
/// <param name="args"></param>
public static void Main(string[] args)
{
//NLogBuilder.ConfigureNLog("nlog.config").GetCurrentClassLogger();
//CreateHostBuilder(args).Build().Run();
var host = CreateHostBuilder(args).Build();
try
{
using (var scope = host.Services.CreateScope())
{
var configuration = scope.ServiceProvider.GetRequiredService <IConfiguration>();
//获取到appsettings.json中的连接字符串
var sqlString = configuration.GetSection("ConnectionStrings:SqlServerConnection").Value;
//确保NLog.config中连接字符串与appsettings.json中同步
NLogUtil.EnsureNlogConfig("NLog.config", sqlString);
}
//throw new Exception("测试异常");//for test
//其他项目启动时需要做的事情
NLogUtil.WriteDbLog(LogLevel.Trace, LogType.Web, "网站启动成功");
NLogUtil.WriteFileLog(LogLevel.Trace, LogType.Web, "网站启动成功");
host.Run();
}
catch (Exception ex)
{
//使用nlog写到本地日志文件（万一数据库没创建/连接成功）
var errorMessage = "网站启动成功初始化数据异常";
NLogUtil.WriteFileLog(LogLevel.Error, LogType.Web, errorMessage, new Exception(errorMessage, ex));
NLogUtil.WriteDbLog(LogLevel.Error, LogType.Web, errorMessage, new Exception(errorMessage, ex));
throw;
}
}
public List <EPlan_FromDouBanTagUrls> InitPlanFromTagUrl(List <Secction_Tag> tagLists)
{
List <EPlan_FromDouBanTagUrls> result = new List <EPlan_FromDouBanTagUrls>();
if (!_PlanFTURepository.IsExistPlan())
{
NLogUtil.InfoTxt("豆瓣计划写入到数据库");
if (tagLists != null && tagLists.Count > 0)
{
List <EDataSection> dsList = new List <EDataSection>();
// List<EPlan_FromDouBanTagUrls> planList = new List<EPlan_FromDouBanTagUrls>();
var allSection = _SectionDb.AllSection();
foreach (Secction_Tag st in tagLists)
{
ESection section = null;
try
{
section = allSection[st.sectionName];
}
catch
{
section = null;
}
foreach (var tag in st.TagList)
{
result.Add(PlanFTURepository.NewModelInstance(tag.Name, tag.Url));
if (section != null)
{
dsList.Add(DataSectionRepository.newModelInstance(section.Code, tag.Code));
}
}
}
var rAll = _Db.Ado.UseTran(async() =>
{
await _DataSectionDb.CoverNewSectionCodeAsync(dsList);
_PlanFTURepository.CoverPlans(GenCodeHelper.Plan_FromDouBanTagUrls, result);
});
if (rAll.IsSuccess)
{
NLogUtil.InfoTxt("【成功】豆瓣计划已在数据库初始化");
}
else
{
NLogUtil.ErrorTxt($"【失败】豆瓣计划:{rAll.ErrorMessage}");
}
}
}
else
{
NLogUtil.InfoTxt("DouBan Plan 已经存在数据库中！");
//DouBand Tag List 只能获取1000内的
result = _PlanFTURepository.QueryPlan(GenCodeHelper.Plan_FromDouBanTagUrls);
}
return(result);
}
private async Task HandleExceptionAsync(HttpContext context, Exception ex)
{
if (ex == null)
{
return;
}
NLogUtil.WriteFileLog(NLog.LogLevel.Error, LogType.Web, "全局捕获异常", new Exception("全局捕获异常", ex));
NLogUtil.WriteDbLog(NLog.LogLevel.Error, LogType.Web, "全局捕获异常", new Exception("全局捕获异常", ex));
await WriteExceptionAsync(context, ex).ConfigureAwait(false);
}
private static void Init()
{
try
{
Console.WriteLine("Init");
InitSystem();
//DbSeed.InitTables(_Db);
//var needInitSection = Convert.ToBoolean(_configuration["InitTask:NeedInitSection"]);
//DbSeed.InitData(_DouBanBookRepository, needInitSection);
}
catch (Exception ex)
{
NLogUtil.ErrorTxt($"Init Error:{ex.Message}", true);
}
}
private void async_CreateCommentMessage(SubmitComment submitComment, long commentId)
{
Task.Run(() =>
{
try{
_messageServices.CreateNotification_Comment(new MsgSubmitComment
{
SubmitComment = submitComment,
CommentId = commentId,
});
}
catch (Exception msgEx) {
NLogUtil.cc_ErrorTxt("【评论通知】错误:" + msgEx.Message);
}
});
}
public ResultPager <VueResInfo> GetAllResourcesByRefCode(QRes qRes)
{
ResultPager <VueResInfo> result = new ResultPager <VueResInfo>();
try
{
qRes.reqUserId = this.getUserId();
result.PageData = _resourceServices.getResByRefCode(qRes);
}
catch (Exception ex)
{
NLogUtil.cc_ErrorTxt("[BookController]GetAllResourcesByRefCode:" + ex.Message);
result.ErrorMsg = "查询失败";
}
return(result);
}
private static void RunSingle()
{
//string url = "https://book.douban.com/subject/34845963/";
string url = "https://book.douban.com/subject/2669319/";
try
{
SinglgTask singlgTask = new SinglgTask(_DetailBook, _DouBanBookRepository);
singlgTask.runAsync(url);
}
catch (ExceptionProxyConnect epc)
{
NLogUtil.ErrorTxt($"代理连接错误:{epc.Message}");
}
}
private static void RunLatestTask()
{
try
{
LatestTask latestTask = new LatestTask(_LatestBatchBook, _DetailBook, _DouBanBookRepository);
latestTask.Run();
}
catch (ExceptionProxyConnect epc)
{
NLogUtil.ErrorTxt($"代理连接错误:{epc.Message}");
}
catch (Exception ex)
{
NLogUtil.ErrorTxt($"RunLatestTask Error:{ex.Message}", true);
}
}
//异步方法 查询后获取未读消息转成已读
public void Async_MsgToReadAfterQuery(QMsgUser query,
List <VueMsgInfoNotification> queryResult = null,
List <VueSystemNotification> systemResult = null //由于系统消息对象不同，暂时只能这样写。。。
)
{
Task.Run(() =>
{
try
{
SubmitUnReadMsgIdList unReadList = new SubmitUnReadMsgIdList();
if (query.notificationType == NotificationType.system)
{
foreach (var msg in systemResult)
{
if (msg.NotificationStatus != NotificationStatus.read)
{
unReadList.msgIdList.Add(msg.msgId);
}
}
}
else
{
foreach (var msg in queryResult)
{
if (msg.NotificationStatus != NotificationStatus.read)
{
unReadList.msgIdList.Add(msg.msgId);
}
}
}
if (unReadList.msgIdList.Count > 0)
{
unReadList.notificationType = query.notificationType;
unReadList.targetStatus = NotificationStatus.read;
unReadList.userId = query.userId;
this.updateMsgToRead(unReadList);
}
}
catch (Exception ex)
{
NLogUtil.cc_ErrorTxt("[MessageService]MsgToReadAfterQuery_Async:" + ex.Message);
}
});
}
public void OnException(ExceptionContext context)
{
NLogUtil.WriteFileLog(NLog.LogLevel.Error, LogType.ApiRequest, "全局捕获异常", new Exception("全局捕获异常", context.Exception));
NLogUtil.WriteDbLog(NLog.LogLevel.Error, LogType.ApiRequest, "全局捕获异常", new Exception("全局捕获异常", context.Exception));
HttpStatusCode status = HttpStatusCode.InternalServerError;
//处理各种异常
var jm = new AdminUiCallBack();
jm.code = (int)status;
jm.msg = "发生了全局异常请联系管理员";
jm.data = context.Exception;
context.ExceptionHandled = true;
context.Result = new ObjectResult(jm);
}
public List <Secction_Tag> getUrls(string entryUrl)
{
if (string.IsNullOrEmpty(entryUrl))
{
entryUrl = DouBanTagsUrl;
}
List <Secction_Tag> result = new List <Secction_Tag>();
NLogUtil.InfoTxt("开始抓爬豆瓣 All Tag");
try
{
HtmlDocument doc = getDocbyEntryUrl(entryUrl);
VerifyHeader(doc);
var root = doc.DocumentNode.SelectNodes("//div[@class='article']/div")[1];
if (root == null)
{
throw new ExceptionProxyConnect("getUrls Null");
}
var secNodes = root.SelectNodes(".//div");
foreach (var sec in secNodes)
{
var secInfo = sec.SelectSingleNode(".//a");
Secction_Tag secTag = new Secction_Tag();
var sname = secInfo.Attributes["name"].Value;
// sname = sname.Replace(".", "");
secTag.sectionName = sname;
result.Add(secTag);
var allHref = sec.SelectNodes(".//table[@class='tagCol']//a");
foreach (var tagnode in allHref)
{
secTag.TagList.Add(newTag(tagnode.InnerText.Trim()));
}
}
}
catch (ExceptionProxyConnect epc)
{
throw epc;
}
catch (Exception ex)
{
NLogUtil.InfoTxt($"抓爬豆瓣错误:{ex.Message}");
}
NLogUtil.InfoTxt("抓爬豆瓣Tag结束");
return(result);
}
private void async_CreateReplyMessage(SubmitReply submitReply, long replyId)
{
Task.Run(() =>
{
try
{
_messageServices.CreateNotification_Reply(new MsgSubmitReply
{
SubmitReply = submitReply,
ReplyId = replyId,
});
}
catch (Exception msgEx)
{
NLogUtil.cc_ErrorTxt("【回复通知】错误:" + msgEx.Message);
}
});
}
public ResultNormal ossDelete(string ossPath)
{
ResultNormal result = new ResultNormal();
try
{
if (OssClient.DoesObjectExist(_OssConfig.bucketName, ossPath))
{
OssClient.DeleteObject(_OssConfig.bucketName, ossPath);
}
}
catch (Exception ex)
{
NLogUtil.cc_ErrorTxt("Oss Services-deleteOss:" + ex.Message);
result.ErrorMsg = ex.Message;
}
return(result);
}
private void asyncCreateMessage(SubmitPraize submitPraize, long praizeId)
{
Task.Run(() =>
{
try
{
_messageServices.CreateNotification_Praize(new MsgSubmitPraize
{
SubmitPraize = submitPraize,
PraizeId = praizeId,
});
}
catch (Exception msgEx)
{
NLogUtil.cc_ErrorTxt("【点赞通知】错误:" + msgEx.Message);
}
});
}
private string RequireVerifySMSCode(SMSRequire sms)
{
string url = _configuration["Id4Config:Id4Url"] + "sms/SubmitVerifyCode";
string json = JsonConvert.SerializeObject(sms);
string res = "";
try
{
res = HttpUtil.Send(url, HttpUtil.HttpMethod.Post, json, HttpUtil.application_json);
}
catch (Exception ex)
{
NLogUtil.cc_ErrorTxt($"【UserController】RequireVerifySMSCode:{ex.Message}");
throw new CCException(CCWebMsg.SMS_RequireVerifySMSCode);
}
return(res);
}
public ResultNormal uploadToOss(string localfilePath, string ossKey, bool isCover = true)
{
ResultNormal result = new ResultNormal();
try
{
var l = ossKey.Length;
if (isCover || (!isCover && !ossExist(ossKey)))
{
OssClient.PutObject(_OssConfig.bucketName, ossKey, localfilePath);
}
}
catch (Exception ex)
{
NLogUtil.cc_ErrorTxt("Oss Services-uploadFile:" + ex.Message);
result.ErrorMsg = ex.Message;
}
return(result);
}
public List <BookDetail_middle> Crawler(string entryUrl = "")
{
NLogUtil.InfoTxt($"[开始]抓爬TagList{entryUrl}");
var urlList = CrawlerUrls(entryUrl);
List <BookDetail_middle> result = new List <BookDetail_middle>();
foreach (var bookUrl in urlList)
{
var bd = ToDetail(bookUrl.DetailUrl, _CrawlerBook);
if (bd != null)
{
result.Add(bd);
}
}
NLogUtil.InfoTxt($"[结束]抓爬TagList{entryUrl}");
return(result);
}
