/// <summary> /// worker健康检查 /// </summary> public void WorkerHealthCheck() { var workers = _repositoryFactory.ServerNodes.Where(x => x.NodeType == "worker" && x.Status != 0).ToList(); if (!workers.Any()) { return; } //允许最大失败次数 int allowMaxFailed = ConfigurationCache.GetField <int>("System_WorkerUnHealthTimes"); if (allowMaxFailed <= 0) { allowMaxFailed = 3; } //遍历处理 workers.ForEach((w) => { //初始化计数器 ConfigurationCache.WorkerUnHealthCounter.TryAdd(w.NodeName, 0); //获取已失败次数 int failedCount = ConfigurationCache.WorkerUnHealthCounter[w.NodeName]; var success = WorkerRequest(w, "health", "get", null); if (!success) { System.Threading.Interlocked.Increment(ref failedCount); } if (failedCount >= allowMaxFailed) { w.Status = 0;//标记下线,实际上可能存在因为网络抖动等原因导致检查失败但worker进程还在运行的情况 w.LastUpdateTime = DateTime.Now; _repositoryFactory.ServerNodes.Update(w); //释放该节点占据的锁 _repositoryFactory.ScheduleLocks.UpdateBy( x => x.LockedNode == w.NodeName && x.Status == 1 , x => new ScheduleLockEntity { Status = 0, LockedNode = null, LockedTime = null }); _unitOfWork.Commit(); //重置计数器 ConfigurationCache.WorkerUnHealthCounter[w.NodeName] = 0; } else { ConfigurationCache.WorkerUnHealthCounter[w.NodeName] = failedCount; } }); }
private static async Task LoadPluginFile(ScheduleEntity model) { bool pull = true; var pluginPath = $"{ConfigurationCache.PluginPathPrefix}\\{model.Id}".ToPhysicalPath(); //看一下拉取策略 string policy = ConfigurationCache.GetField <string>("Assembly_ImagePullPolicy"); if (policy == "IfNotPresent" && System.IO.Directory.Exists(pluginPath)) { pull = false; } if (pull) { using (var scope = new ScopeDbContext()) { var master = scope.GetDbContext().ServerNodes.FirstOrDefault(x => x.NodeType == "master"); if (master == null) { throw new InvalidOperationException("master not found."); } var sourcePath = $"{master.AccessProtocol}://{master.Host}/static/downloadpluginfile?pluginname={model.AssemblyName}"; var zipPath = $"{ConfigurationCache.PluginPathPrefix}\\{model.Id.ToString("n")}.zip".ToPhysicalPath(); try { //下载文件 var httpClient = scope.GetService <IHttpClientFactory>().CreateClient(); var array = await httpClient.GetByteArrayAsync(sourcePath); System.IO.FileStream fs = new System.IO.FileStream(zipPath, System.IO.FileMode.Create); fs.Write(array, 0, array.Length); fs.Close(); fs.Dispose(); } catch (Exception ex) { LogHelper.Warn($"下载程序包异常,地址:{sourcePath}", model.Id); throw ex.InnerException ?? ex; } //将指定 zip 存档中的所有文件都解压缩到各自对应的目录下 ZipFile.ExtractToDirectory(zipPath, pluginPath, true); System.IO.File.Delete(zipPath); } } }
/// <summary> ///发送邮件 /// </summary> /// <param name="toAddressList">接收人</param> /// <param name="title">标题</param> /// <param name="content">内容</param> /// <param name="attachments">附件</param> /// <returns></returns> public static void SendMail(List <KeyValuePair <string, string> > toAddressList, string title, string content, List <KeyValuePair <string, byte[]> > attachments = null, Guid?sid = null, Guid?traceId = null) { LogHelper.Info($"发送邮件开始:{title}\r\n收件人{string.Join(",",toAddressList)}\r\n", sid ?? Guid.Empty, traceId ?? Guid.Empty); try { string server = ConfigurationCache.GetField <string>(ConfigurationCache.Email_SmtpServer); int port = ConfigurationCache.GetField <int>(ConfigurationCache.Email_SmtpPort); string account = ConfigurationCache.GetField <string>(ConfigurationCache.Email_FromAccount); string pwd = ConfigurationCache.GetField <string>(ConfigurationCache.Email_FromAccountPwd); var mailMessage = new MimeMessage(); var fromMailAddress = new MailboxAddress("ScheduleMaster", account); mailMessage.From.Add(fromMailAddress); var toMailAddress = toAddressList.Select(x => new MailboxAddress(x.Key, x.Value)); mailMessage.To.AddRange(toMailAddress); var bodyBuilder = new BodyBuilder() { HtmlBody = content }; if (attachments != null) { foreach (var item in attachments) { bodyBuilder.Attachments.Add(item.Key, item.Value); } } mailMessage.Body = bodyBuilder.ToMessageBody(); mailMessage.Subject = title; using (var smtpClient = new MailKit.Net.Smtp.SmtpClient()) { smtpClient.Timeout = 10 * 1000; //设置超时时间 smtpClient.Connect(server, port, MailKit.Security.SecureSocketOptions.Auto); //连接到远程smtp服务器 smtpClient.Authenticate(account, pwd); smtpClient.Send(mailMessage); //发送邮件 smtpClient.Disconnect(true); } LogHelper.Info($"发送邮件结束", sid ?? Guid.Empty, traceId ?? Guid.Empty); } catch (Exception eaf) { LogHelper.Error($"发送邮件异常:{title}", eaf, sid ?? Guid.Empty, traceId ?? Guid.Empty); } }
private IRestResponse DoRequest() { var client = new RestClient(HttpOption.RequestUrl); var request = new RestRequest(GetRestSharpMethod(HttpOption.Method)); var headers = HosScheduleFactory.ConvertParamsJson(HttpOption.Headers); foreach (var (key, value) in headers) { request.AddHeader(key, value.ToString()); } request.AddHeader("content-type", HttpOption.ContentType); request.Timeout = 10000; var config = ConfigurationCache.GetField<int>("Http_RequestTimeout"); if (config > 0) { request.Timeout = config * 1000; } var requestBody = string.Empty; switch (HttpOption.ContentType) { case "application/json" when HttpOption.Body != null: requestBody = HttpOption.Body.Replace("\r\n", ""); break; case "application/x-www-form-urlencoded" when HttpOption.Body != null: { var formData = HosScheduleFactory.ConvertParamsJson(HttpOption.Body); requestBody = string.Join('&', formData.Select(x => $"{x.Key}={System.Net.WebUtility.UrlEncode(x.Value.ToString())}")); if (request.Method == Method.GET && formData.Count > 0) { client.BaseUrl = new Uri($"{HttpOption.RequestUrl}?{requestBody}"); } break; } } if (request.Method != Method.GET) { request.AddParameter(HttpOption.ContentType, requestBody, ParameterType.RequestBody); } var response = client.Execute(request); return response; }
private static async Task LoadPluginFile(ScheduleEntity model) { bool pull = true; var pluginPath = $"{ConfigurationCache.PluginPathPrefix}\\{model.Id}".ToPhysicalPath(); //看一下拉取策略 string policy = ConfigurationCache.GetField <string>("Assembly_ImagePullPolicy"); if (policy == "IfNotPresent" && System.IO.Directory.Exists(pluginPath)) { pull = false; } if (pull) { using (var scope = new ScopeDbContext()) { var master = scope.GetDbContext().ServerNodes.FirstOrDefault(x => x.NodeType == "master"); if (master == null) { throw new InvalidOperationException("master not found."); } var sourcePath = $"{master.AccessProtocol}://{master.Host}/static/downloadpluginfile?pluginname={model.AssemblyName}"; var zipPath = $"{ConfigurationCache.PluginPathPrefix}\\{model.Id.ToString("n")}.zip".ToPhysicalPath(); using (WebClient client = new WebClient()) { try { await client.DownloadFileTaskAsync(new Uri(sourcePath), zipPath); } catch (Exception ex) { LogHelper.Warn($"下载程序包异常,地址:{sourcePath}", model.Id); throw ex; } } //将指定 zip 存档中的所有文件都解压缩到各自对应的目录下 ZipFile.ExtractToDirectory(zipPath, pluginPath, true); System.IO.File.Delete(zipPath); } } }
public HttpTask(ScheduleHttpOptionEntity httpOption) { if (httpOption != null) { _option = httpOption; _headers = HosScheduleFactory.ConvertParamsJson(httpOption.Headers); if (_headers.ContainsKey(HEADER_TIMEOUT) && int.TryParse(_headers[HEADER_TIMEOUT].ToString(), out int result) && result > 0) { _timeout = TimeSpan.FromSeconds(result); } else { int config = ConfigurationCache.GetField <int>("Http_RequestTimeout"); if (config > 0) { _timeout = TimeSpan.FromSeconds(config); } } string requestBody = string.Empty; string url = httpOption.RequestUrl; if (httpOption.ContentType == "application/json") { requestBody = httpOption.Body?.Replace("\r\n", ""); } else if (httpOption.ContentType == "application/x-www-form-urlencoded") { var formData = HosScheduleFactory.ConvertParamsJson(httpOption.Body); requestBody = string.Join('&', formData.Select(x => $"{x.Key}={System.Net.WebUtility.UrlEncode(x.Value.ToString())}")); if (httpOption.Method.ToLower() == "get" && formData.Count > 0) { url = $"{httpOption.RequestUrl}?{requestBody}"; } } _option.RequestUrl = url; _option.Body = requestBody; } }
/// <summary> /// 根据任务实体插入 /// </summary> /// <param name="entity"></param> /// <returns></returns> private static bool InsertByEntity(ScheduleDelayedEntity entity) { bool success = false; NotifyPlan plan = new NotifyPlan() { Key = entity.Id.ToString(), NotifyUrl = entity.NotifyUrl, NotifyDataType = entity.NotifyDataType, NotifyBody = entity.NotifyBody, Callback = NotifyExecutedEvent }; string pattern = ConfigurationCache.GetField <string>("DelayTask_DelayPattern"); if (pattern.ToLower() == "absolute") { success = Insert(plan, entity.DelayAbsoluteTime); } else { success = Insert(plan, entity.DelayTimeSpan); } return(success); }
private static async Task NotifyRequest(NotifyPlan plan) { Guid sid = Guid.Parse(plan.Key); Guid traceId = Guid.NewGuid(); using (var scope = new ScopeDbContext()) { var db = scope.GetDbContext(); var tracer = scope.GetService <Common.RunTracer>(); var entity = await db.ScheduleDelayeds.FirstOrDefaultAsync(x => x.Id == sid); if (entity == null) { LogHelper.Info($"不存在的任务ID。", sid, traceId); return; } entity.ExecuteTime = DateTime.Now; Exception failedException = null; try { //创建一条trace await tracer.Begin(traceId, plan.Key); var httpClient = scope.GetService <IHttpClientFactory>().CreateClient(); plan.NotifyBody = plan.NotifyBody.Replace("\r\n", ""); HttpContent reqContent = new StringContent(plan.NotifyBody, System.Text.Encoding.UTF8, "application/json"); if (plan.NotifyDataType == "application/x-www-form-urlencoded") { //任务创建时要确保参数是键值对的json格式 reqContent = new FormUrlEncodedContent(Newtonsoft.Json.JsonConvert.DeserializeObject <IEnumerable <KeyValuePair <string, string> > >(plan.NotifyBody)); } LogHelper.Info($"即将请求:{entity.NotifyUrl}", sid, traceId); var response = await httpClient.PostAsync(plan.NotifyUrl, reqContent); var content = await response.Content.ReadAsStringAsync(); LogHelper.Info($"请求结束,响应码:{response.StatusCode.GetHashCode().ToString()},响应内容:{(response.Content.Headers.GetValues("Content-Type").Any(x => x.Contains("text/html")) ? "html文档" : content)}", sid, traceId); if (response.IsSuccessStatusCode && content.Contains("success")) { await tracer.Complete(ScheduleRunResult.Success); //更新结果字段 entity.FinishTime = DateTime.Now; entity.Status = (int)ScheduleDelayStatus.Successed; //更新日志 LogHelper.Info($"延时任务[{entity.Topic}:{entity.ContentKey}]执行成功。", sid, traceId); } else { failedException = new Exception("异常的返回结果。"); } } catch (Exception ex) { failedException = ex; } // 对异常进行处理 if (failedException != null) { //更新trace await tracer.Complete(ScheduleRunResult.Failed); //失败重试策略 int maxRetry = ConfigurationCache.GetField <int>("DelayTask_RetryTimes"); if (entity.FailedRetrys < (maxRetry > 0 ? maxRetry : 3)) { //更新结果字段 entity.FailedRetrys++; //计算下次延时间隔 int timespan = ConfigurationCache.GetField <int>("DelayTask_RetrySpans"); int delay = (timespan > 0 ? timespan : 10) * entity.FailedRetrys; //重新进入延时队列 Insert(plan, delay); //更新日志 LogHelper.Error($"延时任务[{entity.Topic}:{entity.ContentKey}]执行失败,将在{delay.ToString()}秒后开始第{entity.FailedRetrys.ToString()}次重试。", failedException, sid, traceId); } else { entity.Status = (int)ScheduleDelayStatus.Failed; entity.Remark = $"重试{entity.FailedRetrys}次后失败结束"; //更新日志 LogHelper.Error($"延时任务[{entity.Topic}:{entity.ContentKey}]重试{entity.FailedRetrys}次后失败结束。", failedException, sid, traceId); //邮件通知 var user = await db.SystemUsers.FirstOrDefaultAsync(x => x.UserName == entity.CreateUserName && !string.IsNullOrEmpty(x.Email)); if (user != null) { var keeper = new List <KeyValuePair <string, string> >() { new KeyValuePair <string, string>(user.RealName, user.Email) }; MailKitHelper.SendMail(keeper, $"延时任务异常 — {entity.Topic}:{entity.ContentKey}", Common.QuartzManager.GetErrorEmailContent($"{entity.Topic}:{entity.ContentKey}", failedException) , sid: sid, traceId: traceId); } else { LogHelper.Error($"用户无邮箱,无法发送邮件:{entity.CreateUserName}", new Exception("用户无邮箱"), sid, traceId); } } // ..... // 其实这个重试策略稍微有点问题,只能在抢锁成功的节点上进行重试,如果遭遇单点故障会导致任务丢失 // 严格来说应该通知到master让其对所有节点执行重试策略,但考虑到master也会有单点问题,综合考虑后还是放到当前worker中重试,若worker节点异常可以在控制台中人工干预进行重置或立即执行 } db.Update(entity); await db.SaveChangesAsync(); } }
/// <summary> /// worker健康检查 /// </summary> public void WorkerHealthCheck() { var workers = _repositoryFactory.ServerNodes.Where(x => x.NodeType == "worker" && x.Status != 0).ToList(); if (!workers.Any()) { return; } //允许最大失败次数 int allowMaxFailed = ConfigurationCache.GetField <int>("System_WorkerUnHealthTimes"); if (allowMaxFailed <= 0) { allowMaxFailed = 3; } //遍历处理 workers.ForEach(async(w) => { using (var scope = new Core.ScopeDbContext()) { var db = scope.GetDbContext(); _serverClient.Server = w; //初始化计数器 ConfigurationCache.WorkerUnHealthCounter.TryAdd(w.NodeName, 0); var success = await _serverClient.HealthCheck(); if (success) { w.LastUpdateTime = DateTime.Now; db.ServerNodes.Update(w); await db.SaveChangesAsync(); ConfigurationCache.WorkerUnHealthCounter[w.NodeName] = 0; } else { //获取已失败次数 int failedCount = ConfigurationCache.WorkerUnHealthCounter[w.NodeName]; System.Threading.Interlocked.Increment(ref failedCount); if (failedCount >= allowMaxFailed) { w.Status = 0;//标记下线,实际上可能存在因为网络抖动等原因导致检查失败但worker进程还在运行的情况 db.ServerNodes.Update(w); //释放该节点占据的锁 var locks = db.ScheduleLocks.Where(x => x.LockedNode == w.NodeName && x.Status == 1).ToList(); locks.ForEach(x => { x.Status = 0; x.LockedNode = null; x.LockedTime = null; }); db.ScheduleLocks.UpdateRange(locks); await db.SaveChangesAsync(); //重置计数器 ConfigurationCache.WorkerUnHealthCounter[w.NodeName] = 0; } else { ConfigurationCache.WorkerUnHealthCounter[w.NodeName] = failedCount; } } } }); }