/// <summary>
/// worker健康检查
/// </summary>
public void WorkerHealthCheck()
{
var workers = _repositoryFactory.ServerNodes.Where(x => x.NodeType == "worker" && x.Status != 0).ToList();
if (!workers.Any())
{
return;
}
//允许最大失败次数
int allowMaxFailed = ConfigurationCache.GetField <int>("System_WorkerUnHealthTimes");
if (allowMaxFailed <= 0)
{
allowMaxFailed = 3;
}
//遍历处理
workers.ForEach((w) =>
{
//初始化计数器
ConfigurationCache.WorkerUnHealthCounter.TryAdd(w.NodeName, 0);
//获取已失败次数
int failedCount = ConfigurationCache.WorkerUnHealthCounter[w.NodeName];
var success = WorkerRequest(w, "health", "get", null);
if (!success)
{
System.Threading.Interlocked.Increment(ref failedCount);
}
if (failedCount >= allowMaxFailed)
{
w.Status = 0;//标记下线，实际上可能存在因为网络抖动等原因导致检查失败但worker进程还在运行的情况
w.LastUpdateTime = DateTime.Now;
_repositoryFactory.ServerNodes.Update(w);
//释放该节点占据的锁
_repositoryFactory.ScheduleLocks.UpdateBy(
x => x.LockedNode == w.NodeName && x.Status == 1
, x => new ScheduleLockEntity
{
Status = 0,
LockedNode = null,
LockedTime = null
});
_unitOfWork.Commit();
//重置计数器
ConfigurationCache.WorkerUnHealthCounter[w.NodeName] = 0;
}
else
{
ConfigurationCache.WorkerUnHealthCounter[w.NodeName] = failedCount;
}
});
}
private static async Task LoadPluginFile(ScheduleEntity model)
{
bool pull = true;
var pluginPath = $"{ConfigurationCache.PluginPathPrefix}\\{model.Id}".ToPhysicalPath();
//看一下拉取策略
string policy = ConfigurationCache.GetField <string>("Assembly_ImagePullPolicy");
if (policy == "IfNotPresent" && System.IO.Directory.Exists(pluginPath))
{
pull = false;
}
if (pull)
{
using (var scope = new ScopeDbContext())
{
var master = scope.GetDbContext().ServerNodes.FirstOrDefault(x => x.NodeType == "master");
if (master == null)
{
throw new InvalidOperationException("master not found.");
}
var sourcePath = $"{master.AccessProtocol}://{master.Host}/static/downloadpluginfile?pluginname={model.AssemblyName}";
var zipPath = $"{ConfigurationCache.PluginPathPrefix}\\{model.Id.ToString("n")}.zip".ToPhysicalPath();
try
{
//下载文件
var httpClient = scope.GetService <IHttpClientFactory>().CreateClient();
var array = await httpClient.GetByteArrayAsync(sourcePath);
System.IO.FileStream fs = new System.IO.FileStream(zipPath, System.IO.FileMode.Create);
fs.Write(array, 0, array.Length);
fs.Close();
fs.Dispose();
}
catch (Exception ex)
{
LogHelper.Warn($"下载程序包异常，地址：{sourcePath}", model.Id);
throw ex.InnerException ?? ex;
}
//将指定 zip 存档中的所有文件都解压缩到各自对应的目录下
ZipFile.ExtractToDirectory(zipPath, pluginPath, true);
System.IO.File.Delete(zipPath);
}
}
}
/// <summary>
///发送邮件
/// </summary>
/// <param name="toAddressList">接收人</param>
/// <param name="title">标题</param>
/// <param name="content">内容</param>
/// <param name="attachments">附件</param>
/// <returns></returns>
public static void SendMail(List <KeyValuePair <string, string> > toAddressList, string title, string content, List <KeyValuePair <string, byte[]> > attachments = null, Guid?sid = null, Guid?traceId = null)
{
LogHelper.Info($"发送邮件开始：{title}\r\n收件人{string.Join(",",toAddressList)}\r\n", sid ?? Guid.Empty, traceId ?? Guid.Empty);
try
{
string server = ConfigurationCache.GetField <string>(ConfigurationCache.Email_SmtpServer);
int port = ConfigurationCache.GetField <int>(ConfigurationCache.Email_SmtpPort);
string account = ConfigurationCache.GetField <string>(ConfigurationCache.Email_FromAccount);
string pwd = ConfigurationCache.GetField <string>(ConfigurationCache.Email_FromAccountPwd);
var mailMessage = new MimeMessage();
var fromMailAddress = new MailboxAddress("ScheduleMaster", account);
mailMessage.From.Add(fromMailAddress);
var toMailAddress = toAddressList.Select(x => new MailboxAddress(x.Key, x.Value));
mailMessage.To.AddRange(toMailAddress);
var bodyBuilder = new BodyBuilder()
{
HtmlBody = content
};
if (attachments != null)
{
foreach (var item in attachments)
{
bodyBuilder.Attachments.Add(item.Key, item.Value);
}
}
mailMessage.Body = bodyBuilder.ToMessageBody();
mailMessage.Subject = title;
using (var smtpClient = new MailKit.Net.Smtp.SmtpClient())
{
smtpClient.Timeout = 10 * 1000; //设置超时时间
smtpClient.Connect(server, port, MailKit.Security.SecureSocketOptions.Auto); //连接到远程smtp服务器
smtpClient.Authenticate(account, pwd);
smtpClient.Send(mailMessage); //发送邮件
smtpClient.Disconnect(true);
}
LogHelper.Info($"发送邮件结束", sid ?? Guid.Empty, traceId ?? Guid.Empty);
}
catch (Exception eaf) {
LogHelper.Error($"发送邮件异常:{title}", eaf, sid ?? Guid.Empty, traceId ?? Guid.Empty);
}
}
private IRestResponse DoRequest()
{
var client = new RestClient(HttpOption.RequestUrl);
var request = new RestRequest(GetRestSharpMethod(HttpOption.Method));
var headers = HosScheduleFactory.ConvertParamsJson(HttpOption.Headers);
foreach (var (key, value) in headers)
{
request.AddHeader(key, value.ToString());
}
request.AddHeader("content-type", HttpOption.ContentType);
request.Timeout = 10000;
var config = ConfigurationCache.GetField<int>("Http_RequestTimeout");
if (config > 0)
{
request.Timeout = config * 1000;
}
var requestBody = string.Empty;
switch (HttpOption.ContentType)
{
case "application/json" when HttpOption.Body != null:
requestBody = HttpOption.Body.Replace("\r\n", "");
break;
case "application/x-www-form-urlencoded" when HttpOption.Body != null:
{
var formData = HosScheduleFactory.ConvertParamsJson(HttpOption.Body);
requestBody = string.Join('&', formData.Select(x => $"{x.Key}={System.Net.WebUtility.UrlEncode(x.Value.ToString())}"));
if (request.Method == Method.GET && formData.Count > 0)
{
client.BaseUrl = new Uri($"{HttpOption.RequestUrl}?{requestBody}");
}
break;
}
}
if (request.Method != Method.GET)
{
request.AddParameter(HttpOption.ContentType, requestBody, ParameterType.RequestBody);
}
var response = client.Execute(request);
return response;
}
private static async Task LoadPluginFile(ScheduleEntity model)
{
bool pull = true;
var pluginPath = $"{ConfigurationCache.PluginPathPrefix}\\{model.Id}".ToPhysicalPath();
//看一下拉取策略
string policy = ConfigurationCache.GetField <string>("Assembly_ImagePullPolicy");
if (policy == "IfNotPresent" && System.IO.Directory.Exists(pluginPath))
{
pull = false;
}
if (pull)
{
using (var scope = new ScopeDbContext())
{
var master = scope.GetDbContext().ServerNodes.FirstOrDefault(x => x.NodeType == "master");
if (master == null)
{
throw new InvalidOperationException("master not found.");
}
var sourcePath = $"{master.AccessProtocol}://{master.Host}/static/downloadpluginfile?pluginname={model.AssemblyName}";
var zipPath = $"{ConfigurationCache.PluginPathPrefix}\\{model.Id.ToString("n")}.zip".ToPhysicalPath();
using (WebClient client = new WebClient())
{
try
{
await client.DownloadFileTaskAsync(new Uri(sourcePath), zipPath);
}
catch (Exception ex)
{
LogHelper.Warn($"下载程序包异常，地址：{sourcePath}", model.Id);
throw ex;
}
}
//将指定 zip 存档中的所有文件都解压缩到各自对应的目录下
ZipFile.ExtractToDirectory(zipPath, pluginPath, true);
System.IO.File.Delete(zipPath);
}
}
}
public HttpTask(ScheduleHttpOptionEntity httpOption)
{
if (httpOption != null)
{
_option = httpOption;
_headers = HosScheduleFactory.ConvertParamsJson(httpOption.Headers);
if (_headers.ContainsKey(HEADER_TIMEOUT) && int.TryParse(_headers[HEADER_TIMEOUT].ToString(), out int result) && result > 0)
{
_timeout = TimeSpan.FromSeconds(result);
}
else
{
int config = ConfigurationCache.GetField <int>("Http_RequestTimeout");
if (config > 0)
{
_timeout = TimeSpan.FromSeconds(config);
}
}
string requestBody = string.Empty;
string url = httpOption.RequestUrl;
if (httpOption.ContentType == "application/json")
{
requestBody = httpOption.Body?.Replace("\r\n", "");
}
else if (httpOption.ContentType == "application/x-www-form-urlencoded")
{
var formData = HosScheduleFactory.ConvertParamsJson(httpOption.Body);
requestBody = string.Join('&', formData.Select(x => $"{x.Key}={System.Net.WebUtility.UrlEncode(x.Value.ToString())}"));
if (httpOption.Method.ToLower() == "get" && formData.Count > 0)
{
url = $"{httpOption.RequestUrl}?{requestBody}";
}
}
_option.RequestUrl = url;
_option.Body = requestBody;
}
}
/// <summary>
/// 根据任务实体插入
/// </summary>
/// <param name="entity"></param>
/// <returns></returns>
private static bool InsertByEntity(ScheduleDelayedEntity entity)
{
bool success = false;
NotifyPlan plan = new NotifyPlan()
{
Key = entity.Id.ToString(),
NotifyUrl = entity.NotifyUrl,
NotifyDataType = entity.NotifyDataType,
NotifyBody = entity.NotifyBody,
Callback = NotifyExecutedEvent
};
string pattern = ConfigurationCache.GetField <string>("DelayTask_DelayPattern");
if (pattern.ToLower() == "absolute")
{
success = Insert(plan, entity.DelayAbsoluteTime);
}
else
{
success = Insert(plan, entity.DelayTimeSpan);
}
return(success);
}
private static async Task NotifyRequest(NotifyPlan plan)
{
Guid sid = Guid.Parse(plan.Key);
Guid traceId = Guid.NewGuid();
using (var scope = new ScopeDbContext())
{
var db = scope.GetDbContext();
var tracer = scope.GetService <Common.RunTracer>();
var entity = await db.ScheduleDelayeds.FirstOrDefaultAsync(x => x.Id == sid);
if (entity == null)
{
LogHelper.Info($"不存在的任务ID。", sid, traceId);
return;
}
entity.ExecuteTime = DateTime.Now;
Exception failedException = null;
try
{
//创建一条trace
await tracer.Begin(traceId, plan.Key);
var httpClient = scope.GetService <IHttpClientFactory>().CreateClient();
plan.NotifyBody = plan.NotifyBody.Replace("\r\n", "");
HttpContent reqContent = new StringContent(plan.NotifyBody, System.Text.Encoding.UTF8, "application/json");
if (plan.NotifyDataType == "application/x-www-form-urlencoded")
{
//任务创建时要确保参数是键值对的json格式
reqContent = new FormUrlEncodedContent(Newtonsoft.Json.JsonConvert.DeserializeObject <IEnumerable <KeyValuePair <string, string> > >(plan.NotifyBody));
}
LogHelper.Info($"即将请求：{entity.NotifyUrl}", sid, traceId);
var response = await httpClient.PostAsync(plan.NotifyUrl, reqContent);
var content = await response.Content.ReadAsStringAsync();
LogHelper.Info($"请求结束，响应码：{response.StatusCode.GetHashCode().ToString()}，响应内容：{(response.Content.Headers.GetValues("Content-Type").Any(x => x.Contains("text/html")) ? "html文档" : content)}", sid, traceId);
if (response.IsSuccessStatusCode && content.Contains("success"))
{
await tracer.Complete(ScheduleRunResult.Success);
//更新结果字段
entity.FinishTime = DateTime.Now;
entity.Status = (int)ScheduleDelayStatus.Successed;
//更新日志
LogHelper.Info($"延时任务[{entity.Topic}:{entity.ContentKey}]执行成功。", sid, traceId);
}
else
{
failedException = new Exception("异常的返回结果。");
}
}
catch (Exception ex)
{
failedException = ex;
}
// 对异常进行处理
if (failedException != null)
{
//更新trace
await tracer.Complete(ScheduleRunResult.Failed);
//失败重试策略
int maxRetry = ConfigurationCache.GetField <int>("DelayTask_RetryTimes");
if (entity.FailedRetrys < (maxRetry > 0 ? maxRetry : 3))
{
//更新结果字段
entity.FailedRetrys++;
//计算下次延时间隔
int timespan = ConfigurationCache.GetField <int>("DelayTask_RetrySpans");
int delay = (timespan > 0 ? timespan : 10) * entity.FailedRetrys;
//重新进入延时队列
Insert(plan, delay);
//更新日志
LogHelper.Error($"延时任务[{entity.Topic}:{entity.ContentKey}]执行失败，将在{delay.ToString()}秒后开始第{entity.FailedRetrys.ToString()}次重试。", failedException, sid, traceId);
}
else
{
entity.Status = (int)ScheduleDelayStatus.Failed;
entity.Remark = $"重试{entity.FailedRetrys}次后失败结束";
//更新日志
LogHelper.Error($"延时任务[{entity.Topic}:{entity.ContentKey}]重试{entity.FailedRetrys}次后失败结束。", failedException, sid, traceId);
//邮件通知
var user = await db.SystemUsers.FirstOrDefaultAsync(x => x.UserName == entity.CreateUserName && !string.IsNullOrEmpty(x.Email));
if (user != null)
{
var keeper = new List <KeyValuePair <string, string> >()
{
new KeyValuePair <string, string>(user.RealName, user.Email)
};
MailKitHelper.SendMail(keeper, $"延时任务异常 — {entity.Topic}:{entity.ContentKey}",
Common.QuartzManager.GetErrorEmailContent($"{entity.Topic}:{entity.ContentKey}", failedException)
, sid: sid, traceId: traceId);
}
else
{
LogHelper.Error($"用户无邮箱，无法发送邮件：{entity.CreateUserName}", new Exception("用户无邮箱"), sid, traceId);
}
}
// .....
// 其实这个重试策略稍微有点问题，只能在抢锁成功的节点上进行重试，如果遭遇单点故障会导致任务丢失
// 严格来说应该通知到master让其对所有节点执行重试策略，但考虑到master也会有单点问题，综合考虑后还是放到当前worker中重试，若worker节点异常可以在控制台中人工干预进行重置或立即执行
}
db.Update(entity);
await db.SaveChangesAsync();
}
}
/// <summary>
/// worker健康检查
/// </summary>
public void WorkerHealthCheck()
{
var workers = _repositoryFactory.ServerNodes.Where(x => x.NodeType == "worker" && x.Status != 0).ToList();
if (!workers.Any())
{
return;
}
//允许最大失败次数
int allowMaxFailed = ConfigurationCache.GetField <int>("System_WorkerUnHealthTimes");
if (allowMaxFailed <= 0)
{
allowMaxFailed = 3;
}
//遍历处理
workers.ForEach(async(w) =>
{
using (var scope = new Core.ScopeDbContext())
{
var db = scope.GetDbContext();
_serverClient.Server = w;
//初始化计数器
ConfigurationCache.WorkerUnHealthCounter.TryAdd(w.NodeName, 0);
var success = await _serverClient.HealthCheck();
if (success)
{
w.LastUpdateTime = DateTime.Now;
db.ServerNodes.Update(w);
await db.SaveChangesAsync();
ConfigurationCache.WorkerUnHealthCounter[w.NodeName] = 0;
}
else
{
//获取已失败次数
int failedCount = ConfigurationCache.WorkerUnHealthCounter[w.NodeName];
System.Threading.Interlocked.Increment(ref failedCount);
if (failedCount >= allowMaxFailed)
{
w.Status = 0;//标记下线，实际上可能存在因为网络抖动等原因导致检查失败但worker进程还在运行的情况
db.ServerNodes.Update(w);
//释放该节点占据的锁
var locks = db.ScheduleLocks.Where(x => x.LockedNode == w.NodeName && x.Status == 1).ToList();
locks.ForEach(x =>
{
x.Status = 0;
x.LockedNode = null;
x.LockedTime = null;
});
db.ScheduleLocks.UpdateRange(locks);
await db.SaveChangesAsync();
//重置计数器
ConfigurationCache.WorkerUnHealthCounter[w.NodeName] = 0;
}
else
{
ConfigurationCache.WorkerUnHealthCounter[w.NodeName] = failedCount;
}
}
}
});
}
