private async Task DoRequest(TaskContext context)
{
using (var scope = new ScopeDbContext())
{
var httpClient = scope.GetService <IHttpClientFactory>().CreateClient();
foreach (var item in _headers)
{
httpClient.DefaultRequestHeaders.Add(item.Key, item.Value.ToString());
}
httpClient.Timeout = _timeout;
var httpRequest = new HttpRequestMessage
{
Content = new StringContent(_option.Body, System.Text.Encoding.UTF8, _option.ContentType),
Method = new HttpMethod(_option.Method),
RequestUri = new Uri(_option.RequestUrl)
};
var response = await httpClient.SendAsync(httpRequest, CancellationToken);
if (response.IsSuccessStatusCode)
{
context.WriteLog($"请求结束，响应码：{response.StatusCode.GetHashCode().ToString()}，响应内容：{(response.Content.Headers.GetValues("content-type").Any(x => x.Contains("text/html")) ? "html文档" : await response.Content.ReadAsStringAsync())}");
}
response.Dispose();
}
}
public static void Init()
{
Queue = new BufferQueue <SystemLogEntity>();
var td = new System.Threading.Thread(() =>
{
using (var scope = new ScopeDbContext())
{
var db = scope.GetDbContext();
while (true)
{
Queue.Read((item, index) =>
{
item.Node = ConfigurationCache.NodeSetting?.IdentityName;
///item.CreateTime = DateTime.Now;
db.SystemLogs.Add(item);
});
db.SaveChanges();
System.Threading.Thread.Sleep(5000);
}
}
});
td.IsBackground = true;
td.Start();
}
public static void Init()
{
Queue = new BufferQueue <SystemLogEntity>();
var td = new System.Threading.Thread(() =>
{
using (var scope = new ScopeDbContext())
{
var db = scope.GetDbContext();
db.ChangeTracker.AutoDetectChangesEnabled = false;
while (true)
{
Queue.Read((item, index) =>
{
item.Node = ConfigurationCache.NodeSetting?.IdentityName;
db.SystemLogs.Add(item);
});
if (db.ChangeTracker.HasChanges())
{
db.SaveChanges();
}
foreach (var item in db.ChangeTracker.Entries())
{
item.State = Microsoft.EntityFrameworkCore.EntityState.Detached;
}
System.Threading.Thread.Sleep(5000);
}
}
});
td.IsBackground = true;
td.Start();
}
public async Task Execute(IJobExecutionContext context)
{
using (var scope = new ScopeDbContext())
{
var _db = scope.GetDbContext();
var stoppedList = _db.Schedules.Where(x => x.Status == (int)ScheduleStatus.Stop).Select(x => x.Id).ToList();
foreach (var sid in stoppedList)
{
JobKey jk = new JobKey(sid.ToString().ToLower());
if (await context.Scheduler.CheckExists(jk))
{
await QuartzManager.Stop(sid);
}
}
}
}
private static async Task LoadPluginFile(ScheduleEntity model)
{
bool pull = true;
var pluginPath = $"{ConfigurationCache.PluginPathPrefix}\\{model.Id}".ToPhysicalPath();
//看一下拉取策略
string policy = ConfigurationCache.GetField <string>("Assembly_ImagePullPolicy");
if (policy == "IfNotPresent" && System.IO.Directory.Exists(pluginPath))
{
pull = false;
}
if (pull)
{
using (var scope = new ScopeDbContext())
{
var master = scope.GetDbContext().ServerNodes.FirstOrDefault(x => x.NodeType == "master");
if (master == null)
{
throw new InvalidOperationException("master not found.");
}
var sourcePath = $"{master.AccessProtocol}://{master.Host}/static/downloadpluginfile?pluginname={model.AssemblyName}";
var zipPath = $"{ConfigurationCache.PluginPathPrefix}\\{model.Id.ToString("n")}.zip".ToPhysicalPath();
try
{
//下载文件
var httpClient = scope.GetService <IHttpClientFactory>().CreateClient();
var array = await httpClient.GetByteArrayAsync(sourcePath);
System.IO.FileStream fs = new System.IO.FileStream(zipPath, System.IO.FileMode.Create);
fs.Write(array, 0, array.Length);
fs.Close();
fs.Dispose();
}
catch (Exception ex)
{
LogHelper.Warn($"下载程序包异常，地址：{sourcePath}", model.Id);
throw ex.InnerException ?? ex;
}
//将指定 zip 存档中的所有文件都解压缩到各自对应的目录下
ZipFile.ExtractToDirectory(zipPath, pluginPath, true);
System.IO.File.Delete(zipPath);
}
}
}
public async Task Execute(IJobExecutionContext context)
{
_sid = Guid.Parse(context.JobDetail.Key.Name);
using (var scope = new ScopeDbContext())
{
_db = scope.GetDbContext();
var locker = scope.GetService <HosLock.IHosLock>();
if (locker.TryGetLock(context.JobDetail.Key.Name))
{
await InnerRun(context);
}
else
{
throw new JobExecutionException("lock_failed");
}
}
}
public async Task Execute(IJobExecutionContext context)
{
using (var scope = new ScopeDbContext())
{
var _db = scope.GetDbContext();
var stoppedList = (from s in _db.Schedules
join n in _db.ScheduleExecutors on s.Id equals n.ScheduleId
where s.Status == (int)ScheduleStatus.Stop && n.WorkerName == ConfigurationCache.NodeSetting.IdentityName
select n.ScheduleId
).ToList();
foreach (var sid in stoppedList)
{
JobKey jk = new JobKey(sid.ToString().ToLower());
if (await context.Scheduler.CheckExists(jk))
{
await QuartzManager.Stop(sid);
}
}
}
}
private static async Task LoadPluginFile(ScheduleEntity model)
{
bool pull = true;
var pluginPath = $"{ConfigurationCache.PluginPathPrefix}\\{model.Id}".ToPhysicalPath();
//看一下拉取策略
string policy = ConfigurationCache.GetField <string>("Assembly_ImagePullPolicy");
if (policy == "IfNotPresent" && System.IO.Directory.Exists(pluginPath))
{
pull = false;
}
if (pull)
{
using (var scope = new ScopeDbContext())
{
var master = scope.GetDbContext().ServerNodes.FirstOrDefault(x => x.NodeType == "master");
if (master == null)
{
throw new InvalidOperationException("master not found.");
}
var sourcePath = $"{master.AccessProtocol}://{master.Host}/static/downloadpluginfile?pluginname={model.AssemblyName}";
var zipPath = $"{ConfigurationCache.PluginPathPrefix}\\{model.Id.ToString("n")}.zip".ToPhysicalPath();
using (WebClient client = new WebClient())
{
try
{
await client.DownloadFileTaskAsync(new Uri(sourcePath), zipPath);
}
catch (Exception ex)
{
LogHelper.Warn($"下载程序包异常，地址：{sourcePath}", model.Id);
throw ex;
}
}
//将指定 zip 存档中的所有文件都解压缩到各自对应的目录下
ZipFile.ExtractToDirectory(zipPath, pluginPath, true);
System.IO.File.Delete(zipPath);
}
}
}
private static async Task NotifyRequest(NotifyPlan plan)
{
Guid sid = Guid.Parse(plan.Key);
Guid traceId = Guid.NewGuid();
using (var scope = new ScopeDbContext())
{
var db = scope.GetDbContext();
var tracer = scope.GetService <Common.RunTracer>();
var entity = await db.ScheduleDelayeds.FirstOrDefaultAsync(x => x.Id == sid);
if (entity == null)
{
LogHelper.Info($"不存在的任务ID。", sid, traceId);
return;
}
entity.ExecuteTime = DateTime.Now;
Exception failedException = null;
try
{
//创建一条trace
await tracer.Begin(traceId, plan.Key);
var httpClient = scope.GetService <IHttpClientFactory>().CreateClient();
plan.NotifyBody = plan.NotifyBody.Replace("\r\n", "");
HttpContent reqContent = new StringContent(plan.NotifyBody, System.Text.Encoding.UTF8, "application/json");
if (plan.NotifyDataType == "application/x-www-form-urlencoded")
{
//任务创建时要确保参数是键值对的json格式
reqContent = new FormUrlEncodedContent(Newtonsoft.Json.JsonConvert.DeserializeObject <IEnumerable <KeyValuePair <string, string> > >(plan.NotifyBody));
}
LogHelper.Info($"即将请求：{entity.NotifyUrl}", sid, traceId);
var response = await httpClient.PostAsync(plan.NotifyUrl, reqContent);
var content = await response.Content.ReadAsStringAsync();
LogHelper.Info($"请求结束，响应码：{response.StatusCode.GetHashCode().ToString()}，响应内容：{(response.Content.Headers.GetValues("Content-Type").Any(x => x.Contains("text/html")) ? "html文档" : content)}", sid, traceId);
if (response.IsSuccessStatusCode && content.Contains("success"))
{
await tracer.Complete(ScheduleRunResult.Success);
//更新结果字段
entity.FinishTime = DateTime.Now;
entity.Status = (int)ScheduleDelayStatus.Successed;
//更新日志
LogHelper.Info($"延时任务[{entity.Topic}:{entity.ContentKey}]执行成功。", sid, traceId);
}
else
{
failedException = new Exception("异常的返回结果。");
}
}
catch (Exception ex)
{
failedException = ex;
}
// 对异常进行处理
if (failedException != null)
{
//更新trace
await tracer.Complete(ScheduleRunResult.Failed);
//失败重试策略
int maxRetry = ConfigurationCache.GetField <int>("DelayTask_RetryTimes");
if (entity.FailedRetrys < (maxRetry > 0 ? maxRetry : 3))
{
//更新结果字段
entity.FailedRetrys++;
//计算下次延时间隔
int timespan = ConfigurationCache.GetField <int>("DelayTask_RetrySpans");
int delay = (timespan > 0 ? timespan : 10) * entity.FailedRetrys;
//重新进入延时队列
Insert(plan, delay);
//更新日志
LogHelper.Error($"延时任务[{entity.Topic}:{entity.ContentKey}]执行失败，将在{delay.ToString()}秒后开始第{entity.FailedRetrys.ToString()}次重试。", failedException, sid, traceId);
}
else
{
entity.Status = (int)ScheduleDelayStatus.Failed;
entity.Remark = $"重试{entity.FailedRetrys}次后失败结束";
//更新日志
LogHelper.Error($"延时任务[{entity.Topic}:{entity.ContentKey}]重试{entity.FailedRetrys}次后失败结束。", failedException, sid, traceId);
//邮件通知
var user = await db.SystemUsers.FirstOrDefaultAsync(x => x.UserName == entity.CreateUserName && !string.IsNullOrEmpty(x.Email));
if (user != null)
{
var keeper = new List <KeyValuePair <string, string> >()
{
new KeyValuePair <string, string>(user.RealName, user.Email)
};
MailKitHelper.SendMail(keeper, $"延时任务异常 — {entity.Topic}:{entity.ContentKey}",
Common.QuartzManager.GetErrorEmailContent($"{entity.Topic}:{entity.ContentKey}", failedException)
, sid: sid, traceId: traceId);
}
else
{
LogHelper.Error($"用户无邮箱，无法发送邮件：{entity.CreateUserName}", new Exception("用户无邮箱"), sid, traceId);
}
}
// .....
// 其实这个重试策略稍微有点问题，只能在抢锁成功的节点上进行重试，如果遭遇单点故障会导致任务丢失
// 严格来说应该通知到master让其对所有节点执行重试策略，但考虑到master也会有单点问题，综合考虑后还是放到当前worker中重试，若worker节点异常可以在控制台中人工干预进行重置或立即执行
}
db.Update(entity);
await db.SaveChangesAsync();
}
}
public Task Execute(IJobExecutionContext context)
{
_sid = Guid.Parse(context.JobDetail.Key.Name);
using (var scope = new ScopeDbContext())
{
_db = scope.GetDbContext();
if (!_db.Schedules.Any(x => x.Id == _sid && x.Status == (int)ScheduleStatus.Running))
{
LogHelper.Warn("不存在或没有启动的任务", _sid);
throw new JobExecutionException("不存在或没有启动的任务");
}
bool getLocked = _db.Database.ExecuteSqlRaw($"UPDATE schedulelocks SET Status=1 WHERE ScheduleId='{_sid.ToString()}' and Status=0") > 0;
if (getLocked)
{
LogHelper.Info($"节点{node}抢锁成功！准备执行任务....", _sid);
IJobDetail job = context.JobDetail;
try
{
if (job.JobDataMap["instance"] is TaskBase instance)
{
Guid traceId = GreateRunTrace();
Stopwatch stopwatch = new Stopwatch();
TaskContext tctx = new TaskContext(instance);
tctx.Node = node;
tctx.TaskId = _sid;
tctx.TraceId = traceId;
tctx.ParamsDict = job.JobDataMap["params"] as Dictionary <string, object>;
if (context.MergedJobDataMap["PreviousResult"] is object prev)
{
tctx.PreviousResult = prev;
}
try
{
stopwatch.Restart();
instance.InnerRun(tctx);
stopwatch.Stop();
UpdateRunTrace(traceId, Math.Round(stopwatch.Elapsed.TotalSeconds, 3), ScheduleRunResult.Success);
LogHelper.Info($"任务[{job.JobDataMap["name"]}]运行成功！用时{stopwatch.Elapsed.TotalMilliseconds.ToString()}ms", _sid, traceId);
//保存运行结果用于子任务触发
context.Result = tctx.Result;
}
catch (RunConflictException conflict)
{
stopwatch.Stop();
UpdateRunTrace(traceId, Math.Round(stopwatch.Elapsed.TotalSeconds, 3), ScheduleRunResult.Conflict);
throw conflict;
}
catch (Exception e)
{
stopwatch.Stop();
UpdateRunTrace(traceId, Math.Round(stopwatch.Elapsed.TotalSeconds, 3), ScheduleRunResult.Failed);
LogHelper.Error($"任务\"{job.JobDataMap["name"]}\"运行失败！", e, _sid, traceId);
//这里抛出的异常会在JobListener的JobWasExecuted事件中接住
//如果吃掉异常会导致程序误以为本次任务执行成功
throw new BusinessRunException(e);
}
}
}
finally
{
//为了避免各节点之间的时间差，延迟1秒释放锁
System.Threading.Thread.Sleep(1000);
_db.Database.ExecuteSqlRaw($"UPDATE schedulelocks SET Status=0 WHERE ScheduleId='{_sid.ToString()}'");
}
}
else
{
//LogHelper.Info($"节点{node}抢锁失败！", _sid);
//throw new JobExecutionException("lock_failed");
}
}
return(Task.FromResult(0));
}
public Task Execute(IJobExecutionContext context)
{
_sid = Guid.Parse(context.JobDetail.Key.Name);
using (var scope = new ScopeDbContext())
{
_db = scope.GetDbContext();
bool getLocked = _db.Database.ExecuteSqlRaw($"update schedulelocks set status=1,lockedtime=now(),lockednode='{node}' where scheduleid='{_sid.ToString()}' and status=0") > 0;
if (getLocked)
{
LogHelper.Info($"节点{node}抢锁成功！准备执行任务....", _sid);
IJobDetail job = context.JobDetail;
try
{
if (job.JobDataMap["instance"] is IHosSchedule instance)
{
Guid traceId = GreateRunTrace();
Stopwatch stopwatch = new Stopwatch();
TaskContext tctx = new TaskContext(instance.RunnableInstance);
tctx.Node = node;
tctx.TraceId = traceId;
tctx.ParamsDict = instance.CustomParams;
if (context.MergedJobDataMap["PreviousResult"] is object prev)
{
tctx.PreviousResult = prev;
}
try
{
stopwatch.Restart();
//执行
OnExecuting(tctx);
stopwatch.Stop();
//更新执行结果
UpdateRunTrace(traceId, Math.Round(stopwatch.Elapsed.TotalSeconds, 3), ScheduleRunResult.Success);
LogHelper.Info($"任务[{instance.Main.Title}]运行成功！用时{Math.Round(stopwatch.Elapsed.TotalMilliseconds, 3).ToString()}ms", _sid, traceId);
//保存运行结果用于子任务触发
context.Result = tctx.Result;
}
catch (RunConflictException conflict)
{
stopwatch.Stop();
UpdateRunTrace(traceId, Math.Round(stopwatch.Elapsed.TotalSeconds, 3), ScheduleRunResult.Conflict);
throw conflict;
}
catch (Exception e)
{
stopwatch.Stop();
UpdateRunTrace(traceId, Math.Round(stopwatch.Elapsed.TotalSeconds, 3), ScheduleRunResult.Failed);
LogHelper.Error($"任务\"{instance.Main.Title}\"运行失败！", e, _sid, traceId);
//这里抛出的异常会在JobListener的JobWasExecuted事件中接住
//如果吃掉异常会导致程序误以为本次任务执行成功
throw new BusinessRunException(e);
}
finally
{
OnExecuted(tctx);
}
}
}
finally
{
//为了避免各节点之间的时间差，延迟1秒释放锁
System.Threading.Thread.Sleep(1000);
_db.Database.ExecuteSqlRaw($"update schedulelocks set status=0,lockedtime=null,lockednode=null where scheduleid='{_sid.ToString()}'");
}
}
else
{
//LogHelper.Info($"节点{node}抢锁失败！", _sid);
throw new JobExecutionException("lock_failed");
}
}
return(Task.CompletedTask);
}
