public ResponseModel <AddContinueJobResponseModel> AddContinueJob([FromBody] List <HttpJobItem> httpJobItems)
{
ResponseModel <AddContinueJobResponseModel> responseModel = new ResponseModel <AddContinueJobResponseModel>();
responseModel.ResultData = new AddContinueJobResponseModel();
var reslut = string.Empty;
var jobid = string.Empty;
try
{
httpJobItems.ForEach(k =>
{
if (!string.IsNullOrEmpty(jobid))
{
jobid = BackgroundJob.ContinueJobWith(jobid, () => RunContinueJob(k));
}
else
{
jobid = BackgroundJob.Enqueue(() => HttpJob.Excute(k, k.JobName, k.QueueName, k.IsRetry, null));
}
});
reslut = "true";
responseModel.ResultData.Result = true;
}
catch (Exception ex)
{
responseModel.ResultCode = ResponseStatusCode.Error;
responseModel.ResultDesc = ex.Message;
}
return(responseModel);
}
public ResponseModel <AddRecurringJobResponseModel> AddOrUpdateRecurringJob([FromBody] HttpJobItem httpJob)
{
ResponseModel <AddRecurringJobResponseModel> responseModel = new ResponseModel <AddRecurringJobResponseModel>();
responseModel.ResultData = new AddRecurringJobResponseModel();
try
{
RecurringJob.AddOrUpdate(httpJob.JobName, () => HttpJob.Excute(httpJob, httpJob.JobName, httpJob.QueueName, httpJob.IsRetry, null), httpJob.Cron, TimeZoneInfo.Local);
responseModel.ResultData.Result = true;
}
catch (Exception ex)
{
responseModel.ResultCode = ResponseStatusCode.Error;
responseModel.ResultDesc = ex.Message;
}
return(responseModel);
}
public ResponseModel <AddScheduleJobResponseModel> AddScheduleJob([FromBody] HttpJobItem httpJob)
{
ResponseModel <AddScheduleJobResponseModel> responseModel = new ResponseModel <AddScheduleJobResponseModel>();
responseModel.ResultData = new AddScheduleJobResponseModel();
var reslut = string.Empty;
try
{
reslut = BackgroundJob.Schedule(() => HttpJob.Excute(httpJob, httpJob.JobName, httpJob.QueueName, false, null), TimeSpan.FromMinutes(httpJob.DelayFromMinutes));
responseModel.ResultData.JobId = reslut;
}
catch (Exception ex)
{
responseModel.ResultCode = ResponseStatusCode.Error;
responseModel.ResultDesc = ex.Message;
}
return(responseModel);
}
/// <summary>
/// 测试计划任务(httpjob)
/// </summary>
/// <param name="minutes">多少分钟后执行</param>
public virtual void AddJobTest(int minutes)
{
// 获取配置信息方式一，通过配置管理器
var deminutes = Convert.ToInt32(_settingManager.GetSettingValue("config.defaultMinutes"));
//获取配置信息方式二，通过配置文件接口获取
var deminutes2 = Convert.ToInt32(_appConfigurtaionService.AppConfigurations.defaultMinutes);
var job = new HttpJobItem()
{
JobName = "PlanJob",
QueueName = "apis",
Method = "get",
Url = "https://www.baidu.com",
IsRetry = false,
Corn = "",
DelayFromMinutes = minutes > 0 ? minutes : deminutes
};
BackgroundJob.Schedule(() => HttpJob.Excute(job, job.JobName, job.QueueName, job.IsRetry, null), TimeSpan.FromMinutes(job.DelayFromMinutes));
}
public ResponseModel <AddBackgroundJobResponseModel> AddBackGroundJob([FromBody] HttpJobItem httpJob)
{
ResponseModel <AddBackgroundJobResponseModel> responseModel = new ResponseModel <AddBackgroundJobResponseModel>();
responseModel.ResultData = new AddBackgroundJobResponseModel();
var addreslut = string.Empty;
try
{
addreslut = BackgroundJob.Enqueue(() => HttpJob.Excute(httpJob, httpJob.JobName, httpJob.QueueName, false, null));
responseModel.ResultData.JobId = addreslut;
}
catch (Exception ex)
{
responseModel.ResultCode = ResponseStatusCode.Error;
responseModel.ResultDesc = ex.Message;
//return Json(new Message() { Code = false, ErrorMessage = ec.ToString() });
}
return(responseModel);
}
/// <summary>
/// 给agent发送命令后 agent自己会上报数据到storage的
/// </summary>
/// <returns></returns>
private static async Task SendHeartbeat(string agentKey, string url, string basicUserName, string basicPassword)
{
var agentServerId = string.Empty;
try
{
var stroageString = HttpJob.GetJobStorage().Value;
if (string.IsNullOrEmpty(stroageString))
{
return;
}
HttpClient client = HangfireHttpClientFactory.HttpJobInstance.GetHttpClient(url);
var request = new HttpRequestMessage(new HttpMethod("Post"), url);
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
if (!string.IsNullOrEmpty(basicUserName) && !string.IsNullOrEmpty(basicPassword))
{
var byteArray = Encoding.ASCII.GetBytes(basicUserName + ":" + basicPassword);
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
}
request.Headers.Add("x-job-storage", Convert.ToBase64String(Encoding.UTF8.GetBytes(stroageString)));
request.Headers.Add("x-job-agent-action", "heartbeat");
var uri = new Uri(url);
request.Headers.Add("x-job-agent-server", uri.Host + ":" + uri.Port);
if (!string.IsNullOrEmpty(ProcessMonitor.CurrentServerId))
{
request.Headers.Add("x-job-server", Convert.ToBase64String(Encoding.UTF8.GetBytes(ProcessMonitor.CurrentServerId)));
}
var cancelToken = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var content = await client.SendAsync(request, cancelToken.Token);
//jobagent的话 在header里面有一个agentServerId
agentServerId = content.Headers.GetValues("agentServerId").FirstOrDefault() ?? "";
}
catch (Exception e)
{
//ignore agent挂了就到这
Logger.ErrorException("send agent heartbeat fail,agent:" + url, e);
}
try
{
using (var connection = JobStorage.Current.GetConnection())
using (var writeTransaction = connection.CreateWriteTransaction())
{
var values = new Dictionary <string, string>
{
[agentKey] = SerializationHelper.Serialize(new { id = agentServerId, time = DateTime.UtcNow })
};
var hashKey = "activeAgent:" + agentKey;
writeTransaction.SetRangeInHash(hashKey, values);
if (writeTransaction is JobStorageTransaction jsTransaction)
{
jsTransaction.ExpireHash(hashKey, TimeSpan.FromDays(1));
}
writeTransaction.Commit();
}
}
catch (Exception e)
{
//ignore
Logger.ErrorException("get agent heartbeat fail", e);
}
}
private static void OnVerify(object state)
{
mDetectionTimer.Change(-1, -1);
try
{
using (var connection = JobStorage.Current.GetConnection())
using (var lockStorage = connection.AcquireDistributedLock("JobAgentServer", TimeSpan.FromSeconds(30)))//防止多个server端竞争
{
//拿到有上报的jobId集合
var jobIdList = connection.GetAllItemsFromSet(keyPrefix);
if (!jobIdList.Any())
{
return;
}
foreach (var jobId in jobIdList)
{
JobData jobData = connection.GetJobData(jobId);
//拿到真正的运行结果
var hashKey = keyPrefix + jobId;
var result = connection.GetAllEntriesFromHash(hashKey);
using (var tran = connection.CreateWriteTransaction())
{
//job已经不存在了 就直接删除set
if (jobData == null)
{
tran.RemoveFromSet(keyPrefix, jobId);
tran.Commit();
continue;
}
double totalMilliseconds = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
long latency = (long)totalMilliseconds;
//如果job存在 但是没有拿到hash数据 认为成功
if (result == null || !result.Any())
{
tran.SetJobState(jobId, new SucceededState(jobId, latency, latency));
tran.RemoveFromSet(keyPrefix, jobId);
tran.RemoveHash(hashKey);
tran.Commit();
continue;
}
var resultOfAgent = result.First();
JobAgentResult resultData = CodingUtil.FromJson <JobAgentResult>(resultOfAgent.Value);
//异常数据 认为成功
if (resultData == null)
{
tran.SetJobState(jobId, new SucceededState(jobId, latency, latency));
tran.RemoveFromSet(keyPrefix, jobId);
tran.RemoveHash(hashKey);
tran.Commit();
continue;
}
//jobagent实际上运行的时长
long.TryParse(resultOfAgent.Key, out var realTotalMilliseconds);
if (realTotalMilliseconds < 1)
{
realTotalMilliseconds = latency;
}
var isSuccess = resultData.R == "ok";
tran.RemoveFromSet(keyPrefix, jobId);
tran.RemoveHash(hashKey);
// latency 代表的是 从开始调度 到 实际结束 总共的时长
// realTotalMilliseconds 代表的是 jobagent开始执行 到 实际结束的 总共的时长
if (isSuccess)
{
var currentState = connection.GetStateData(jobId);
if (currentState != null && !string.IsNullOrEmpty(currentState.Name) &&
currentState.Name.Equals("Failed"))
{
tran.AddJobState(jobId, new SucceededState(null, latency, realTotalMilliseconds));
}
else
{
new BackgroundJobClient().ChangeState(jobId, new SucceededState(null, latency, realTotalMilliseconds));
}
}
else
{
var jobItem = jobData.Job.Args.FirstOrDefault() as HttpJobItem;
var ex = new AgentJobException(jobItem.AgentClass, resultData.E);
new BackgroundJobClient().ChangeState(jobId, new FailedState(ex));
HttpJob.SendFail(jobId, jobItem, "AgentJobFail", ex);
}
//如果是stop上报过来的时候 记录这个job最后的执行id
if (!string.IsNullOrEmpty(resultData.Action) && resultData.Action.Equals("stop") && !string.IsNullOrEmpty(resultData.RunId))
{
var jobItem = jobData.Job.Args.FirstOrDefault() as HttpJobItem;
var jobKeyName =
$"recurring-job:{(!string.IsNullOrEmpty(jobItem.RecurringJobIdentifier) ? jobItem.RecurringJobIdentifier : jobItem.JobName)}";
tran.SetRangeInHash(jobKeyName, new List <KeyValuePair <string, string> > {
new KeyValuePair <string, string>("LastJobId", resultData.RunId)
});
}
//出错的话 需要走通用的出错流程
tran.Commit();
}
}
}
}
catch (Exception e)
{
Logger.ErrorException("agent reporter fail", e);
}
finally
{
mDetectionTimer.Change(1000 * 2, 1000 * 2);
}
}
/// <summary>
/// 执行连续任务
/// </summary>
/// <param name="httpJob"></param>
public void RunContinueJob(HttpJobItem httpJob)
{
BackgroundJob.Enqueue(() => HttpJob.Excute(httpJob, httpJob.JobName, httpJob.QueueName, false, null));
}
private static void OnVerify(object state)
{
mDetectionTimer.Change(-1, -1);
if (string.IsNullOrEmpty(ProcessMonitor.CurrentServerId))
{
mDetectionTimer.Change(1000 * 1, 1000 * 1);
return;
}
try
{
var api = JobStorage.Current.GetMonitoringApi();
var totalCount = api.ProcessingCount();
if (totalCount < 1)
{
return;
}
var pageSize = 50;
var totalPages = (totalCount - 1) / pageSize + 1;
for (int i = 0; i < totalPages; i++)
{
var list = api.ProcessingJobs(pageSize * i, pageSize);
using (var connection = JobStorage.Current.GetConnection())
{
foreach (var job in list)
{
if (job.Value.StartedAt != null && (DateTime.UtcNow - job.Value.StartedAt.Value).TotalMinutes < 5)
{
//给5分钟缓冲
continue;
}
var jobItem = job.Value.Job.Args.FirstOrDefault() as HttpJobItem;
//如果是起多个server的情况 检查 job.Value.ServerId 是否还存活
var servers = JobStorage.Current.GetMonitoringApi().Servers();
var targetServer = servers.FirstOrDefault(r => r.Name.Equals(job.Value.ServerId));
//如果server不存在了 或者 server的最后心跳已经是10分钟之前了
if (targetServer == null || (targetServer.Heartbeat != null && (DateTime.UtcNow - targetServer.Heartbeat.Value).TotalMinutes > 10))
{
//hangfire的server挂了导致滞留在processing的job
//转移到error里面去
var ex1 = new HangfireServerShutDownError();
backClient.ChangeState(job.Key, new FailedState(ex1));
if (jobItem != null)
{
HttpJob.SendFail(job.Key, jobItem, "AgentJobFail", ex1);
}
continue;
}
//查看是否是agentJob
if (jobItem == null || string.IsNullOrEmpty(jobItem.AgentClass))
{
continue;
}
//查看这个job的运行的参数是哪个agentserverId
var agentServerId = SerializationHelper.Deserialize <string>(connection.GetJobParameter(job.Key, "agentServerId"), SerializationOption.User);
if (string.IsNullOrEmpty(agentServerId))
{
continue;
}
//查看这个agent最新的 agentserverId
var uri = new Uri(jobItem.Url);
var key = uri.Host + ":" + uri.Port;
var lastAgentServerHash = connection.GetAllEntriesFromHash("activeAgent:" + key);
if (lastAgentServerHash == null || lastAgentServerHash.Count < 1)
{
continue;
}
var lastAgentServer = lastAgentServerHash.First().Value;
if (string.IsNullOrEmpty(lastAgentServer))
{
continue;
}
var currentAgentServer = SerializationHelper.Deserialize <AgentServerInfo>(lastAgentServer);
if (currentAgentServer == null)
{
continue;
}
if (!string.IsNullOrEmpty(currentAgentServer.Id) && currentAgentServer.Id == agentServerId)
{
//虽然没有变化 但是agent其实已经挂了 （其实已经最少挂了15分钟了）
if (currentAgentServer.Time != DateTime.MinValue && (DateTime.UtcNow - currentAgentServer.Time).TotalMinutes > 10)
{
var ex2 = new HangfireAgentShutDownError();
backClient.ChangeState(job.Key, new FailedState(ex2));
HttpJob.SendFail(job.Key, jobItem, "AgentJobFail", ex2);
}
continue;
}
//说明这个agent已经挂了 /说明这个agent重启了
var ex = new HangfireAgentShutDownError();
backClient.ChangeState(job.Key, new FailedState(ex));
HttpJob.SendFail(job.Key, jobItem, "AgentJobFail", ex);
}
}
}
}
catch
{
//ignore
}
finally
{
mDetectionTimer.Change(1000 * 5, 1000 * 5);
}
}