public ResponseModel <AddContinueJobResponseModel> AddContinueJob([FromBody] List <HttpJobItem> httpJobItems) { ResponseModel <AddContinueJobResponseModel> responseModel = new ResponseModel <AddContinueJobResponseModel>(); responseModel.ResultData = new AddContinueJobResponseModel(); var reslut = string.Empty; var jobid = string.Empty; try { httpJobItems.ForEach(k => { if (!string.IsNullOrEmpty(jobid)) { jobid = BackgroundJob.ContinueJobWith(jobid, () => RunContinueJob(k)); } else { jobid = BackgroundJob.Enqueue(() => HttpJob.Excute(k, k.JobName, k.QueueName, k.IsRetry, null)); } }); reslut = "true"; responseModel.ResultData.Result = true; } catch (Exception ex) { responseModel.ResultCode = ResponseStatusCode.Error; responseModel.ResultDesc = ex.Message; } return(responseModel); }
public ResponseModel <AddRecurringJobResponseModel> AddOrUpdateRecurringJob([FromBody] HttpJobItem httpJob) { ResponseModel <AddRecurringJobResponseModel> responseModel = new ResponseModel <AddRecurringJobResponseModel>(); responseModel.ResultData = new AddRecurringJobResponseModel(); try { RecurringJob.AddOrUpdate(httpJob.JobName, () => HttpJob.Excute(httpJob, httpJob.JobName, httpJob.QueueName, httpJob.IsRetry, null), httpJob.Cron, TimeZoneInfo.Local); responseModel.ResultData.Result = true; } catch (Exception ex) { responseModel.ResultCode = ResponseStatusCode.Error; responseModel.ResultDesc = ex.Message; } return(responseModel); }
public ResponseModel <AddScheduleJobResponseModel> AddScheduleJob([FromBody] HttpJobItem httpJob) { ResponseModel <AddScheduleJobResponseModel> responseModel = new ResponseModel <AddScheduleJobResponseModel>(); responseModel.ResultData = new AddScheduleJobResponseModel(); var reslut = string.Empty; try { reslut = BackgroundJob.Schedule(() => HttpJob.Excute(httpJob, httpJob.JobName, httpJob.QueueName, false, null), TimeSpan.FromMinutes(httpJob.DelayFromMinutes)); responseModel.ResultData.JobId = reslut; } catch (Exception ex) { responseModel.ResultCode = ResponseStatusCode.Error; responseModel.ResultDesc = ex.Message; } return(responseModel); }
/// <summary> /// 测试计划任务(httpjob) /// </summary> /// <param name="minutes">多少分钟后执行</param> public virtual void AddJobTest(int minutes) { // 获取配置信息方式一,通过配置管理器 var deminutes = Convert.ToInt32(_settingManager.GetSettingValue("config.defaultMinutes")); //获取配置信息方式二,通过配置文件接口获取 var deminutes2 = Convert.ToInt32(_appConfigurtaionService.AppConfigurations.defaultMinutes); var job = new HttpJobItem() { JobName = "PlanJob", QueueName = "apis", Method = "get", Url = "https://www.baidu.com", IsRetry = false, Corn = "", DelayFromMinutes = minutes > 0 ? minutes : deminutes }; BackgroundJob.Schedule(() => HttpJob.Excute(job, job.JobName, job.QueueName, job.IsRetry, null), TimeSpan.FromMinutes(job.DelayFromMinutes)); }
public ResponseModel <AddBackgroundJobResponseModel> AddBackGroundJob([FromBody] HttpJobItem httpJob) { ResponseModel <AddBackgroundJobResponseModel> responseModel = new ResponseModel <AddBackgroundJobResponseModel>(); responseModel.ResultData = new AddBackgroundJobResponseModel(); var addreslut = string.Empty; try { addreslut = BackgroundJob.Enqueue(() => HttpJob.Excute(httpJob, httpJob.JobName, httpJob.QueueName, false, null)); responseModel.ResultData.JobId = addreslut; } catch (Exception ex) { responseModel.ResultCode = ResponseStatusCode.Error; responseModel.ResultDesc = ex.Message; //return Json(new Message() { Code = false, ErrorMessage = ec.ToString() }); } return(responseModel); }
/// <summary> /// 给agent发送命令后 agent自己会上报数据到storage的 /// </summary> /// <returns></returns> private static async Task SendHeartbeat(string agentKey, string url, string basicUserName, string basicPassword) { var agentServerId = string.Empty; try { var stroageString = HttpJob.GetJobStorage().Value; if (string.IsNullOrEmpty(stroageString)) { return; } HttpClient client = HangfireHttpClientFactory.HttpJobInstance.GetHttpClient(url); var request = new HttpRequestMessage(new HttpMethod("Post"), url); request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); if (!string.IsNullOrEmpty(basicUserName) && !string.IsNullOrEmpty(basicPassword)) { var byteArray = Encoding.ASCII.GetBytes(basicUserName + ":" + basicPassword); request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray)); } request.Headers.Add("x-job-storage", Convert.ToBase64String(Encoding.UTF8.GetBytes(stroageString))); request.Headers.Add("x-job-agent-action", "heartbeat"); var uri = new Uri(url); request.Headers.Add("x-job-agent-server", uri.Host + ":" + uri.Port); if (!string.IsNullOrEmpty(ProcessMonitor.CurrentServerId)) { request.Headers.Add("x-job-server", Convert.ToBase64String(Encoding.UTF8.GetBytes(ProcessMonitor.CurrentServerId))); } var cancelToken = new CancellationTokenSource(TimeSpan.FromSeconds(5)); var content = await client.SendAsync(request, cancelToken.Token); //jobagent的话 在header里面有一个agentServerId agentServerId = content.Headers.GetValues("agentServerId").FirstOrDefault() ?? ""; } catch (Exception e) { //ignore agent挂了就到这 Logger.ErrorException("send agent heartbeat fail,agent:" + url, e); } try { using (var connection = JobStorage.Current.GetConnection()) using (var writeTransaction = connection.CreateWriteTransaction()) { var values = new Dictionary <string, string> { [agentKey] = SerializationHelper.Serialize(new { id = agentServerId, time = DateTime.UtcNow }) }; var hashKey = "activeAgent:" + agentKey; writeTransaction.SetRangeInHash(hashKey, values); if (writeTransaction is JobStorageTransaction jsTransaction) { jsTransaction.ExpireHash(hashKey, TimeSpan.FromDays(1)); } writeTransaction.Commit(); } } catch (Exception e) { //ignore Logger.ErrorException("get agent heartbeat fail", e); } }
private static void OnVerify(object state) { mDetectionTimer.Change(-1, -1); try { using (var connection = JobStorage.Current.GetConnection()) using (var lockStorage = connection.AcquireDistributedLock("JobAgentServer", TimeSpan.FromSeconds(30)))//防止多个server端竞争 { //拿到有上报的jobId集合 var jobIdList = connection.GetAllItemsFromSet(keyPrefix); if (!jobIdList.Any()) { return; } foreach (var jobId in jobIdList) { JobData jobData = connection.GetJobData(jobId); //拿到真正的运行结果 var hashKey = keyPrefix + jobId; var result = connection.GetAllEntriesFromHash(hashKey); using (var tran = connection.CreateWriteTransaction()) { //job已经不存在了 就直接删除set if (jobData == null) { tran.RemoveFromSet(keyPrefix, jobId); tran.Commit(); continue; } double totalMilliseconds = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds; long latency = (long)totalMilliseconds; //如果job存在 但是没有拿到hash数据 认为成功 if (result == null || !result.Any()) { tran.SetJobState(jobId, new SucceededState(jobId, latency, latency)); tran.RemoveFromSet(keyPrefix, jobId); tran.RemoveHash(hashKey); tran.Commit(); continue; } var resultOfAgent = result.First(); JobAgentResult resultData = CodingUtil.FromJson <JobAgentResult>(resultOfAgent.Value); //异常数据 认为成功 if (resultData == null) { tran.SetJobState(jobId, new SucceededState(jobId, latency, latency)); tran.RemoveFromSet(keyPrefix, jobId); tran.RemoveHash(hashKey); tran.Commit(); continue; } //jobagent实际上运行的时长 long.TryParse(resultOfAgent.Key, out var realTotalMilliseconds); if (realTotalMilliseconds < 1) { realTotalMilliseconds = latency; } var isSuccess = resultData.R == "ok"; tran.RemoveFromSet(keyPrefix, jobId); tran.RemoveHash(hashKey); // latency 代表的是 从开始调度 到 实际结束 总共的时长 // realTotalMilliseconds 代表的是 jobagent开始执行 到 实际结束的 总共的时长 if (isSuccess) { var currentState = connection.GetStateData(jobId); if (currentState != null && !string.IsNullOrEmpty(currentState.Name) && currentState.Name.Equals("Failed")) { tran.AddJobState(jobId, new SucceededState(null, latency, realTotalMilliseconds)); } else { new BackgroundJobClient().ChangeState(jobId, new SucceededState(null, latency, realTotalMilliseconds)); } } else { var jobItem = jobData.Job.Args.FirstOrDefault() as HttpJobItem; var ex = new AgentJobException(jobItem.AgentClass, resultData.E); new BackgroundJobClient().ChangeState(jobId, new FailedState(ex)); HttpJob.SendFail(jobId, jobItem, "AgentJobFail", ex); } //如果是stop上报过来的时候 记录这个job最后的执行id if (!string.IsNullOrEmpty(resultData.Action) && resultData.Action.Equals("stop") && !string.IsNullOrEmpty(resultData.RunId)) { var jobItem = jobData.Job.Args.FirstOrDefault() as HttpJobItem; var jobKeyName = $"recurring-job:{(!string.IsNullOrEmpty(jobItem.RecurringJobIdentifier) ? jobItem.RecurringJobIdentifier : jobItem.JobName)}"; tran.SetRangeInHash(jobKeyName, new List <KeyValuePair <string, string> > { new KeyValuePair <string, string>("LastJobId", resultData.RunId) }); } //出错的话 需要走通用的出错流程 tran.Commit(); } } } } catch (Exception e) { Logger.ErrorException("agent reporter fail", e); } finally { mDetectionTimer.Change(1000 * 2, 1000 * 2); } }
/// <summary> /// 执行连续任务 /// </summary> /// <param name="httpJob"></param> public void RunContinueJob(HttpJobItem httpJob) { BackgroundJob.Enqueue(() => HttpJob.Excute(httpJob, httpJob.JobName, httpJob.QueueName, false, null)); }
private static void OnVerify(object state) { mDetectionTimer.Change(-1, -1); if (string.IsNullOrEmpty(ProcessMonitor.CurrentServerId)) { mDetectionTimer.Change(1000 * 1, 1000 * 1); return; } try { var api = JobStorage.Current.GetMonitoringApi(); var totalCount = api.ProcessingCount(); if (totalCount < 1) { return; } var pageSize = 50; var totalPages = (totalCount - 1) / pageSize + 1; for (int i = 0; i < totalPages; i++) { var list = api.ProcessingJobs(pageSize * i, pageSize); using (var connection = JobStorage.Current.GetConnection()) { foreach (var job in list) { if (job.Value.StartedAt != null && (DateTime.UtcNow - job.Value.StartedAt.Value).TotalMinutes < 5) { //给5分钟缓冲 continue; } var jobItem = job.Value.Job.Args.FirstOrDefault() as HttpJobItem; //如果是起多个server的情况 检查 job.Value.ServerId 是否还存活 var servers = JobStorage.Current.GetMonitoringApi().Servers(); var targetServer = servers.FirstOrDefault(r => r.Name.Equals(job.Value.ServerId)); //如果server不存在了 或者 server的最后心跳已经是10分钟之前了 if (targetServer == null || (targetServer.Heartbeat != null && (DateTime.UtcNow - targetServer.Heartbeat.Value).TotalMinutes > 10)) { //hangfire的server挂了导致滞留在processing的job //转移到error里面去 var ex1 = new HangfireServerShutDownError(); backClient.ChangeState(job.Key, new FailedState(ex1)); if (jobItem != null) { HttpJob.SendFail(job.Key, jobItem, "AgentJobFail", ex1); } continue; } //查看是否是agentJob if (jobItem == null || string.IsNullOrEmpty(jobItem.AgentClass)) { continue; } //查看这个job的运行的参数是哪个agentserverId var agentServerId = SerializationHelper.Deserialize <string>(connection.GetJobParameter(job.Key, "agentServerId"), SerializationOption.User); if (string.IsNullOrEmpty(agentServerId)) { continue; } //查看这个agent最新的 agentserverId var uri = new Uri(jobItem.Url); var key = uri.Host + ":" + uri.Port; var lastAgentServerHash = connection.GetAllEntriesFromHash("activeAgent:" + key); if (lastAgentServerHash == null || lastAgentServerHash.Count < 1) { continue; } var lastAgentServer = lastAgentServerHash.First().Value; if (string.IsNullOrEmpty(lastAgentServer)) { continue; } var currentAgentServer = SerializationHelper.Deserialize <AgentServerInfo>(lastAgentServer); if (currentAgentServer == null) { continue; } if (!string.IsNullOrEmpty(currentAgentServer.Id) && currentAgentServer.Id == agentServerId) { //虽然没有变化 但是agent其实已经挂了 (其实已经最少挂了15分钟了) if (currentAgentServer.Time != DateTime.MinValue && (DateTime.UtcNow - currentAgentServer.Time).TotalMinutes > 10) { var ex2 = new HangfireAgentShutDownError(); backClient.ChangeState(job.Key, new FailedState(ex2)); HttpJob.SendFail(job.Key, jobItem, "AgentJobFail", ex2); } continue; } //说明这个agent已经挂了 /说明这个agent重启了 var ex = new HangfireAgentShutDownError(); backClient.ChangeState(job.Key, new FailedState(ex)); HttpJob.SendFail(job.Key, jobItem, "AgentJobFail", ex); } } } } catch { //ignore } finally { mDetectionTimer.Change(1000 * 5, 1000 * 5); } }