protected override void Run()
{
try
{
var req = new UpdateNodeRefreshCommandQueueStatusRequest()
{
NodeId = GlobalNodeConfig.NodeID, Source = Source.Web, RefreshStatus = RefreshCommandQueueStatus.Refreshing, MonitorClassName = this.GetType().Name
};
var r = NodeProxy.PostToServer <EmptyResponse, UpdateNodeRefreshCommandQueueStatusRequest>(ProxyUrl.UpdateNodeRefreshCmdQueueStatus_Url, req);
if (r.Status != ResponesStatus.Success)
{
string title = "节点(编号):" + GlobalNodeConfig.NodeInfo.nodename + "(" + GlobalNodeConfig.NodeInfo.id.ToString() + ")上报监听命令状态失败,请及时处理!";
StringBuilder strContent = new StringBuilder();
strContent.AppendLine("请求地址:" + ProxyUrl.UpdateNodeRefreshCmdQueueStatus_Url + "<br/>");
strContent.AppendLine("请求参数:" + JsonConvert.SerializeObject(req) + "<br/>");
strContent.AppendLine("响应结果:" + JsonConvert.SerializeObject(r) + "<br/>");
AlarmHelper.AlarmAsync(GlobalNodeConfig.NodeInfo.isenablealarm, (AlarmType)GlobalNodeConfig.NodeInfo.alarmtype, GlobalNodeConfig.Alarm, title, strContent.ToString());
LogProxy.AddNodeErrorLog(strContent.ToString());
}
}
catch (Exception ex)
{
LogProxy.AddNodeErrorLog("节点(" + GlobalNodeConfig.NodeInfo.nodename + "(" + GlobalNodeConfig.NodeInfo.id.ToString() + ")" + ")监听命令队列监控异常:" + ex.Message + ",异常:" + JsonConvert.SerializeObject(ex));
AlarmHelper.AlarmAsync(GlobalNodeConfig.NodeInfo.isenablealarm, (AlarmType)GlobalNodeConfig.NodeInfo.alarmtype, GlobalNodeConfig.Alarm, "节点(" + GlobalNodeConfig.NodeInfo.nodename + "(" + GlobalNodeConfig.NodeInfo.id.ToString() + ")" + ")监听命令队列监控异常", JsonConvert.SerializeObject(ex));
}
}
protected override void Run()
{
lock (lockObj)
{
int monitorstatus = (int)MonitorStatus.Monitoring;
List <tb_nodemonitor> nodemonitorlist = nodemonitorRep.Find(x => x.isdel == 0 && x.monitorstatus == monitorstatus).ToList();
List <tb_user> userList = userRep.Find(x => x.isdel == 0).ToList();
StringBuilder strLog = new StringBuilder();
nodemonitorlist.ForEach(x =>
{
try
{
tb_node node = nodeRep.FindSingle(m => m.id == x.nodeid);
List <int> uidlist = node.alarmuserid.ConvertToList();
List <tb_user> userlist = userList.Where(m => uidlist.Contains(m.id)).ToList();
string alaramperson = node.alarmtype == (int)AlarmType.Email ? string.Join(",", userlist.Select(m => m.useremail).ToArray()) : string.Join(",", userlist.Select(m => m.usermobile).ToArray());
if (x.monitorstatus == (int)MonitorStatus.Monitoring)//运行中
{
#region 检测心跳
double totalsecond = Math.Abs(x.lastmonitortime.Subtract(DateTime.Now).TotalSeconds);
if (totalsecond > (20 + x.interval / 1000) || totalsecond < 0)//大于10s 说明心跳不正常
{
nodemonitorRep.UpdateById(new List <int>()
{
x.id
}, new Dictionary <string, string>()
{
{ "monitorstatus", ((int)NodeStatus.NoRun).ToString() }
}); //, { "refreshcommandqueuestatus", ((int)RefreshCommandQueueStatus.NoRefresh).ToString() }
if (node.nodestatus != (int)NodeStatus.NoRun)
{
string title = "节点(编号):" + node.nodename + "(" + x.nodeid.ToString() + "),监控组件:" +
x.classname + ",心跳异常,已自动更新为未监控状态,请及时处理该节点下该监控组件!";
StringBuilder strContent = new StringBuilder();
strContent.AppendLine("节点(编号):" + node.nodename + "(" + node.id.ToString() +
")<br/>");
strContent.AppendLine("节点监控组件名称(编号):" + x.name + "(" + x.id.ToString() + ")<br/>");
strContent.AppendLine("节点监控组件描述:" + x.discription + "<br/>");
strContent.AppendLine("节点监控类名,命名空间:" + x.classname + "," +
x.classnamespace.ToString() + "<br/>");
strContent.AppendLine("节点监控组件最后一次心跳时间:" + x.lastmonitortime + "<br/>");
AlarmHelper.AlarmAsync(node.isenablealarm, (AlarmType)node.alarmtype, alaramperson,
title, strContent.ToString());
}
}
#endregion
}
}
catch (Exception ex)
{
strLog.AppendLine("监控节点监控组件异常:" + JsonConvert.SerializeObject(ex));
}
});
if (strLog.Length > 0)
{
log.Error(strLog.ToString());
}
}
}
protected override void Run()
{
try
{
var res = NodeProxy.PostToServer <EmptyResponse, UpdateNodeMonitorRequest>(ProxyUrl.UpdateNodeMonitorRequest_Url, new UpdateNodeMonitorRequest()
{
MonitorClassName = this.GetType().Name, MonitorStatus = (int)MonitorStatus.Monitoring, NodeId = GlobalNodeConfig.NodeID, Source = Source.Node
});
if (res.Status != ResponesStatus.Success)
{
LogProxy.AddNodeErrorLog(this.GetType().Name + "监控组件上报监控信息失败,请求地址:" + ProxyUrl.UpdateNodeMonitorRequest_Url + ",返回结果:" + JsonConvert.SerializeObject(res));
}
LoadTaskIdListRequest req = new LoadTaskIdListRequest()
{
Source = Source.Node, TaskScheduleStatus = (int)TaskScheduleStatus.Scheduling, NodeId = GlobalNodeConfig.NodeID
};
var r = NodeProxy.PostToServer <LoadTaskIdListResponse, LoadTaskIdListRequest>(ProxyUrl.LoadTaskIdList_Url, req);
List <int> taskschdulinglist = TaskPoolManager.CreateInstance().GetList().Select(x => x.TaskModel.id).ToList();
if (r.Status != ResponesStatus.Success)
{
//如果服务端没有调度中的，则把本地在调度中的上报状态
if (taskschdulinglist.Count > 0)
{
UploadLocalTask(taskschdulinglist);
}
return;
}
List <int> taskidnotserver = taskschdulinglist.Where(x => !(r.Data.TaskIdList.Contains(x))).ToList();
r.Data.TaskIdList.ForEach(x =>
{
var taskruntimeInfo = TaskPoolManager.CreateInstance().Get(x.ToString());
if (taskruntimeInfo == null)//如果等于空值则报警,说明该任务再
{
string title = "节点(id):" + GlobalNodeConfig.NodeInfo.nodename + "(" + GlobalNodeConfig.NodeInfo.id.ToString() + "),任务id:(" + x + ")" + ",调度异常,请及时处理!";
StringBuilder strContent = new StringBuilder();
strContent.AppendLine("所在节点名称(编号):" + GlobalNodeConfig.NodeInfo.nodename + "(" + GlobalNodeConfig.NodeID + ")<br/>");
strContent.AppendLine("任务编号:" + x + "<br/>");
strContent.AppendLine("服务端任务状态:调度中<br/>");
strContent.AppendLine("节点端任务状态:任务池中已不存在该任务,调度异常<br/>");
AlarmHelper.AlarmAsync(GlobalNodeConfig.NodeInfo.isenablealarm, (AlarmType)GlobalNodeConfig.NodeInfo.alarmtype, GlobalNodeConfig.Alarm, title, strContent.ToString());
LogProxy.AddNodeErrorLog(strContent.ToString());
}
});
if (taskidnotserver.Count > 0)
{
UploadLocalTask(taskidnotserver);
}
}
catch (Exception ex)
{
LogProxy.AddNodeErrorLog("节点(" + GlobalNodeConfig.NodeInfo.nodename + "(" + GlobalNodeConfig.NodeInfo.id.ToString() + ")" + ")监控调度中任务异常:" + ex.Message + ",异常:" + JsonConvert.SerializeObject(ex));
AlarmHelper.AlarmAsync(GlobalNodeConfig.NodeInfo.isenablealarm, (AlarmType)GlobalNodeConfig.NodeInfo.alarmtype, GlobalNodeConfig.Alarm, "节点(" + GlobalNodeConfig.NodeInfo.nodename + "(" + GlobalNodeConfig.NodeInfo.id.ToString() + ")" + ")监控调度中任务异常", JsonConvert.SerializeObject(ex));
}
}
private static void Alaram(UpdateTaskExecuteLogRequest req2, NodeTaskRunTimeInfo task, AbstractTask abstracttask)
{
if (task.TaskModel.isenablealarm == 1)
{
string alarmperson = task.TaskModel.alarmtype == (int)AlarmType.Email ? abstracttask.GetAlarmEmailList() : abstracttask.GetAlarmMobileList();
string title = "任务名称(" + task.TaskModel.taskname + ")" + req2.RunStatus.description() + ",请及时处理!";
StringBuilder strContent = new StringBuilder();
strContent.AppendLine("所在节点名称(编号):" + GlobalNodeConfig.NodeInfo.nodename + "(" + GlobalNodeConfig.NodeID + ")<br/>");
strContent.AppendLine("任务名称(编号):" + task.TaskModel.taskname + "(" + task.TaskModel.id + ")<br/>");
strContent.AppendLine("任务类型:" + ((TaskType)task.TaskModel.tasktype).description() + "<br/>");
strContent.AppendLine("任务执行参数:" + task.TaskVersionModel.taskparams + "<br/>");
strContent.AppendLine("任务执行日志:" + req2.LogMsg + "<br/>");
strContent.AppendLine("任务执行结果,状态:" + req2.RunStatus.description() + ",结果:" + JsonConvert.SerializeObject(req2.TaskResult) + "<br/>");
strContent.AppendLine("任务执行耗时(s):" + req2.TotalRunTime + "<br/>");
AlarmHelper.AlarmAsync(task.TaskModel.isenablealarm, (AlarmType)task.TaskModel.alarmtype, alarmperson, title, strContent.ToString());
}
}
} //从左到右,第一位:代表大版本迭代 ，第二位：代表大版本下的大更新 第三位:代表bug修复次数
/// <summary>
/// 命令执行方法约定
/// </summary>
public void TryExecute()
{
try
{
ShowCommandLog("----------------------" + DateTime.Now + ":开始执行命令-----------------------------------------");
DateTime startTime = DateTime.Now;
Stopwatch sw = new Stopwatch();
RunCommandResult r;
sw.Start();
try
{
r = Execute();
int retryCount = 0;
while (r.ExecuteStatus == ExecuteStatus.ExecuteException && retryCount < this.CommandDetail.maxexeceptionretrycount)
{
int ct = retryCount + 1;
ShowCommandLog("**********第" + ct + "次重试 Start**********");
r = Execute();
retryCount += 1;
ShowCommandLog("**********第" + ct + "次重试 End**********");
}
r.RetryCount += retryCount;
}
catch (Exception ex)
{
r = new RunCommandResult()
{
ExecuteStatus = ExecuteStatus.ExecuteException, Message = ex.Message, Ex = ex
};
ShowCommandLog("执行命令异常,异常信息:" + JsonConvert.SerializeObject(ex));
}
sw.Stop();
ShowCommandLog("----------------------" + DateTime.Now + ":执行命令完成-----------------------------------------");
DateTime endTime = DateTime.Now;
TimeSpan ts = sw.Elapsed;
long times = sw.ElapsedMilliseconds / 1000;//秒
AddCommandExecuteLogRequest addLogReq = new AddCommandExecuteLogRequest()
{
NodeId = GlobalNodeConfig.NodeID,
Source = Source.Node,
CommandEndTime = endTime.ToString("yyyy-MM-dd HH:mm:ss"),
CommandExecuteLog = strLog.ToString(),
CommandParams = JsonConvert.SerializeObject(this.AppConfig),
CommandQueueId = CommandQueue.id,
CommandResult = JsonConvert.SerializeObject(r),
ExecuteStatus = (int)r.ExecuteStatus,
CommandStartTime = startTime.ToString("yyyy-MM-dd HH:mm:ss"),
TotalTime = times.ToString(),
CommandDetailId = CommandDetail.id,
};
//上报命令执行日志和执行结果
var r2 = NodeProxy.PostToServer <EmptyResponse, AddCommandExecuteLogRequest>(ProxyUrl.AddCommandExecuteLog_Url, addLogReq);
if (r2.Status != ResponesStatus.Success)
{
ShowCommandLog("上报命令(" + CommandDetail.commandmainclassname + ")的执行日志失败,请求地址:" + ProxyUrl.AddCommandExecuteLog_Url + ",请求参数:" + JsonConvert.SerializeObject(addLogReq) + ",服务器返回参数:" + JsonConvert.SerializeObject(r2));
}
if (r.ExecuteStatus == ExecuteStatus.ExecuteException)//命令执行异常,报警
{
string title = "当前命令队列(" + CommandQueue.id + ")执行失败,请及时处理";
StringBuilder strContent = new StringBuilder();
strContent.AppendLine("节点编号:" + GlobalNodeConfig.NodeID);
strContent.AppendLine("命令队列编号/命令编号:" + CommandQueue.id + "/" + CommandDetail.id.ToString());
strContent.AppendLine("命令执行参数:" + JsonConvert.SerializeObject(this.AppConfig));
strContent.AppendLine("命令执行起/止时间:" + startTime.ToString("yyyy-MM-dd HH:mm:ss") + "/" + endTime.ToString("yyyy-MM-dd HH:mm:ss"));
strContent.AppendLine("命令执行耗时(s):" + times.ToString());
strContent.AppendLine("命令执行结果,状态:" + r.ExecuteStatus.description() + ",执行结果:" + JsonConvert.SerializeObject(r));
AlarmHelper.AlarmAsync(GlobalNodeConfig.NodeInfo.isenablealarm, (AlarmType)GlobalNodeConfig.NodeInfo.alarmtype, GlobalNodeConfig.Alarm, title, strContent.ToString());
}
}
catch (Exception ex)
{
ShowCommandLog("执行命令异常,异常信息:" + JsonConvert.SerializeObject(ex));
ShowCommandLog("----------------------" + DateTime.Now + ":执行命令完成-----------------------------------------");
log.Error(strLog.ToString());
}
finally
{
strLog.Clear();//正常执行完情况strLog日志
}
}
public void Start()
{
//OnInit("\r\n");
// OnInit("\r\n");
OnInit("----------------------" + DateTime.Now + ":初始化节点开始-----------------------------------------");
try
{
OnInit("**开始请求节点配置信息**");
if (System.Configuration.ConfigurationSettings.AppSettings.AllKeys.Contains("NodeID"))
{
GlobalNodeConfig.NodeID = Convert.ToInt32(System.Configuration.ConfigurationSettings.AppSettings["NodeID"]);
}
var r = NodeProxy.PostToServer <LoadNodeConfigResponse, LoadNodeConfigRequest>(ProxyUrl.LoadNodeConfig_Url, new LoadNodeConfigRequest()
{
NodeId = GlobalNodeConfig.NodeID.ToString(), Source = Source.Node
});
if (r.Status != ResponesStatus.Success)
{
//记录日志，并抛出异常
LogProxy.AddNodeErrorLog("请求" + ProxyUrl.LoadNodeConfig_Url + "获取节点配置失败,服务端返回信息:" + JsonConvert.SerializeObject(r));
throw new Exception("请求" + ProxyUrl.LoadNodeConfig_Url + "获取节点配置失败,服务端返回信息:" + JsonConvert.SerializeObject(r));
}
GlobalNodeConfig.NodeInfo = r.Data.Node;
GlobalNodeConfig.Alarm = r.Data.AlarmPerson.ToString();
if (GlobalNodeConfig.NodeID <= 0)
{
GlobalNodeConfig.NodeID = r.Data.Node.id;
}
//初始化配置信息
OnInit("**请求节点配置信息完成,请求结果:" + JsonConvert.SerializeObject(r) + "**");
//初始化命令池
OnInit("**开启初始化节点命令池**");
ICommandPoolBuilder builder = new CommandPoolBuilder();
builder.OnInitEvent += builder_OnInitEvent;
builder.BuildCommandPool();
OnInit("**初始化节点命令池完成:本地节点命令池数量:" + CommandPoolManager.CreateInstance().GetList().Count + "**");
OnInit("**开始初始化监控线程**");
//初始化监控信息
GlobalNodeConfig.Monitors.Add(new NodeHeartBeatMonitor()); //心跳
GlobalNodeConfig.Monitors.Add(new TaskPerformanceMonitor()); //任务性能监控
GlobalNodeConfig.Monitors.Add(new NodePerformanceMonitor()); //节点性能监控
GlobalNodeConfig.Monitors.Add(new NodeTaskSchedulingMonitor()); //节点调度中的任务监控
GlobalNodeConfig.Monitors.Add(new NodeListenComandQueueMonitor()); //节点监听命令队列监控
GlobalNodeConfig.Monitors.Add(new TimingGcCollectMonitor()); //开启定时收集GC
OnInit("**监控线程已开启**");
OnInit("**开启循环监听命令队列线程**");
IOHelper.CreateDirectory(AppDomain.CurrentDomain.BaseDirectory.TrimEnd('\\') + "\\" + GlobalNodeConfig.TaskSharedDllsDir + @"\");
CommandQueueScanManger.StartScanCommandQueueAsync();//循环监听命令队列线程
OnInit("**循环监听命令队列线程已开启**");
//刷新节点运行状态
var r2 = NodeProxy.PostToServer <EmptyResponse, UpdateNodeStatusRequest>(ProxyUrl.UpdateNodeStatus_Url, new UpdateNodeStatusRequest()
{
NodeId = GlobalNodeConfig.NodeID, Source = Source.Node, NodeStatus = Model.enums.NodeStatus.Running
});
if (r2.Status != ResponesStatus.Success)
{
string msg = "更新节点运行状态失败,请求地址:" + ProxyUrl.UpdateNodeStatus_Url + ",服务器返回参数:" + JsonConvert.SerializeObject(r2);
OnInit(msg);
AlarmHelper.AlarmAsync(GlobalNodeConfig.NodeInfo.isenablealarm, (AlarmType)GlobalNodeConfig.NodeInfo.alarmtype, GlobalNodeConfig.Alarm, "更新节点(" + GlobalNodeConfig.NodeID.ToString() + ")的运行状态失败", msg + ",节点启动日志:" + strLog.ToString());
}
else
{
OnInit("更新节点运行状态成功");
}
OnInit("节点启动成功");
}
catch (Exception ex)
{
string exmsg = "节点启动异常:" + JsonConvert.SerializeObject(ex);
OnInit(exmsg);
AlarmHelper.AlarmAsync(GlobalNodeConfig.NodeInfo.isenablealarm, (AlarmType)GlobalNodeConfig.NodeInfo.alarmtype, GlobalNodeConfig.Alarm, "节点(" + GlobalNodeConfig.NodeID.ToString() + ")启动异常,请注意查看", exmsg + ",节点启动日志:" + strLog.ToString());
}
finally
{
OnInit("----------------------" + DateTime.Now + ":初始化节点结束-----------------------------------------");
LogProxy.AddNodeLog(strLog.ToString(), LogType.NodeInitLog);
//log.Info(strLog.ToString());
// EventLogHelper.Info(strLog.ToString());
strLog.Clear();
}
}
protected override void Run()
{
lock (lockObj)
{
List <tb_node> nodelist = nodeRep.Find(x => x.isdel == 0 && x.ifcheckstate == true).ToList();
List <tb_user> userList = userRep.Find(x => x.isdel == 0).ToList();
StringBuilder strLog = new StringBuilder();
nodelist.ForEach(x =>
{
try
{
List <int> uidlist = x.alarmuserid.ConvertToList();
List <tb_user> userlist = userList.Where(m => uidlist.Contains(m.id)).ToList();
string alaramperson = x.alarmtype == (int)AlarmType.Email ? string.Join(",", userlist.Select(m => m.useremail).ToArray()) : string.Join(",", userlist.Select(m => m.usermobile).ToArray());
if (x.nodestatus == (int)NodeStatus.Running)//运行中
{
#region 检测心跳
double totalsecond = Math.Abs(x.nodelastupdatetime.Subtract(DateTime.Now).TotalSeconds);
if (totalsecond > 20 || totalsecond < 0)//大于10s 说明心跳不正常
{
List <int> taskidlist = new List <int>();
taskVersionRep.Find(m => m.nodeid == x.id && m.isdel == 0).ToList().ForEach(m =>
{
taskidlist.Add(m.taskid);
});
nodeRep.UpdateNodeById(x.id, new Dictionary <string, string>()
{
{ "nodestatus", ((int)NodeStatus.NoRun).ToString() }, { "refreshcommandqueuestatus", ((int)RefreshCommandQueueStatus.NoRefresh).ToString() }
}); //, { "refreshcommandqueuestatus", ((int)RefreshCommandQueueStatus.NoRefresh).ToString() }
taskRep.UpdateById(taskidlist, new Dictionary <string, string>()
{
{ "taskschedulestatus", ((int)TaskScheduleStatus.StopSchedule).ToString() }
});
string title = "节点(编号):" + x.nodename + "(" + x.id.ToString() + "),心跳异常,已自动更新为未运行状态,请及时处理该节点下的任务!";
StringBuilder strContent = new StringBuilder();
strContent.AppendLine("节点(编号):" + x.nodename + "(" + x.id.ToString() + ")<br/>");
strContent.AppendLine("节点最后一次心跳时间:" + x.nodelastupdatetime + "<br/>");
//当前节点心跳不正常，已修改节点状态为未运行
AlarmHelper.AlarmAsync(x.isenablealarm, (AlarmType)x.alarmtype, alaramperson, title, strContent.ToString());
}
else
{
if (x.refreshcommandqueuestatus == (int)RefreshCommandQueueStatus.Refreshing)
{
double totalsecond2 = Math.Abs(x.lastrefreshcommandqueuetime.Subtract(DateTime.Now).TotalSeconds);
if (totalsecond2 > 20 || totalsecond2 < 0)//大于10s 说明心跳不正常
{
nodeRep.UpdateNodeById(x.id, new Dictionary <string, string>()
{
{ "refreshcommandqueuestatus", ((int)RefreshCommandQueueStatus.NoRefresh).ToString() }
}); //,
string title = "节点(编号):" + x.nodename + "(" + x.id.ToString() + "),监听队列异常,已自动更新为未监听状态,请及时处理该节点!";
StringBuilder strContent = new StringBuilder();
strContent.AppendLine("节点(编号):" + x.nodename + "(" + x.id.ToString() + ")<br/>");
strContent.AppendLine("节点最后一次监听队列时间:" + x.lastrefreshcommandqueuetime + "<br/>");
//当前节点心跳不正常，已修改节点状态为未运行
AlarmHelper.AlarmAsync(x.isenablealarm, (AlarmType)x.alarmtype, alaramperson, title, strContent.ToString());
}
}
}
#endregion
}
}
catch (Exception ex)
{
strLog.AppendLine("检测节点心跳异常:" + JsonConvert.SerializeObject(ex));
}
});
if (strLog.Length > 0)
{
log.Error(strLog.ToString());
}
}
}