/// <summary> /// 任务调度平台根据发布的任务时间配置,定时回调运行方法 /// 开发人员的任务插件必须要重载并该方法 /// </summary> public override void Run() { ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]); List<tb_partition_model> userdpartitions = new List<tb_partition_model>(); Dictionary<int, tb_datanode_model> datanodemodels = new Dictionary<int, tb_datanode_model>(); DateTime servertime = DateTime.Now; SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) => { tb_partition_dal partitiondal = new tb_partition_dal(); userdpartitions = partitiondal.List(c, true); tb_datanode_dal datanodedal = new tb_datanode_dal(); var ms = datanodedal.List(c); foreach (var m in ms) { datanodemodels.Add(m.datanodepartition, m); } servertime = c.GetServerDate(); }); List<Exception> exps = new List<Exception>(); foreach (var p in userdpartitions) { var partitioninfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(p.partitionid); List<DateTime> days = new List<DateTime>() { servertime.Date, servertime.Date.AddDays(1), servertime.Date.AddDays(2) }; //检查三天表创建情况 foreach (var day in days) { var tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, day);// SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitioninfo.DataNodePartition]), (c) => { bool exsit = c.TableIsExist(tablename); if (exsit != true) { exps.Add(new Exception(string.Format("分区{0}表{1}不存在", p.partitionid, tablename))); } }); } //检测当天的数据量是否过大 var todaytablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, servertime.Date);// SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitioninfo.DataNodePartition]), (c) => { tb_messagequeue_dal dal = new tb_messagequeue_dal(); dal.TableName = todaytablename; var maxid = dal.GetMaxID(c); if (maxid % 100000000 >= 70000000) { exps.Add(new Exception(string.Format("数据量超过70000000,当前最大mqid:{0}", maxid))); } }); } Error(this.AppConfig["BusinessMQManageConnectString"], "数据异常任务之表创建检查", exps); }
/// <summary> /// 任务调度平台根据发布的任务时间配置,定时回调运行方法 /// 开发人员的任务插件必须要重载并该方法 /// </summary> public override void Run() { /* * this.OpenOperator 用于任务调度平台提供给第三方使用的所有api接口封装 */ ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]); var sql = @""; List<tb_partition_model> userdpartitions = new List<tb_partition_model>(); Dictionary<int, tb_datanode_model> datanodemodels = new Dictionary<int, tb_datanode_model>(); DateTime servertime = DateTime.Now; SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) => { tb_config_dal configdal = new tb_config_dal(); sql = configdal.Get(c, "MQCreateTableSql").value; tb_partition_dal partitiondal = new tb_partition_dal(); userdpartitions = partitiondal.List(c, true); tb_datanode_dal datanodedal = new tb_datanode_dal(); var ms = datanodedal.List(c); foreach (var m in ms) { datanodemodels.Add(m.datanodepartition, m); } servertime = c.GetServerDate(); }); if (string.IsNullOrWhiteSpace(sql)) { Error(this.AppConfig["BusinessMQManageConnectString"], "MQCreateTableSql未配置参数", new Exception("MQ未找到或者未配置MQCreateTableSql参数,用于每天创建消息分区表")); return; } foreach (var p in userdpartitions) { var partitioninfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(p.partitionid); DateTime currenttime = servertime.Date; while (currenttime <= servertime.Date.AddDays(3))//预创建3天的表 { var tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, currenttime);// SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitioninfo.DataNodePartition]), (c) => { bool exsit = c.TableIsExist(tablename); if (exsit != true) { string cmd = sql.Replace("{tablepartiton}", XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.PartitionNameRule(partitioninfo.TablePartition )) .Replace("{daypartition}", currenttime.ToString("yyMMdd")).Replace("{datanodepartiton}",XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.PartitionNameRule(partitioninfo.DataNodePartition)); c.ExecuteSql(cmd,new List<XXF.Db.ProcedureParameter>()); } }); currenttime = currenttime.AddDays(1); } } }
/// <summary> /// 任务调度平台根据发布的任务时间配置,定时回调运行方法 /// 开发人员的任务插件必须要重载并该方法 /// </summary> public override void Run() { List<tb_partition_model> userdpartitions = new List<tb_partition_model>(); Dictionary<int, tb_datanode_model> datanodemodels = new Dictionary<int, tb_datanode_model>(); DateTime servertime = DateTime.Now; SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) => { tb_partition_dal partitiondal = new tb_partition_dal(); userdpartitions = partitiondal.List(c, true); tb_datanode_dal datanodedal = new tb_datanode_dal(); var ms = datanodedal.List(c); foreach (var m in ms) { datanodemodels.Add(m.datanodepartition, m); } servertime = c.GetServerDate(); }); foreach (var p in userdpartitions) { var partitioninfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(p.partitionid); DateTime fromdate = servertime.Date.AddDays(-3).Date;//自动迁移3天内的消息至今天,否则需要手工迁移 while (fromdate < servertime.Date) { var fromtimetablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, fromdate); try { MoveDataToToday(fromdate, servertime.Date, p, datanodemodels); } catch (Exception exp) { this.OpenOperator.Error(string.Format("数据迁移出错,partitionid:{0},fromdate:{1},todate:{2}", p.partitionid,fromdate,servertime.Date), exp); } fromdate = fromdate.AddDays(1); } } }
/// <summary> /// 任务调度平台根据发布的任务时间配置,定时回调运行方法 /// 开发人员的任务插件必须要重载并该方法 /// </summary> public override void Run() { runcount++; ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]); List<tb_partition_model> userdpartitions = new List<tb_partition_model>(); Dictionary<int, tb_datanode_model> datanodemodels = new Dictionary<int, tb_datanode_model>(); List<RegisterdConsumersModel> registeredconsumermodels = new List<RegisterdConsumersModel>(); List<tb_consumer_model> consumers = new List<tb_consumer_model>(); DateTime servertime = DateTime.Now; SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) => { tb_partition_dal partitiondal = new tb_partition_dal(); userdpartitions = partitiondal.List(c, true); tb_datanode_dal datanodedal = new tb_datanode_dal(); var ms = datanodedal.List(c); foreach (var m in ms) { datanodemodels.Add(m.datanodepartition, m); } servertime = c.GetServerDate(); //获取当前注册的消费者 tb_consumer_dal consumerdal = new tb_consumer_dal(); registeredconsumermodels = consumerdal.GetRegisterdConsumers(c); consumers = consumerdal.GetAllList(c); }); //消费者端心跳停止,异常退出检测(超过1分钟) List<Exception> exps = new List<Exception>(); foreach (var c in consumers) { if ((servertime - c.lastheartbeat) > TimeSpan.FromMinutes(1)) { exps.Add(new Exception(string.Format("当前消费者tempid:{0},consumerclientid:{1}", c.tempid, c.consumerclientid))); } } Error(this.AppConfig["BusinessMQManageConnectString"], "消费者端心跳停止,异常退出检测(超过1分钟),若是异常停止,请手工清除消费者中断", exps); //消费者端长时间无消费,但是还有多余消息(超过10分钟未消费,但还有多余消息未消费) exps.Clear(); foreach (var c in registeredconsumermodels) { if ((servertime - c.consumerpartitionmodel.lastupdatetime) > TimeSpan.FromMinutes(10)) { var partitionidinfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(c.consumerpartitionmodel.partitionid); var tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitionidinfo.TablePartition, servertime.Date); SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitionidinfo.DataNodePartition]), (conn) => { tb_messagequeue_dal messagedal = new tb_messagequeue_dal(); messagedal.TableName = tablename; long maxmqid = messagedal.GetMaxID(conn); if (maxmqid > c.consumerpartitionmodel.lastmqid && (servertime - messagedal.GetIDCreateTime(conn, maxmqid).Value > TimeSpan.FromMinutes(10)))//还有消息未消费,且该消息的创建时间是10分钟前 { exps.Add(new Exception(string.Format("当前消费者tempid:{0},consumerclientid:{1},client:{2},clientname:{3}", c.consumermodel.tempid, c.consumermodel.consumerclientid, c.consumerclientmodel.client, c.consumermodel.clientname))); } }); } } Error(this.AppConfig["BusinessMQManageConnectString"], "消费者端长时间无消费,但是还有多余消息(超过已创建超过10分钟的消息未消费),可能是消费者处理逻辑异常", exps); //某个分区没有指定消费者(分区没有消费信息,或者分区消费最后消费时间超过1个小时无消费) exps.Clear(); if (runcount % (12*6) == 0) { foreach (var u in userdpartitions) { bool isfind = false; foreach (var c in registeredconsumermodels) { if (c.consumerpartitionmodel.partitionid == u.partitionid) { isfind = true; break; } } if (isfind == false) { SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) => { tb_consumer_partition_dal dal = new tb_consumer_partition_dal(); var model = dal.GetByPartitionId(c, u.partitionid); if (model == null) exps.Add(new Exception(string.Format("分区{0}未指定消费者", u.partitionid))); //else if((servertime - model.lastupdatetime) > TimeSpan.FromHours(24)) // exps.Add(new Exception(string.Format("分区{0}消费者已注销,且有1天未消费", u.partitionid))); }); } } Error(this.AppConfig["BusinessMQManageConnectString"], "某个分区没有指定消费者(分区没有消费信息)", exps); } }