/// <summary> /// 任务调度平台根据发布的任务时间配置,定时回调运行方法 /// 开发人员的任务插件必须要重载并该方法 /// </summary> public override void Run() { ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]); List <tb_partition_model> userdpartitions = new List <tb_partition_model>(); Dictionary <int, tb_datanode_model> datanodemodels = new Dictionary <int, tb_datanode_model>(); DateTime servertime = DateTime.Now; SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) => { tb_partition_dal partitiondal = new tb_partition_dal(); userdpartitions = partitiondal.List(c, true); tb_datanode_dal datanodedal = new tb_datanode_dal(); var ms = datanodedal.List(c); foreach (var m in ms) { datanodemodels.Add(m.datanodepartition, m); } servertime = c.GetServerDate(); }); List <Exception> exps = new List <Exception>(); foreach (var p in userdpartitions) { var partitioninfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(p.partitionid); List <DateTime> days = new List <DateTime>() { servertime.Date, servertime.Date.AddDays(1), servertime.Date.AddDays(2) }; //检查三天表创建情况 foreach (var day in days) { var tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, day);// SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitioninfo.DataNodePartition]), (c) => { bool exsit = c.TableIsExist(tablename); if (exsit != true) { exps.Add(new Exception(string.Format("分区{0}表{1}不存在", p.partitionid, tablename))); } }); } //检测当天的数据量是否过大 var todaytablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, servertime.Date);// SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitioninfo.DataNodePartition]), (c) => { tb_messagequeue_dal dal = new tb_messagequeue_dal(); dal.TableName = todaytablename; var maxid = dal.GetMaxID(c); if (maxid % 100000000 >= 70000000) { exps.Add(new Exception(string.Format("数据量超过70000000,当前最大mqid:{0}", maxid))); } }); } Error(this.AppConfig["BusinessMQManageConnectString"], "数据异常任务之表创建检查", exps); }
public ActionResult Add(int nodeid, int count) { try { using (DbConn conn = DbConfig.CreateConn(DataConfig.MqManage)) { conn.Open(); try { conn.BeginTransaction(); int r = 0; var partitons = new tb_partition_dal().GetPageList(conn, "", nodeid, -1, 1, 100, ref r); List <int> usedpartitionids = new List <int>(); foreach (var d in partitons) { if (!usedpartitionids.Contains(d.partitionid)) { usedpartitionids.Add(d.partitionid); } } List <int> canusepartitionids = new List <int>(); for (var i = 1; i < 100; i++) { var partition = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionID(new XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionIDInfo() { DataNodePartition = nodeid, TablePartition = i }); if (!usedpartitionids.Contains(partition)) { canusepartitionids.Add(partition); } } for (int i = 0; i < count; i++) { tb_partition_model model = new tb_partition_model(); model.isused = false; model.partitionid = canusepartitionids[i]; dal.AddPartition(conn, model); } conn.Commit(); } catch (Exception exp) { conn.Rollback(); throw exp; } } return(RedirectToAction("index", new { nodeId = nodeid })); } catch (Exception exp) { ModelState.AddModelError("Error", "添加失败" + exp.Message); return(View()); } }
/// <summary> /// 任务调度平台根据发布的任务时间配置,定时回调运行方法 /// 开发人员的任务插件必须要重载并该方法 /// </summary> public override void Run() { /* * this.OpenOperator 用于任务调度平台提供给第三方使用的所有api接口封装 */ ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]); var sql = @""; List <tb_partition_model> userdpartitions = new List <tb_partition_model>(); Dictionary <int, tb_datanode_model> datanodemodels = new Dictionary <int, tb_datanode_model>(); DateTime servertime = DateTime.Now; SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) => { tb_config_dal configdal = new tb_config_dal(); sql = configdal.Get(c, "MQCreateTableSql").value; tb_partition_dal partitiondal = new tb_partition_dal(); userdpartitions = partitiondal.List(c, true); tb_datanode_dal datanodedal = new tb_datanode_dal(); var ms = datanodedal.List(c); foreach (var m in ms) { datanodemodels.Add(m.datanodepartition, m); } servertime = c.GetServerDate(); }); if (string.IsNullOrWhiteSpace(sql)) { Error(this.AppConfig["BusinessMQManageConnectString"], "MQCreateTableSql未配置参数", new Exception("MQ未找到或者未配置MQCreateTableSql参数,用于每天创建消息分区表")); return; } foreach (var p in userdpartitions) { var partitioninfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(p.partitionid); DateTime currenttime = servertime.Date; while (currenttime <= servertime.Date.AddDays(3)) //预创建3天的表 { var tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, currenttime); // SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitioninfo.DataNodePartition]), (c) => { bool exsit = c.TableIsExist(tablename); if (exsit != true) { string cmd = sql.Replace("{tablepartiton}", XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.PartitionNameRule(partitioninfo.TablePartition)) .Replace("{daypartition}", currenttime.ToString("yyMMdd")).Replace("{datanodepartiton}", XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.PartitionNameRule(partitioninfo.DataNodePartition)); c.ExecuteSql(cmd, new List <XXF.Db.ProcedureParameter>()); } }); currenttime = currenttime.AddDays(1); } } }
/// <summary> /// 任务调度平台根据发布的任务时间配置,定时回调运行方法 /// 开发人员的任务插件必须要重载并该方法 /// </summary> public override void Run() { List <tb_partition_model> userdpartitions = new List <tb_partition_model>(); Dictionary <int, tb_datanode_model> datanodemodels = new Dictionary <int, tb_datanode_model>(); DateTime servertime = DateTime.Now; SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) => { tb_partition_dal partitiondal = new tb_partition_dal(); userdpartitions = partitiondal.List(c, true); tb_datanode_dal datanodedal = new tb_datanode_dal(); var ms = datanodedal.List(c); foreach (var m in ms) { datanodemodels.Add(m.datanodepartition, m); } servertime = c.GetServerDate(); }); foreach (var p in userdpartitions) { var partitioninfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(p.partitionid); DateTime fromdate = servertime.Date.AddDays(-3).Date;//自动迁移3天内的消息至今天,否则需要手工迁移 while (fromdate < servertime.Date) { var fromtimetablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, fromdate); try { MoveDataToToday(fromdate, servertime.Date, p, datanodemodels); } catch (Exception exp) { this.OpenOperator.Error(string.Format("数据迁移出错,partitionid:{0},fromdate:{1},todate:{2}", p.partitionid, fromdate, servertime.Date), exp); } fromdate = fromdate.AddDays(1); } } }
/// <summary> /// 任务调度平台根据发布的任务时间配置,定时回调运行方法 /// 开发人员的任务插件必须要重载并该方法 /// </summary> public override void Run() { runcount++; ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]); List <tb_partition_model> userdpartitions = new List <tb_partition_model>(); Dictionary <int, tb_datanode_model> datanodemodels = new Dictionary <int, tb_datanode_model>(); List <RegisterdConsumersModel> registeredconsumermodels = new List <RegisterdConsumersModel>(); List <tb_consumer_model> consumers = new List <tb_consumer_model>(); DateTime servertime = DateTime.Now; SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) => { tb_partition_dal partitiondal = new tb_partition_dal(); userdpartitions = partitiondal.List(c, true); tb_datanode_dal datanodedal = new tb_datanode_dal(); var ms = datanodedal.List(c); foreach (var m in ms) { datanodemodels.Add(m.datanodepartition, m); } servertime = c.GetServerDate(); //获取当前注册的消费者 tb_consumer_dal consumerdal = new tb_consumer_dal(); registeredconsumermodels = consumerdal.GetRegisterdConsumers(c); consumers = consumerdal.GetAllList(c); }); //消费者端心跳停止,异常退出检测(超过1分钟) List <Exception> exps = new List <Exception>(); foreach (var c in consumers) { if ((servertime - c.lastheartbeat) > TimeSpan.FromMinutes(1)) { exps.Add(new Exception(string.Format("当前消费者tempid:{0},consumerclientid:{1}", c.tempid, c.consumerclientid))); } } Error(this.AppConfig["BusinessMQManageConnectString"], "消费者端心跳停止,异常退出检测(超过1分钟),若是异常停止,请手工清除消费者中断", exps); //消费者端长时间无消费,但是还有多余消息(超过10分钟未消费,但还有多余消息未消费) exps.Clear(); foreach (var c in registeredconsumermodels) { if ((servertime - c.consumerpartitionmodel.lastupdatetime) > TimeSpan.FromMinutes(10)) { var partitionidinfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(c.consumerpartitionmodel.partitionid); var tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitionidinfo.TablePartition, servertime.Date); SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitionidinfo.DataNodePartition]), (conn) => { tb_messagequeue_dal messagedal = new tb_messagequeue_dal(); messagedal.TableName = tablename; long maxmqid = messagedal.GetMaxID(conn); if (maxmqid > c.consumerpartitionmodel.lastmqid && (servertime - messagedal.GetIDCreateTime(conn, maxmqid).Value > TimeSpan.FromMinutes(10)))//还有消息未消费,且该消息的创建时间是10分钟前 { exps.Add(new Exception(string.Format("当前消费者tempid:{0},consumerclientid:{1},client:{2},clientname:{3}", c.consumermodel.tempid, c.consumermodel.consumerclientid, c.consumerclientmodel.client, c.consumermodel.clientname))); } }); } } Error(this.AppConfig["BusinessMQManageConnectString"], "消费者端长时间无消费,但是还有多余消息(超过已创建超过10分钟的消息未消费),可能是消费者处理逻辑异常", exps); //某个分区没有指定消费者(分区没有消费信息,或者分区消费最后消费时间超过1个小时无消费) exps.Clear(); if (runcount % (12 * 6) == 0) { foreach (var u in userdpartitions) { bool isfind = false; foreach (var c in registeredconsumermodels) { if (c.consumerpartitionmodel.partitionid == u.partitionid) { isfind = true; break; } } if (isfind == false) { SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) => { tb_consumer_partition_dal dal = new tb_consumer_partition_dal(); var model = dal.GetByPartitionId(c, u.partitionid); if (model == null) { exps.Add(new Exception(string.Format("分区{0}未指定消费者", u.partitionid))); } //else if((servertime - model.lastupdatetime) > TimeSpan.FromHours(24)) // exps.Add(new Exception(string.Format("分区{0}消费者已注销,且有1天未消费", u.partitionid))); }); } } Error(this.AppConfig["BusinessMQManageConnectString"], "某个分区没有指定消费者(分区没有消费信息)", exps); } }