Exemple #1
0
        /// <summary>
        /// 任务调度平台根据发布的任务时间配置,定时回调运行方法
        /// 开发人员的任务插件必须要重载并该方法
        /// </summary>
        public override void Run()
        {
            /*
             * this.OpenOperator 用于任务调度平台提供给第三方使用的所有api接口封装
             */
            ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]);
            var sql = @"";
            List<tb_partition_model> userdpartitions = new List<tb_partition_model>();
            Dictionary<int, tb_datanode_model> datanodemodels = new Dictionary<int, tb_datanode_model>();
            DateTime servertime = DateTime.Now;
            SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) =>
            {
                tb_config_dal configdal = new tb_config_dal();
                sql = configdal.Get(c, "MQCreateTableSql").value;
                tb_partition_dal partitiondal = new tb_partition_dal();
                userdpartitions = partitiondal.List(c, true);
                tb_datanode_dal datanodedal = new tb_datanode_dal();
                var ms = datanodedal.List(c);
                foreach (var m in ms)
                {
                    datanodemodels.Add(m.datanodepartition, m);
                }
                servertime = c.GetServerDate();
            });

            if (string.IsNullOrWhiteSpace(sql))
            {
                Error(this.AppConfig["BusinessMQManageConnectString"], "MQCreateTableSql未配置参数", new Exception("MQ未找到或者未配置MQCreateTableSql参数,用于每天创建消息分区表"));
                return;
            }

            foreach (var p in userdpartitions)
            {
                var partitioninfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(p.partitionid);
                DateTime currenttime = servertime.Date;
                while (currenttime <= servertime.Date.AddDays(3))//预创建3天的表
                {
                    var tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, currenttime);//
                    SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitioninfo.DataNodePartition]), (c) =>
                    {
                        bool exsit = c.TableIsExist(tablename);
                        if (exsit != true)
                        {
                            string cmd = sql.Replace("{tablepartiton}", XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.PartitionNameRule(partitioninfo.TablePartition ))
                                .Replace("{daypartition}", currenttime.ToString("yyMMdd")).Replace("{datanodepartiton}",XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.PartitionNameRule(partitioninfo.DataNodePartition));
                            c.ExecuteSql(cmd,new List<XXF.Db.ProcedureParameter>());
                        }
                    });
                    currenttime = currenttime.AddDays(1);
                }
            }
        }
Exemple #2
0
        /// <summary>
        /// 任务调度平台根据发布的任务时间配置,定时回调运行方法
        /// 开发人员的任务插件必须要重载并该方法
        /// </summary>
        public override void Run()
        {
            ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]);
            List<tb_partition_model> userdpartitions = new List<tb_partition_model>();
            Dictionary<int, tb_datanode_model> datanodemodels = new Dictionary<int, tb_datanode_model>();
            DateTime servertime = DateTime.Now;
            SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) =>
            {
                tb_partition_dal partitiondal = new tb_partition_dal();
                userdpartitions = partitiondal.List(c, true);
                tb_datanode_dal datanodedal = new tb_datanode_dal();
                var ms = datanodedal.List(c);
                foreach (var m in ms)
                {
                    datanodemodels.Add(m.datanodepartition, m);
                }
                servertime = c.GetServerDate();
            });

            List<Exception> exps = new List<Exception>();
            foreach (var p in userdpartitions)
            {
                var partitioninfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(p.partitionid);
                List<DateTime> days = new List<DateTime>() { servertime.Date, servertime.Date.AddDays(1), servertime.Date.AddDays(2) };
                //检查三天表创建情况
                foreach (var day in days)
                {
                    var tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, day);//
                    SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitioninfo.DataNodePartition]), (c) =>
                    {
                        bool exsit = c.TableIsExist(tablename);
                        if (exsit != true)
                        {
                            exps.Add(new Exception(string.Format("分区{0}表{1}不存在", p.partitionid, tablename)));
                        }
                    });
                }

                //检测当天的数据量是否过大
                var todaytablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, servertime.Date);//
                SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitioninfo.DataNodePartition]), (c) =>
                {
                    tb_messagequeue_dal dal = new tb_messagequeue_dal(); dal.TableName = todaytablename;
                    var maxid = dal.GetMaxID(c);
                    if (maxid % 100000000 >= 70000000)
                    {
                        exps.Add(new Exception(string.Format("数据量超过70000000,当前最大mqid:{0}", maxid)));
                    }
                });
            }
            Error(this.AppConfig["BusinessMQManageConnectString"], "数据异常任务之表创建检查", exps);
        }
Exemple #3
0
 public ActionResult Add(int nodeid,int count)
 {
     try
     {
         using (DbConn conn = DbConfig.CreateConn(DataConfig.MqManage))
         {
             conn.Open();
             try
             {
                 conn.BeginTransaction();
                 int r = 0;
                 var partitons = new tb_partition_dal().GetPageList(conn, "", nodeid, -1, 1, 100, ref r);
                 List<int> usedpartitionids = new List<int>();
                 foreach (var d in partitons)
                 {
                     if (!usedpartitionids.Contains(d.partitionid))
                         usedpartitionids.Add(d.partitionid);
                 }
                 List<int> canusepartitionids = new List<int>();
                 for (var i = 1; i < 100; i++)
                 {
                     var partition = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionID(new XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionIDInfo() { DataNodePartition = nodeid, TablePartition =i  });
                     if (!usedpartitionids.Contains(partition))
                         canusepartitionids.Add(partition);
                 }
                 for (int i = 0; i < count; i++)
                 {
                     tb_partition_model model = new tb_partition_model();
                     model.isused = false;
                     model.partitionid = canusepartitionids[i];
                     dal.AddPartition(conn, model);
                 }
                 conn.Commit();
             }
             catch (Exception exp)
             {
                 conn.Rollback();
                 throw exp;
             }
         }
         return RedirectToAction("index", new { nodeId = nodeid });
     }
     catch (Exception exp)
     {
         ModelState.AddModelError("Error","添加失败"+ exp.Message);
         return View();
     }
 }
Exemple #4
0
        /// <summary>
        /// 任务调度平台根据发布的任务时间配置,定时回调运行方法
        /// 开发人员的任务插件必须要重载并该方法
        /// </summary>
        public override void Run()
        {
            List<tb_partition_model> userdpartitions = new List<tb_partition_model>();
            Dictionary<int, tb_datanode_model> datanodemodels = new Dictionary<int, tb_datanode_model>();
            DateTime servertime = DateTime.Now;
            SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) =>
            {
                tb_partition_dal partitiondal = new tb_partition_dal();
                userdpartitions = partitiondal.List(c, true);
                tb_datanode_dal datanodedal = new tb_datanode_dal();
                var ms = datanodedal.List(c);
                foreach (var m in ms)
                {
                    datanodemodels.Add(m.datanodepartition, m);
                }
                servertime = c.GetServerDate();
            });

            foreach (var p in userdpartitions)
            {
                var partitioninfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(p.partitionid);
                DateTime fromdate = servertime.Date.AddDays(-3).Date;//自动迁移3天内的消息至今天,否则需要手工迁移
                while (fromdate < servertime.Date)
                {
                    var fromtimetablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, fromdate);
                    try
                    {
                        MoveDataToToday(fromdate, servertime.Date, p, datanodemodels);
                    }
                    catch (Exception exp)
                    {
                        this.OpenOperator.Error(string.Format("数据迁移出错,partitionid:{0},fromdate:{1},todate:{2}", p.partitionid,fromdate,servertime.Date), exp);
                    }
                    fromdate = fromdate.AddDays(1);
                }
            }
        }
Exemple #5
0
        public IList <ConsumerModel> GetPageList(DbConn conn, string name, int pageIndex, int pageSize, ref int count)
        {
            int tempCount = 0;
            IList <ConsumerModel> list = new List <ConsumerModel>();
            ConsumerModel         cm   = new ConsumerModel();
            var result = SqlHelper.Visit((ps) =>
            {
                StringBuilder where = new StringBuilder(" WHERE 1=1 ");
                if (!string.IsNullOrEmpty(name))
                {
                    where.AppendFormat(" AND  clientname LIKE '%{0}%'", name);
                }
                string sql      = "SELECT ROW_NUMBER() OVER(ORDER BY Id DESC) AS rownum,* FROM tb_consumer WITH(NOLOCK)";
                string countSql = "SELECT COUNT(1) FROM tb_consumer WITH(NOLOCK) " + where.ToString();
                object obj      = conn.ExecuteScalar(countSql, null);
                if (obj != DBNull.Value && obj != null)
                {
                    tempCount = LibConvert.ObjToInt(obj);
                }
                string sqlPage = string.Concat("SELECT * FROM (", sql.ToString(), where.ToString(), ") A WHERE rownum BETWEEN ", ((pageIndex - 1) * pageSize + 1), " AND ", pageSize * pageIndex);
                DataTable dt   = conn.SqlToDataTable(sqlPage, null);
                if (dt != null && dt.Rows.Count > 0)
                {
                    foreach (DataRow dr in dt.Rows)
                    {
                        ConsumerModel model = cm.CreateModel(dr);

                        IList <tb_consumer_partition_model> consumerList = partitionDal.GetPartitionByConsumerId(conn, model.consumerclientid);
                        if (consumerList != null && consumerList.Count > 0)
                        {
                            IList <ConsumerPartition> partitionList = new List <ConsumerPartition>();

                            foreach (var item in consumerList)
                            {
                                ConsumerPartition m = new ConsumerPartition();
                                m.PartitionId       = item.partitionid;

                                PartitionIDInfo partitionInfo = PartitionRuleHelper.GetPartitionIDInfo(item.partitionid);
                                string node = string.Empty;
                                if (partitionInfo.DataNodePartition < 10)
                                {
                                    node = "0" + partitionInfo.DataNodePartition.Tostring();
                                }
                                else
                                {
                                    node = partitionInfo.DataNodePartition.Tostring();
                                }

                                using (DbConn nodeConn = DbConfig.CreateConn(DataConfig.DataNodeParConn(node)))
                                {
                                    nodeConn.Open();
                                    tb_partition_model partitionModel = new tb_partition_dal().Get(conn, item.partitionid);
                                    if (partitionModel != null)
                                    {
                                        m.IsOnline = partitionModel.isused;
                                    }
                                    string table = msgDal.GetMaxMqTable(nodeConn, node);
                                    m.Msg        = msgDal.GetMsgCount(nodeConn, table, 0);
                                    m.NonMsg     = msgDal.GetMsgCount(nodeConn, table, 1);
                                    partitionList.Add(m);
                                }
                            }
                            model.PartitionList = partitionList;
                        }
                        list.Add(model);
                    }
                }
                return(list);
            });

            count = tempCount;
            return(result);
        }
Exemple #6
0
        /// <summary>
        /// 任务调度平台根据发布的任务时间配置,定时回调运行方法
        /// 开发人员的任务插件必须要重载并该方法
        /// </summary>
        public override void Run()
        {
            runcount++;

            ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]);

            List<tb_partition_model> userdpartitions = new List<tb_partition_model>();
            Dictionary<int, tb_datanode_model> datanodemodels = new Dictionary<int, tb_datanode_model>();
            List<RegisterdConsumersModel> registeredconsumermodels = new List<RegisterdConsumersModel>();
            List<tb_consumer_model> consumers = new List<tb_consumer_model>();
            DateTime servertime = DateTime.Now;
            SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) =>
            {
                tb_partition_dal partitiondal = new tb_partition_dal();
                userdpartitions = partitiondal.List(c, true);
                tb_datanode_dal datanodedal = new tb_datanode_dal();
                var ms = datanodedal.List(c);
                foreach (var m in ms)
                {
                    datanodemodels.Add(m.datanodepartition, m);
                }
                servertime = c.GetServerDate();

                //获取当前注册的消费者
                tb_consumer_dal consumerdal = new tb_consumer_dal();
                registeredconsumermodels = consumerdal.GetRegisterdConsumers(c);
                consumers = consumerdal.GetAllList(c);
            });
            //消费者端心跳停止,异常退出检测(超过1分钟)
            List<Exception> exps = new List<Exception>();
            foreach (var c in consumers)
            {
                if ((servertime - c.lastheartbeat) > TimeSpan.FromMinutes(1))
                {
                    exps.Add(new Exception(string.Format("当前消费者tempid:{0},consumerclientid:{1}", c.tempid, c.consumerclientid)));
                }
            }
            Error(this.AppConfig["BusinessMQManageConnectString"], "消费者端心跳停止,异常退出检测(超过1分钟),若是异常停止,请手工清除消费者中断", exps);
            //消费者端长时间无消费,但是还有多余消息(超过10分钟未消费,但还有多余消息未消费)
            exps.Clear();
            foreach (var c in registeredconsumermodels)
            {
                if ((servertime - c.consumerpartitionmodel.lastupdatetime) > TimeSpan.FromMinutes(10))
                {
                    var partitionidinfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(c.consumerpartitionmodel.partitionid);
                    var tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitionidinfo.TablePartition, servertime.Date);
                    SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitionidinfo.DataNodePartition]), (conn) =>
                    {
                        tb_messagequeue_dal messagedal = new tb_messagequeue_dal(); messagedal.TableName = tablename;
                        long maxmqid = messagedal.GetMaxID(conn);
                        if (maxmqid > c.consumerpartitionmodel.lastmqid && (servertime - messagedal.GetIDCreateTime(conn, maxmqid).Value > TimeSpan.FromMinutes(10)))//还有消息未消费,且该消息的创建时间是10分钟前
                        {
                            exps.Add(new Exception(string.Format("当前消费者tempid:{0},consumerclientid:{1},client:{2},clientname:{3}", c.consumermodel.tempid,
                                                   c.consumermodel.consumerclientid, c.consumerclientmodel.client, c.consumermodel.clientname)));
                        }
                    });
                }
            }
            Error(this.AppConfig["BusinessMQManageConnectString"], "消费者端长时间无消费,但是还有多余消息(超过已创建超过10分钟的消息未消费),可能是消费者处理逻辑异常", exps);
            //某个分区没有指定消费者(分区没有消费信息,或者分区消费最后消费时间超过1个小时无消费)
            exps.Clear();
            if (runcount % (12*6) == 0)
            {
                foreach (var u in userdpartitions)
                {
                    bool isfind = false;
                    foreach (var c in registeredconsumermodels)
                    {
                        if (c.consumerpartitionmodel.partitionid == u.partitionid)
                        {
                            isfind = true; break;
                        }
                    }
                    if (isfind == false)
                    {
                        SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) =>
                        {
                            tb_consumer_partition_dal dal = new tb_consumer_partition_dal();
                            var model = dal.GetByPartitionId(c, u.partitionid);
                            if (model == null)
                                exps.Add(new Exception(string.Format("分区{0}未指定消费者", u.partitionid)));
                            //else if((servertime - model.lastupdatetime) > TimeSpan.FromHours(24))
                            //     exps.Add(new Exception(string.Format("分区{0}消费者已注销,且有1天未消费", u.partitionid)));

                        });

                    }
                }
                Error(this.AppConfig["BusinessMQManageConnectString"], "某个分区没有指定消费者(分区没有消费信息)", exps);
            }
        }
Exemple #7
0
        public IList<ConsumerModel> GetPageList(DbConn conn, string name, int pageIndex, int pageSize, ref int count)
        {
            int tempCount = 0;
            IList<ConsumerModel> list = new List<ConsumerModel>();
            ConsumerModel cm = new ConsumerModel();
            var result = SqlHelper.Visit((ps) =>
            {
                StringBuilder where = new StringBuilder(" WHERE 1=1 ");
                if (!string.IsNullOrEmpty(name))
                {
                    where.AppendFormat(" AND  clientname LIKE '%{0}%'", name);
                }
                string sql = "SELECT ROW_NUMBER() OVER(ORDER BY Id DESC) AS rownum,* FROM tb_consumer WITH(NOLOCK)";
                string countSql = "SELECT COUNT(1) FROM tb_consumer WITH(NOLOCK) " + where.ToString();
                object obj = conn.ExecuteScalar(countSql, null);
                if (obj != DBNull.Value && obj != null)
                {
                    tempCount = LibConvert.ObjToInt(obj);
                }
                string sqlPage = string.Concat("SELECT * FROM (", sql.ToString(), where.ToString(), ") A WHERE rownum BETWEEN ", ((pageIndex - 1) * pageSize + 1), " AND ", pageSize * pageIndex);
                DataTable dt = conn.SqlToDataTable(sqlPage, null);
                if (dt != null && dt.Rows.Count > 0)
                {
                    foreach (DataRow dr in dt.Rows)
                    {
                        ConsumerModel model = cm.CreateModel(dr);

                        IList<tb_consumer_partition_model> consumerList = partitionDal.GetPartitionByConsumerId(conn, model.consumerclientid);
                        if (consumerList != null && consumerList.Count > 0)
                        {
                            IList<ConsumerPartition> partitionList = new List<ConsumerPartition>();

                            foreach (var item in consumerList)
                            {
                                ConsumerPartition m = new ConsumerPartition();
                                m.PartitionId = item.partitionid;

                                PartitionIDInfo partitionInfo = PartitionRuleHelper.GetPartitionIDInfo(item.partitionid);
                                string node = string.Empty;
                                if (partitionInfo.DataNodePartition < 10)
                                {
                                    node = "0" + partitionInfo.DataNodePartition.Tostring();
                                }
                                else
                                {
                                    node = partitionInfo.DataNodePartition.Tostring();
                                }

                                using (DbConn nodeConn = DbConfig.CreateConn(DataConfig.DataNodeParConn(node)))
                                {
                                    nodeConn.Open();
                                    tb_partition_model partitionModel = new tb_partition_dal().Get(conn, item.partitionid);
                                    if (partitionModel != null)
                                    {
                                        m.IsOnline = partitionModel.isused;
                                    }
                                    string table = msgDal.GetMaxMqTable(nodeConn, node);
                                    m.Msg = msgDal.GetMsgCount(nodeConn, table, 0);
                                    m.NonMsg = msgDal.GetMsgCount(nodeConn, table, 1);
                                    partitionList.Add(m);
                                }
                            }
                            model.PartitionList = partitionList;
                        }
                        list.Add(model);
                    }
                }
                return list;
            });
            count = tempCount;
            return result;
        }