/// <summary>
/// 任务调度平台根据发布的任务时间配置，定时回调运行方法
/// 开发人员的任务插件必须要重载并该方法
/// </summary>
public override void Run()
{
ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]);
List<tb_partition_model> userdpartitions = new List<tb_partition_model>();
Dictionary<int, tb_datanode_model> datanodemodels = new Dictionary<int, tb_datanode_model>();
DateTime servertime = DateTime.Now;
SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) =>
{
tb_partition_dal partitiondal = new tb_partition_dal();
userdpartitions = partitiondal.List(c, true);
tb_datanode_dal datanodedal = new tb_datanode_dal();
var ms = datanodedal.List(c);
foreach (var m in ms)
{
datanodemodels.Add(m.datanodepartition, m);
}
servertime = c.GetServerDate();
});
List<Exception> exps = new List<Exception>();
foreach (var p in userdpartitions)
{
var partitioninfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(p.partitionid);
List<DateTime> days = new List<DateTime>() { servertime.Date, servertime.Date.AddDays(1), servertime.Date.AddDays(2) };
//检查三天表创建情况
foreach (var day in days)
{
var tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, day);//
SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitioninfo.DataNodePartition]), (c) =>
{
bool exsit = c.TableIsExist(tablename);
if (exsit != true)
{
exps.Add(new Exception(string.Format("分区{0}表{1}不存在", p.partitionid, tablename)));
}
});
}
//检测当天的数据量是否过大
var todaytablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, servertime.Date);//
SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitioninfo.DataNodePartition]), (c) =>
{
tb_messagequeue_dal dal = new tb_messagequeue_dal(); dal.TableName = todaytablename;
var maxid = dal.GetMaxID(c);
if (maxid % 100000000 >= 70000000)
{
exps.Add(new Exception(string.Format("数据量超过70000000,当前最大mqid:{0}", maxid)));
}
});
}
Error(this.AppConfig["BusinessMQManageConnectString"], "数据异常任务之表创建检查", exps);
}
/// <summary>
/// 任务调度平台根据发布的任务时间配置，定时回调运行方法
/// 开发人员的任务插件必须要重载并该方法
/// </summary>
public override void Run()
{
/*
* this.OpenOperator 用于任务调度平台提供给第三方使用的所有api接口封装
*/
ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]);
var sql = @"";
List<tb_partition_model> userdpartitions = new List<tb_partition_model>();
Dictionary<int, tb_datanode_model> datanodemodels = new Dictionary<int, tb_datanode_model>();
DateTime servertime = DateTime.Now;
SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) =>
{
tb_config_dal configdal = new tb_config_dal();
sql = configdal.Get(c, "MQCreateTableSql").value;
tb_partition_dal partitiondal = new tb_partition_dal();
userdpartitions = partitiondal.List(c, true);
tb_datanode_dal datanodedal = new tb_datanode_dal();
var ms = datanodedal.List(c);
foreach (var m in ms)
{
datanodemodels.Add(m.datanodepartition, m);
}
servertime = c.GetServerDate();
});
if (string.IsNullOrWhiteSpace(sql))
{
Error(this.AppConfig["BusinessMQManageConnectString"], "MQCreateTableSql未配置参数", new Exception("MQ未找到或者未配置MQCreateTableSql参数,用于每天创建消息分区表"));
return;
}
foreach (var p in userdpartitions)
{
var partitioninfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(p.partitionid);
DateTime currenttime = servertime.Date;
while (currenttime <= servertime.Date.AddDays(3))//预创建3天的表
{
var tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, currenttime);//
SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitioninfo.DataNodePartition]), (c) =>
{
bool exsit = c.TableIsExist(tablename);
if (exsit != true)
{
string cmd = sql.Replace("{tablepartiton}", XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.PartitionNameRule(partitioninfo.TablePartition ))
.Replace("{daypartition}", currenttime.ToString("yyMMdd")).Replace("{datanodepartiton}",XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.PartitionNameRule(partitioninfo.DataNodePartition));
c.ExecuteSql(cmd,new List<XXF.Db.ProcedureParameter>());
}
});
currenttime = currenttime.AddDays(1);
}
}
}
public ActionResult Add()
{
IList<string> usednodes = new List<string>();
using (DbConn conn = DbConfig.CreateConn(DataConfig.MqManage))
{
conn.Open();
usednodes = new tb_datanode_dal().GetNodeList(conn);
}
List<string> unUsednodes = new List<string>();
for (var i = 1; i < 100; i++)
{
var nodeid = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.PartitionNameRule(i);
//var partition = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionID(new XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionIDInfo() { DataNodePartition = nodeid, TablePartition = i });
if (!usednodes.Contains(nodeid))
unUsednodes.Add(nodeid);
}
ViewBag.unusednodes = unUsednodes;
return View();
}
public ActionResult Delete(int id)
{
using (DbConn conn = DbConfig.CreateConn(DataConfig.MqManage))
{
conn.Open();
var model = new tb_datanode_dal().Get(conn,id);
bool isExistPartion = partitionDal.IsExistPartition(conn, model.datanodepartition);
if (isExistPartion)
{
return Json(new { code = -1, msg = "该节点存在分区，请先删除分区，再进行删除节点操作" });
}
else
{
bool flag = nodeDal.Delete(conn, id);
if (flag)
{
return Json(new { code = 1, msg = "删除成功" });
}
return Json(new { code = -1, msg = "删除失败" });
}
}
}
/// <summary>
/// 任务调度平台根据发布的任务时间配置，定时回调运行方法
/// 开发人员的任务插件必须要重载并该方法
/// </summary>
public override void Run()
{
List<tb_partition_model> userdpartitions = new List<tb_partition_model>();
Dictionary<int, tb_datanode_model> datanodemodels = new Dictionary<int, tb_datanode_model>();
DateTime servertime = DateTime.Now;
SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) =>
{
tb_partition_dal partitiondal = new tb_partition_dal();
userdpartitions = partitiondal.List(c, true);
tb_datanode_dal datanodedal = new tb_datanode_dal();
var ms = datanodedal.List(c);
foreach (var m in ms)
{
datanodemodels.Add(m.datanodepartition, m);
}
servertime = c.GetServerDate();
});
foreach (var p in userdpartitions)
{
var partitioninfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(p.partitionid);
DateTime fromdate = servertime.Date.AddDays(-3).Date;//自动迁移3天内的消息至今天，否则需要手工迁移
while (fromdate < servertime.Date)
{
var fromtimetablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitioninfo.TablePartition, fromdate);
try
{
MoveDataToToday(fromdate, servertime.Date, p, datanodemodels);
}
catch (Exception exp)
{
this.OpenOperator.Error(string.Format("数据迁移出错,partitionid:{0},fromdate:{1},todate:{2}", p.partitionid,fromdate,servertime.Date), exp);
}
fromdate = fromdate.AddDays(1);
}
}
}
public override void Run()
{
ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]);
List<tb_datanode_model> nodeList = new List<tb_datanode_model>();
using (DbConn conn = DbConfig.CreateConn(this.AppConfig["BusinessMQManageConnectString"]))
{
conn.Open();
List<tb_partition_model> createTimeList = new tb_consumer_dal().GetClientCreateTime(conn);
var serverdate = conn.GetServerDate().Date; var timenow = conn.GetServerDate();
nodeList = new tb_datanode_dal().List(conn);
List<Exception> exps = new List<Exception>();
foreach (var item in nodeList)
{
try
{
string dataNode = PartitionRuleHelper.GetDataNodeName(item.datanodepartition);
string nodeConn = string.Format("server={0};Initial Catalog={1};User ID={2};Password={3};", item.serverip, dataNode, item.username, item.password);
string partitionId = PartitionRuleHelper.PartitionNameRule(item.datanodepartition);
string t = PartitionRuleHelper.GetDataNodeName(item.datanodepartition);
using (DbConn dataNodeConn = DbConfig.CreateConn(nodeConn))
{
dataNodeConn.Open();
var tablesinfo = new tb_messagequeue_dal().GetDataNodeTable(dataNodeConn, serverdate.AddDays(-30));
foreach (var table in tablesinfo)
{
string tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(table.TablePartition, table.Day);
TableInfo info = PartitionRuleHelper.GetTableInfo(tablename);
var beginId = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetMQID(new MQIDInfo() { AutoID = 0, DataNodePartition = item.datanodepartition, Day = table.Day, TablePartition = table.TablePartition });
int mqPartitionId = PartitionRuleHelper.GetPartitionID(new PartitionIDInfo() { DataNodePartition = item.datanodepartition, TablePartition = table.TablePartition });
tb_partition_model timeModel = createTimeList.Where(q => q.partitionid == mqPartitionId).FirstOrDefault();
DateTime time = DateTime.Parse("2015-01-01");
if (timeModel != null)
{
time = timeModel.createtime;
}
var dal = new tb_messagequeue_dal(); dal.TableName = tablename;
int count = dal.GetCount(dataNodeConn, time);
long maxId = dal.GetMaxID(dataNodeConn, time); maxId = (maxId == 0 ? beginId : maxId);
long minId = dal.GetMinID(dataNodeConn, time); minId = (minId == 0 ? beginId : minId);
tb_partition_messagequeue_report_model model = new tb_partition_messagequeue_report_model();
model.partitionid = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionID(new PartitionIDInfo() { DataNodePartition = item.datanodepartition, TablePartition = table.TablePartition });
MQIDInfo mqInfo = PartitionRuleHelper.GetMQIDInfo(maxId);
model.day = mqInfo.Day;
model.lastupdatetime = timenow;
model.createtime = timenow;
model.mqmaxid = maxId;
model.mqminid = minId;
model.mqcount = count;
new tb_partition_messagequeue_report_dal().AddReport(conn, model);
}
}
}
catch (Exception exp)
{
exps.Add(exp);
}
}
Error(this.AppConfig["BusinessMQManageConnectString"], "消息统计出错", exps);
}
}
/// <summary>
/// 任务调度平台根据发布的任务时间配置，定时回调运行方法
/// 开发人员的任务插件必须要重载并该方法
/// </summary>
public override void Run()
{
runcount++;
ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]);
List<tb_partition_model> userdpartitions = new List<tb_partition_model>();
Dictionary<int, tb_datanode_model> datanodemodels = new Dictionary<int, tb_datanode_model>();
List<RegisterdConsumersModel> registeredconsumermodels = new List<RegisterdConsumersModel>();
List<tb_consumer_model> consumers = new List<tb_consumer_model>();
DateTime servertime = DateTime.Now;
SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) =>
{
tb_partition_dal partitiondal = new tb_partition_dal();
userdpartitions = partitiondal.List(c, true);
tb_datanode_dal datanodedal = new tb_datanode_dal();
var ms = datanodedal.List(c);
foreach (var m in ms)
{
datanodemodels.Add(m.datanodepartition, m);
}
servertime = c.GetServerDate();
//获取当前注册的消费者
tb_consumer_dal consumerdal = new tb_consumer_dal();
registeredconsumermodels = consumerdal.GetRegisterdConsumers(c);
consumers = consumerdal.GetAllList(c);
});
//消费者端心跳停止,异常退出检测(超过1分钟)
List<Exception> exps = new List<Exception>();
foreach (var c in consumers)
{
if ((servertime - c.lastheartbeat) > TimeSpan.FromMinutes(1))
{
exps.Add(new Exception(string.Format("当前消费者tempid:{0},consumerclientid:{1}", c.tempid, c.consumerclientid)));
}
}
Error(this.AppConfig["BusinessMQManageConnectString"], "消费者端心跳停止,异常退出检测(超过1分钟),若是异常停止,请手工清除消费者中断", exps);
//消费者端长时间无消费，但是还有多余消息（超过10分钟未消费,但还有多余消息未消费）
exps.Clear();
foreach (var c in registeredconsumermodels)
{
if ((servertime - c.consumerpartitionmodel.lastupdatetime) > TimeSpan.FromMinutes(10))
{
var partitionidinfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionIDInfo(c.consumerpartitionmodel.partitionid);
var tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(partitionidinfo.TablePartition, servertime.Date);
SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[partitionidinfo.DataNodePartition]), (conn) =>
{
tb_messagequeue_dal messagedal = new tb_messagequeue_dal(); messagedal.TableName = tablename;
long maxmqid = messagedal.GetMaxID(conn);
if (maxmqid > c.consumerpartitionmodel.lastmqid && (servertime - messagedal.GetIDCreateTime(conn, maxmqid).Value > TimeSpan.FromMinutes(10)))//还有消息未消费，且该消息的创建时间是10分钟前
{
exps.Add(new Exception(string.Format("当前消费者tempid:{0},consumerclientid:{1},client:{2},clientname:{3}", c.consumermodel.tempid,
c.consumermodel.consumerclientid, c.consumerclientmodel.client, c.consumermodel.clientname)));
}
});
}
}
Error(this.AppConfig["BusinessMQManageConnectString"], "消费者端长时间无消费，但是还有多余消息（超过已创建超过10分钟的消息未消费）,可能是消费者处理逻辑异常", exps);
//某个分区没有指定消费者(分区没有消费信息,或者分区消费最后消费时间超过1个小时无消费)
exps.Clear();
if (runcount % (12*6) == 0)
{
foreach (var u in userdpartitions)
{
bool isfind = false;
foreach (var c in registeredconsumermodels)
{
if (c.consumerpartitionmodel.partitionid == u.partitionid)
{
isfind = true; break;
}
}
if (isfind == false)
{
SqlHelper.ExcuteSql(this.AppConfig["BusinessMQManageConnectString"], (c) =>
{
tb_consumer_partition_dal dal = new tb_consumer_partition_dal();
var model = dal.GetByPartitionId(c, u.partitionid);
if (model == null)
exps.Add(new Exception(string.Format("分区{0}未指定消费者", u.partitionid)));
//else if((servertime - model.lastupdatetime) > TimeSpan.FromHours(24))
// exps.Add(new Exception(string.Format("分区{0}消费者已注销,且有1天未消费", u.partitionid)));
});
}
}
Error(this.AppConfig["BusinessMQManageConnectString"], "某个分区没有指定消费者(分区没有消费信息)", exps);
}
}
/// <summary>
/// 获取所有分区节点
/// </summary>
/// <returns></returns>
public JsonResult GetNode()
{
using (DbConn conn = DbConfig.CreateConn(DataConfig.MqManage))
{
conn.Open();
IList<string> list = new tb_datanode_dal().GetNodeList(conn);
if (list != null && list.Count > 0)
{
return Json(list, JsonRequestBehavior.AllowGet);
}
return Json("", JsonRequestBehavior.AllowGet);
}
}
