public JsonResult GetDayList(string node, string tablepartition)
{
try
{
string mqpath = "";
using (DbConn conn = DbConfig.CreateConn(DataConfig.MqManage))
{
conn.Open();
tb_mqpath_dal mqpathdal = new tb_mqpath_dal();
var model = mqpathdal.GetByPartitionID(conn, PartitionRuleHelper.GetPartitionID(new PartitionIDInfo()
{
DataNodePartition = Convert.ToInt32(node), TablePartition = Convert.ToInt32(tablepartition)
}));
if (model != null)
{
mqpath = model.mqpath;
}
}
using (DbConn conn = DbConfig.CreateConn(DataConfig.DataNodeParConn(node)))
{
conn.Open();
var array = new tb_messagequeue_dal().GetDayPartitions(conn, Convert.ToInt32(tablepartition)).Select(c => c.ToString("yyMMdd")).ToList();
if (array != null && array.Count > 0)
{
return(Json(new { array, mqpath }, JsonRequestBehavior.AllowGet));
}
return(Json("", JsonRequestBehavior.AllowGet));
}
}
catch (Exception ex)
{
return(Json("", JsonRequestBehavior.AllowGet));
}
}
public ActionResult Update(tb_messagequeue_model model, string node, long id)
{
ViewBag.node = node;
var mqidinfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetMQIDInfo(Convert.ToInt64(id));
ViewBag.tablepartition = mqidinfo.TablePartition; ViewBag.daypartition = mqidinfo.Day.ToString("yyMMdd");
string tablename = PartitionRuleHelper.GetTableName(mqidinfo.TablePartition, mqidinfo.Day);
var dal = new tb_messagequeue_dal(); dal.TableName = tablename;
using (DbConn conn = DbConfig.CreateConn(DataConfig.DataNodeParConn(node)))
{
conn.Open();
tb_messagequeue_model result = dal.GetModel(conn, id, tablename);
if (result != null)
{
result.message = model.message;
result.state = model.state;
result.source = model.source;
if (dal.Update(conn, result, tablename))
{
return(RedirectToAction("Index", new { node = node, tablepartition = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.PartitionNameRule(mqidinfo.TablePartition), daypartition = mqidinfo.Day.ToString("yyMMdd"), mqid = id }));
}
}
ModelState.AddModelError("Error", "更新错误");
return(View(result));
}
}
public ProducterInfo GetProducterInfo(DbConn PubConn, string mqpath, string productername)
{
ProducterInfo r = new ProducterInfo();
tb_mqpath_dal mqpathdal = new tb_mqpath_dal();
r.MqPathModel = mqpathdal.Get(PubConn, mqpath);
if (r.MqPathModel == null)
{
throw new BusinessMQException(string.Format("当前mqpath:{0}未在MQ中注册队列,请联系管理员注册成功后使用。", mqpath));
}
tb_mqpath_partition_dal mqpathpartitiondal = new tb_mqpath_partition_dal();
r.MqPathParitionModel = mqpathpartitiondal.GetList(PubConn, r.MqPathModel.id);
if (r.MqPathParitionModel == null || r.MqPathParitionModel.Count == 0)
{
throw new BusinessMQException(string.Format("当前mqpath:{0}未在MQ中分配相应的分区,请联系管理员分配分区后使用。", mqpath));
}
//注册生产者
r.ProducterModel = this.RegisterProducter(PubConn, r.MqPathModel.id, productername);
//获取分区相关节点
List <int> datanodepartition = new List <int>();
foreach (var p in r.MqPathParitionModel)
{
var partitionidinfo = PartitionRuleHelper.GetPartitionIDInfo(p.partitionid);
if (!datanodepartition.Contains(partitionidinfo.DataNodePartition))
{
datanodepartition.Add(partitionidinfo.DataNodePartition);
}
}
r.DataNodeModelDic = this.GetDataNodeModelsDic(PubConn, datanodepartition);
return(r);
}
public ActionResult Index(string node = "", string tablepartition = "", string daypartition = "", string mqid = "", bool isclick = false, int pageIndex = 1, int pageSize = 30)
{
ViewBag.mqid = mqid;
ViewBag.node = node;
ViewBag.tablepartition = tablepartition; ViewBag.daypartition = daypartition; ViewBag.isclick = isclick;
int count = 0;
IList <tb_messagequeue_model> list = new List <tb_messagequeue_model>();
PagedList <tb_messagequeue_model> pageList = new PagedList <tb_messagequeue_model>(list, pageIndex, pageSize, count);
if (!string.IsNullOrWhiteSpace(node))
{
using (DbConn conn = DbConfig.CreateConn(DataConfig.DataNodeParConn(node)))
{
conn.Open();
string tablename = PartitionRuleHelper.GetTableName(Convert.ToInt32(tablepartition), DateTime.ParseExact(daypartition, "yyMMdd", CultureInfo.InvariantCulture));
tb_messagequeue_dal dal = new tb_messagequeue_dal(); dal.TableName = tablename;
list = dal.GetPageList(conn, pageIndex, pageSize, mqid, ref count);
pageList = new PagedList <tb_messagequeue_model>(list, pageIndex, pageSize, count);
}
}
if (Request.IsAjaxRequest())
{
return(PartialView("List", pageList));
}
return(View(pageList));
}
/// <summary>
/// 取得已消费的数量
/// </summary>
/// <param name="conn"></param>
/// <param name="lastMqId"></param>
/// <returns></returns>
public long GetMsgCount(DbConn conn, long lastMqId)
{
return(SqlHelper.Visit((ps) =>
{
MQIDInfo info = PartitionRuleHelper.GetMQIDInfo(lastMqId);
string sql = "SELECT SUM(mqcount) FROM [tb_partition_messagequeue_report] WITH(NOLOCK) WHERE [day]<@day AND partitionid=@partitionid";
ps.Add("@day", info.Day);
ps.Add("@partitionid", PartitionRuleHelper.GetPartitionID(new PartitionIDInfo()
{
DataNodePartition = info.DataNodePartition, TablePartition = info.TablePartition
}));
string tableName = PartitionRuleHelper.GetTableName(info.TablePartition, info.Day);
object obj = conn.ExecuteScalar(sql, ps.ToParameters());
long msgCount = 0;
if (obj != DBNull.Value && obj != null)
{
msgCount = LibConvert.ObjToInt64(obj);
}
long lastCount = 0;
using (DbConn nodeConn = DbConfig.CreateConn(DataConfig.DataNodeParConn(PartitionRuleHelper.PartitionNameRule(info.DataNodePartition))))
{
nodeConn.Open();
var dal = new tb_messagequeue_dal(); dal.TableName = tableName;
lastCount = dal.GetLastDayMsgCount(nodeConn, lastMqId);
}
return msgCount + lastCount;
}));
}
public ActionResult Add(tb_messagequeue_model model, string node, string tablepartition, string daypartition)
{
DateTime serverdate = DateTime.Now;
using (DbConn conn = DbConfig.CreateConn(DataConfig.MqManage))
{
conn.Open();
serverdate = conn.GetServerDate();
}
using (DbConn conn = DbConfig.CreateConn(DataConfig.DataNodeParConn(node)))
{
conn.Open();
model.sqlcreatetime = serverdate;
model.mqcreatetime = DateTime.Now;
string tablename = PartitionRuleHelper.GetTableName(Convert.ToInt32(tablepartition), DateTime.ParseExact(daypartition, "yyMMdd", CultureInfo.InvariantCulture));
var dal = new tb_messagequeue_dal(); dal.TableName = tablename;
if (dal.Add(conn, tablename, model))
{
return(RedirectToAction("index"));
}
else
{
ModelState.AddModelError("Error", "更新错误");
return(View(model));
}
}
}
private void CheckIfScanNewTable(tb_consumer_partition_model partition, MQIDInfo mqidinfo, string datanodeconnectstring, PartitionIDInfo partionidinfo)
{
var serverdate = Context.ManageServerTime.AddSeconds(-1); //此处延迟1秒，停顿消费者日分区表之间的切换
if (serverdate != null && serverdate.Date > mqidinfo.Day.Date) //服务器时间和消息时间对应不上
{
SqlHelper.ExcuteSql(datanodeconnectstring, (c) =>
{
DateTime t = mqidinfo.Day.Date.AddDays(1);
while (serverdate.Date >= t) //查找最后的可用消息表,然后跳出
{
string tablename = PartitionRuleHelper.GetTableName(partionidinfo.TablePartition, t);
if (exsittablenames.ContainsKey(tablename) || c.TableIsExist(tablename))
{
AddExsitTableNameCache(tablename);
_lastpullququeiddic[partition.partitionid] = PartitionRuleHelper.GetMQID(new MQIDInfo()
{
AutoID = 0,
DataNodePartition = mqidinfo.DataNodePartition,
TablePartition = mqidinfo.TablePartition,
Day = t
}); //重设第二天最小的id
break;
}
t = t.AddDays(1);
}
});
}
}
public ActionResult Update(long id, string node)
{
using (DbConn conn = DbConfig.CreateConn(DataConfig.DataNodeParConn(node)))
{
conn.Open();
ViewBag.node = node;
var mqidinfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetMQIDInfo(Convert.ToInt64(id));
ViewBag.tablepartition = mqidinfo.TablePartition; ViewBag.daypartition = mqidinfo.Day.ToString("yyMMdd");
string tablename = PartitionRuleHelper.GetTableName(mqidinfo.TablePartition, mqidinfo.Day);
var dal = new tb_messagequeue_dal(); dal.TableName = tablename;
tb_messagequeue_model model = dal.GetModel(conn, id, tablename);
return(View(model));
}
}
public ActionResult Delete(long id, string node)
{
using (DbConn conn = DbConfig.CreateConn(DataConfig.DataNodeParConn(node)))
{
conn.Open();
var mqidinfo = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetMQIDInfo(Convert.ToInt64(id));
string tablename = PartitionRuleHelper.GetTableName(mqidinfo.TablePartition, mqidinfo.Day);
var dal = new tb_messagequeue_dal(); dal.TableName = tablename;
if (dal.SetState(conn, id, (byte)XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.EnumMessageState.Deleted))
{
return(Json(new { code = 1, msg = "删除成功" }));
}
return(Json(new { code = -1, msg = "删除失败" }));
}
}
/// <summary>
/// 取得未消费的数量
/// </summary>
/// <param name="lastMqId"></param>
/// <returns></returns>
public long GetNonMsgCount(DbConn conn, long lastMqId)
{
return(SqlHelper.Visit((ps) =>
{
MQIDInfo info = PartitionRuleHelper.GetMQIDInfo(lastMqId);
var currentday = conn.GetServerDate().Date;
string sql = "SELECT SUM(mqcount) FROM [tb_partition_messagequeue_report] WITH(NOLOCK) WHERE [day]>@day AND partitionid=@partitionid AND [day]<>@currentday";
ps.Add("@day", info.Day); ps.Add("@currentday", currentday);
ps.Add("@partitionid", PartitionRuleHelper.GetPartitionID(new PartitionIDInfo()
{
DataNodePartition = info.DataNodePartition, TablePartition = info.TablePartition
}));
object obj = conn.ExecuteScalar(sql, ps.ToParameters());
long msgCount = 0;
if (obj != DBNull.Value && obj != null)
{
msgCount = LibConvert.ObjToInt64(obj);
}
long firstCount = 0; long lastCount = 0;
using (DbConn nodeConn = DbConfig.CreateConn(DataConfig.DataNodeParConn(PartitionRuleHelper.PartitionNameRule(info.DataNodePartition))))
{
nodeConn.Open();
string firsttableName = PartitionRuleHelper.GetTableName(info.TablePartition, info.Day);
var msgDal = new tb_messagequeue_dal(); msgDal.TableName = firsttableName;
firstCount = msgDal.GetLastDayNonMsgCount(nodeConn, lastMqId);
if (info.Day != currentday)//不是今天
{
string lasttableName = PartitionRuleHelper.GetTableName(info.TablePartition, currentday);
var dal = new tb_messagequeue_dal(); dal.TableName = lasttableName;
long maxmqid = dal.GetMaxID(nodeConn);
if (lastMqId == 0)
{
lastCount = 0;
}
else
{
lastCount = dal.GetLastDayMsgCount(nodeConn, maxmqid);
}
}
}
//最后一天剩余
return msgCount + firstCount + lastCount;
}));
}
private List <tb_messagequeue_model> GetMessagesOfPatition(tb_consumer_partition_model partition, long scanlastmqid, MQIDInfo mqidinfo, string datanodeconnectstring, PartitionIDInfo partionidinfo)
{
//从数据库取数据,检查数据时间，和数据状态，无问题则插入队列，并修改上一次扫描的mqid
List <tb_messagequeue_model> messages = new List <tb_messagequeue_model>();
SqlHelper.ExcuteSql(datanodeconnectstring, (c) =>
{
string tablename = PartitionRuleHelper.GetTableName(partionidinfo.TablePartition, mqidinfo.Day);
if (exsittablenames.ContainsKey(tablename) || c.TableIsExist(tablename))
{
AddExsitTableNameCache(tablename);
tb_messagequeue_dal dal = new tb_messagequeue_dal(); dal.TableName = tablename;
messages = dal.GetMessages(c, scanlastmqid, SystemParamConfig.Consumer_ReceiveMessageQuque_EVERY_PULL_COUNT);
}
});
return(messages);
}
/// <summary>
/// 移除生产者某个节点相关的分区信息
/// </summary>
/// <param name="datanodepartition"></param>
public void RemoveMQPathPartition(int datanodepartition)
{
lock (_operatorlock)
{
List <tb_mqpath_partition_model> remove = new List <tb_mqpath_partition_model>();
foreach (var p in MqPathParitionModel)
{
var partitionidinfo = PartitionRuleHelper.GetPartitionIDInfo(p.partitionid);
if (partitionidinfo.DataNodePartition == datanodepartition)
{
remove.Add(p);
}
}
foreach (var p in remove)
{
MqPathParitionModel.Remove(p);
}
}
}
public tb_consumer_partition_model RegisterConsumerPartition(DbConn PubConn, int clientid, int partitionindex, string mqpath, long tempid)
{
var mqpathmodel = GetMQPath(PubConn, mqpath);
tb_mqpath_partition_dal mqpathpartitiondal = new tb_mqpath_partition_dal();
var mqpathpartitionmodel = mqpathpartitiondal.GetOfConsumer(PubConn, partitionindex, mqpathmodel.id);
if (mqpathpartitionmodel == null)
{
return(null);
}
tb_consumer_partition_dal dal = new tb_consumer_partition_dal();
if (dal.Edit2(PubConn, new tb_consumer_partition_model()
{
consumerclientid = clientid, lastconsumertempid = tempid, partitionid = mqpathpartitionmodel.partitionid, partitionindex = partitionindex
}) <= 0)
{
var partitionidinfo = PartitionRuleHelper.GetPartitionIDInfo(mqpathpartitionmodel.partitionid); var datanodename = PartitionRuleHelper.GetDataNodeName(partitionidinfo.DataNodePartition);
long maxid = -1; DateTime serverdate = PubConn.GetServerDate(); string tablename = PartitionRuleHelper.GetTableName(partitionidinfo.TablePartition, serverdate);
SqlHelper.ExcuteSql(this.GetDataNodeConnectString(PubConn, partitionidinfo.DataNodePartition), (c) => {
var exist = c.TableIsExist(tablename); if (!exist)
{
throw new BusinessMQException(string.Format("当前数据节点{0},表{1}不存在", partitionidinfo.DataNodePartition, tablename));
}
tb_messagequeue_dal mqdal = new tb_messagequeue_dal(); mqdal.TableName = tablename;
maxid = mqdal.GetMaxId(c);
if (maxid <= 0)
{
maxid = PartitionRuleHelper.GetMQID(new MQIDInfo()
{
AutoID = 0, DataNodePartition = partitionidinfo.DataNodePartition, Day = serverdate, TablePartition = partitionidinfo.TablePartition
});
}
});
dal.Add2(PubConn, new tb_consumer_partition_model()
{
consumerclientid = clientid, lastconsumertempid = tempid, lastmqid = maxid, partitionid = mqpathpartitionmodel.partitionid, partitionindex = partitionindex
});
}
return(dal.Get(PubConn, clientid, mqpathpartitionmodel.partitionid));
}
/// <summary>
/// //顺序轮询节点获取节点及分区,从而达到负载均衡的目的
/// </summary>
/// <param name="sendmessagecount"></param>
/// <returns></returns>
public LoadBalanceNodeInfo GetLoadBalanceNodeInfo()
{
try
{
lock (_operatorlock)
{
LoadBalanceNodeInfo info = new LoadBalanceNodeInfo();
var p = LoadBalance.GetLoadBalancePartitionInfo();
if (p == null)
{
return(null);
}
var partitionidinfo = PartitionRuleHelper.GetPartitionIDInfo(p.PartitionId);
info.DataNodeModel = DataNodeModelDic[partitionidinfo.DataNodePartition]; info.MQPathPartitionModel = p.MQPathParitionModel;
return(info);
}
}
catch (Exception exp)
{
ErrorLogHelper.WriteLine(-1, "", "GetLoadBalanceNodeInfo", "生产者负载均衡出错", exp);
throw exp;
}
}
private void GetMessagesFromPartitions()
{
foreach (var partition in Context.ConsumerInfo.ConsumerPartitionModels)
{
if (cancelSource.IsCancellationRequested)
{
break;
}
try
{
if (errorpartitions.ContainsKey(partition.partitionid))
{
throw new BusinessMQException("分区消息处理出现错误", errorpartitions[partition.partitionid]);
}
var scanlastmqid = _lastpullququeiddic[partition.partitionid]; //上次扫描的mqid
var mqidinfo = PartitionRuleHelper.GetMQIDInfo(scanlastmqid); //解析上次扫描的mqid信息
ConsumerBLL consumerbll = new ConsumerBLL(); var partionidinfo = PartitionRuleHelper.GetPartitionIDInfo(partition.partitionid); //解析分区信息
string datanodeconnectstring = consumerbll.GetDataNodeConnectString(SystemParamConfig.Consumer_DataNode_ConnectString_Template, Context.ConsumerInfo.DataNodeModelDic[partionidinfo.DataNodePartition]); //获取节点连接
var messages = GetMessagesOfPatition(partition, scanlastmqid, mqidinfo, datanodeconnectstring, partionidinfo);
if (messages.Count > 0)
{
ConsumeMessages(messages, partition, mqidinfo);
}
else
{
CheckIfScanNewTable(partition, mqidinfo, datanodeconnectstring, partionidinfo);
}
}
catch (Exception exp)
{
ErrorLog.Write(string.Format("MQ消费者端消息循环出错,clientid:{0},partitionid:{1}", partition.consumerclientid, partition.partitionid), exp);
}
}
}
public override void Run()
{
ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]);
List <tb_datanode_model> nodeList = new List <tb_datanode_model>();
using (DbConn conn = DbConfig.CreateConn(this.AppConfig["BusinessMQManageConnectString"]))
{
conn.Open();
List <tb_partition_model> createTimeList = new tb_consumer_dal().GetClientCreateTime(conn);
var serverdate = conn.GetServerDate().Date; var timenow = conn.GetServerDate();
nodeList = new tb_datanode_dal().List(conn);
List <Exception> exps = new List <Exception>();
foreach (var item in nodeList)
{
try
{
string dataNode = PartitionRuleHelper.GetDataNodeName(item.datanodepartition);
string nodeConn = string.Format("server={0};Initial Catalog={1};User ID={2};Password={3};", item.serverip, dataNode, item.username, item.password);
string partitionId = PartitionRuleHelper.PartitionNameRule(item.datanodepartition);
string t = PartitionRuleHelper.GetDataNodeName(item.datanodepartition);
using (DbConn dataNodeConn = DbConfig.CreateConn(nodeConn))
{
dataNodeConn.Open();
var tablesinfo = new tb_messagequeue_dal().GetDataNodeTable(dataNodeConn, serverdate.AddDays(-30));
foreach (var table in tablesinfo)
{
string tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(table.TablePartition, table.Day);
TableInfo info = PartitionRuleHelper.GetTableInfo(tablename);
var beginId = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetMQID(new MQIDInfo()
{
AutoID = 0, DataNodePartition = item.datanodepartition, Day = table.Day, TablePartition = table.TablePartition
});
int mqPartitionId = PartitionRuleHelper.GetPartitionID(new PartitionIDInfo()
{
DataNodePartition = item.datanodepartition, TablePartition = table.TablePartition
});
tb_partition_model timeModel = createTimeList.Where(q => q.partitionid == mqPartitionId).FirstOrDefault();
DateTime time = DateTime.Parse("2015-01-01");
if (timeModel != null)
{
time = timeModel.createtime;
}
var dal = new tb_messagequeue_dal(); dal.TableName = tablename;
int count = dal.GetCount(dataNodeConn, time);
long maxId = dal.GetMaxID(dataNodeConn, time); maxId = (maxId == 0 ? beginId : maxId);
long minId = dal.GetMinID(dataNodeConn, time); minId = (minId == 0 ? beginId : minId);
tb_partition_messagequeue_report_model model = new tb_partition_messagequeue_report_model();
model.partitionid = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionID(new PartitionIDInfo()
{
DataNodePartition = item.datanodepartition, TablePartition = table.TablePartition
});
MQIDInfo mqInfo = PartitionRuleHelper.GetMQIDInfo(maxId);
model.day = mqInfo.Day;
model.lastupdatetime = timenow;
model.createtime = timenow;
model.mqmaxid = maxId;
model.mqminid = minId;
model.mqcount = count;
new tb_partition_messagequeue_report_dal().AddReport(conn, model);
}
}
}
catch (Exception exp)
{
exps.Add(exp);
}
}
Error(this.AppConfig["BusinessMQManageConnectString"], "消息统计出错", exps);
}
}
private void RegisterConsumerInfo()
{
ConsumerBLL bll = new ConsumerBLL();
DebugHelper.WriteLine(Context.GetMQPathID(), Context.GetMQPath(), "RegisterConsumerInfo", "消费者开始注册消费者信息");
//注册消费者
SqlHelper.ExcuteSql(Config.ManageConnectString, (c) =>
{
try
{
//c.BeginTransaction();
//取消注册用户信息
if (Context.ConsumerInfo != null && Context.ConsumerInfo.ConsumerModel != null)
{
bll.RemoveConsumer(c, Context.ConsumerInfo.ConsumerModel.tempid, Context.ConsumerInfo.ConsumerModel.consumerclientid);
}
Context.ConsumerInfo = new ConsumerInfo(); Context.ConsumerInfo.ConsumerPartitionModels = new List <Model.tb_consumer_partition_model>();
//注册并获取clientid
Context.ConsumerInfo.ConsumerClientModel = bll.RegisterClient(c, Client);
if (Context.ConsumerInfo.ConsumerClientModel == null)
{
throw new BusinessMQException("客户端注册client失败！");
}
DebugHelper.WriteLine(Context.GetMQPathID(), Context.GetMQPath(), "RegisterConsumerInfo", "注册并获取clientid成功");
//清理过期不心跳消费端,并检验partitionindex不重复,并注册消费者
Context.ConsumerInfo.ConsumerModel = bll.RegisterConsumer(c, Context.ConsumerInfo.ConsumerClientModel.id, ClientName, PartitionIndexs);
if (Context.ConsumerInfo.ConsumerModel == null)
{
throw new BusinessMQException("当前客户端注册consumer失败！");
}
DebugHelper.WriteLine(Context.GetMQPathID(), Context.GetMQPath(), "RegisterConsumerInfo", "清理过期不心跳消费端,并检验partitionindex不重复,并注册消费者成功");
//消费者订阅队列信息
Context.ConsumerInfo.MQPathModel = bll.GetMQPath(c, this.MQPath);
DebugHelper.WriteLine(Context.GetMQPathID(), Context.GetMQPath(), "RegisterConsumerInfo", "消费者订阅队列信息");
//注册并更新消费者端分区信息
foreach (var partitionindex in PartitionIndexs)
{
var model = bll.RegisterConsumerPartition(c, Context.ConsumerInfo.ConsumerClientModel.id, partitionindex, MQPath, Context.ConsumerInfo.ConsumerModel.tempid);
if (model != null)
{
Context.ConsumerInfo.ConsumerPartitionModels.Add(model);
}
}
DebugHelper.WriteLine(Context.GetMQPathID(), Context.GetMQPath(), "RegisterConsumerInfo", "注册并更新消费者端分区信息");
//获取分区节点信息缓存
List <int> datanodepartition = new List <int>();
foreach (var p in Context.ConsumerInfo.ConsumerPartitionModels)
{
var partitionidinfo = PartitionRuleHelper.GetPartitionIDInfo(p.partitionid);
if (!datanodepartition.Contains(partitionidinfo.DataNodePartition))
{
datanodepartition.Add(partitionidinfo.DataNodePartition);
}
}
Context.ConsumerInfo.DataNodeModelDic = bll.GetDataNodeModelsDic(c, datanodepartition);
DebugHelper.WriteLine(Context.GetMQPathID(), Context.GetMQPath(), "RegisterConsumerInfo", "获取分区节点信息缓存");
//校准服务器时间
Context.ManageServerTime = c.GetServerDate();
//c.Commit();
}
catch (Exception exp)
{
//c.Rollback();
throw exp;
}
});
DebugHelper.WriteLine(Context.GetMQPathID(), Context.GetMQPath(), "RegisterConsumerInfo", "注册消费者信息完毕");
//获取系统配置信息
ConfigHelper.LoadConfig(Config.ManageConnectString);
DebugHelper.WriteLine(Context.GetMQPathID(), Context.GetMQPath(), "RegisterConsumerInfo", "获取系统配置信息");
}
public override void Run()
{
List <tb_datanode_model> nodeList = new List <tb_datanode_model>();
List <tb_partition_model> list = new List <tb_partition_model>();
using (DbConn conn = DbConfig.CreateConn(this.AppConfig["BusinessMQManageConnectString"]))
{
conn.Open();
nodeList = dataNodeDal.List(conn);
list = partitionDal.GetAllCanUsePartitionList(conn);
}
foreach (var item in nodeList)
{
string dataNode = PartitionRuleHelper.GetDataNodeName(item.datanodepartition);
string nodeConn = string.Format("server={0};Initial Catalog={1};User ID={2};Password={3};", item.serverip, dataNode, item.username, item.password);
try
{
using (DbConn dataNodeConn = DbConfig.CreateConn(nodeConn))
{
dataNodeConn.Open();
foreach (var value in list)
{
PartitionIDInfo info = PartitionRuleHelper.GetPartitionIDInfo(value.partitionid);
string partition = PartitionRuleHelper.PartitionNameRule(info.TablePartition);
ICollection <string> tableList = msgDal.GetTableNameListByPartition(dataNodeConn, partition);
if (tableList != null && tableList.Count > 0)
{
using (DbConn conn = DbConfig.CreateConn(this.AppConfig["BusinessMQManageConnectString"]))
{
foreach (var table in tableList)
{
if (!dataNodeConn.TableIsExist(table))
{
continue;
}
long maxId = msgDal.GetMaxID(dataNodeConn, table);
long minId = msgDal.GetMinID(dataNodeConn, table);
tb_partition_messagequeue_report_model model = new tb_partition_messagequeue_report_model();
model.partitionid = value.partitionid;
if (maxId > 0 && minId > 0)
{
MQIDInfo mqInfo = PartitionRuleHelper.GetMQIDInfo(maxId);
model.day = mqInfo.Day;
model.lastupdatetime = DateTime.Now;
model.createtime = DateTime.Now;
model.mqmaxid = maxId;
model.mqminid = minId;
reportDal.AddReport(conn, model);
}
}
}
}
}
}
}
catch (Exception ex)
{
base.OpenOperator.Error("节点不存在", ex);
continue;
}
}
}
public IList <ConsumerModel> GetPageList(DbConn conn, string name, int pageIndex, int pageSize, ref int count)
{
int tempCount = 0;
IList <ConsumerModel> list = new List <ConsumerModel>();
ConsumerModel cm = new ConsumerModel();
var result = SqlHelper.Visit((ps) =>
{
StringBuilder where = new StringBuilder(" WHERE 1=1 ");
if (!string.IsNullOrEmpty(name))
{
where.AppendFormat(" AND clientname LIKE '%{0}%'", name);
}
string sql = "SELECT ROW_NUMBER() OVER(ORDER BY Id DESC) AS rownum,* FROM tb_consumer WITH(NOLOCK)";
string countSql = "SELECT COUNT(1) FROM tb_consumer WITH(NOLOCK) " + where.ToString();
object obj = conn.ExecuteScalar(countSql, null);
if (obj != DBNull.Value && obj != null)
{
tempCount = LibConvert.ObjToInt(obj);
}
string sqlPage = string.Concat("SELECT * FROM (", sql.ToString(), where.ToString(), ") A WHERE rownum BETWEEN ", ((pageIndex - 1) * pageSize + 1), " AND ", pageSize * pageIndex);
DataTable dt = conn.SqlToDataTable(sqlPage, null);
if (dt != null && dt.Rows.Count > 0)
{
foreach (DataRow dr in dt.Rows)
{
ConsumerModel model = cm.CreateModel(dr);
IList <tb_consumer_partition_model> consumerList = partitionDal.GetPartitionByConsumerId(conn, model.consumerclientid);
if (consumerList != null && consumerList.Count > 0)
{
IList <ConsumerPartition> partitionList = new List <ConsumerPartition>();
foreach (var item in consumerList)
{
ConsumerPartition m = new ConsumerPartition();
m.PartitionId = item.partitionid;
PartitionIDInfo partitionInfo = PartitionRuleHelper.GetPartitionIDInfo(item.partitionid);
string node = string.Empty;
if (partitionInfo.DataNodePartition < 10)
{
node = "0" + partitionInfo.DataNodePartition.Tostring();
}
else
{
node = partitionInfo.DataNodePartition.Tostring();
}
using (DbConn nodeConn = DbConfig.CreateConn(DataConfig.DataNodeParConn(node)))
{
nodeConn.Open();
tb_partition_model partitionModel = new tb_partition_dal().Get(conn, item.partitionid);
if (partitionModel != null)
{
m.IsOnline = partitionModel.isused;
}
string table = msgDal.GetMaxMqTable(nodeConn, node);
m.Msg = msgDal.GetMsgCount(nodeConn, table, 0);
m.NonMsg = msgDal.GetMsgCount(nodeConn, table, 1);
partitionList.Add(m);
}
}
model.PartitionList = partitionList;
}
list.Add(model);
}
}
return(list);
});
count = tempCount;
return(result);
}
/// <summary>
/// 发送消息
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="objmsg"></param>
public void SendMessage <T>(T objmsg)
{
try
{
var json = "";
ProducterTimeWatchInfo.JsonHelperSerializer += Log.TimeWatchLog.Debug(() =>
{
if (!(objmsg is string))
{
json = new Serialization.JsonHelper().Serializer(objmsg);
}
else
{
json = objmsg as string;
}
});
//发送消息有n次重试机会
int errortrycount = 0;
while (errortrycount < Context.ProducterInfo.LoadBalance.SendMessageErrorTryAgainCount)
{
LoadBalanceNodeInfo loadbalancenodeinfo = null;
ProducterTimeWatchInfo.GetLoadBalanceNodeInfo += Log.TimeWatchLog.Debug(() =>
{
loadbalancenodeinfo = Context.ProducterInfo.GetLoadBalanceNodeInfo();
});
if (loadbalancenodeinfo == null)
{
throw new BusinessMQException(string.Format("无法获取队列{0}的可用的负载均衡数据节点", MQPath));
}
string datanodeconnectstring = new ProducterBLL().GetDataNodeConnectString(SystemParamConfig.Producter_DataNode_ConnectString_Template_ToSendMessage, loadbalancenodeinfo.DataNodeModel);
var partitionidinfo = PartitionRuleHelper.GetPartitionIDInfo(loadbalancenodeinfo.MQPathPartitionModel.partitionid);
var manageservertime = Context.ManageServerTime;//.AddSeconds(1);发送消息要比标准时间提前1s，这样消息分表可以提前1s
string tablename = PartitionRuleHelper.GetTableName(partitionidinfo.TablePartition, manageservertime);
try
{
ProducterTimeWatchInfo.SendMessage += Log.TimeWatchLog.Debug(() =>
{
double inserttime = 0;
double allinserttime = Log.TimeWatchLog.Debug(() =>
{
inserttime = Log.TimeWatchLog.Debug(() =>
{
tb_messagequeue_dal dal = new tb_messagequeue_dal();
dal.TableName = tablename;
SqlHelper.ExcuteSql(datanodeconnectstring, (c) =>
{
dal.Add2(c, new tb_messagequeue_model()
{
message = json,
mqcreatetime = DateTime.Now,
sqlcreatetime = manageservertime
,
source = (int)EnumMessageSource.Common,
state = (int)EnumMessageState.CanRead
});
});
});
});
//ProducterTimeWatchTest.AddMessages(string.Format("总插入消息:{0}s,插入消息:{1}s",allinserttime,inserttime));
});
NetCommand.SendMessage(mqpath);
return;
}
catch (SqlException exp)
{
ErrorLogHelper.WriteLine(Context.GetMQPathID(), Context.GetMQPath(), "SendMessage", string.Format("发送消息出现节点错误,节点:{0}", loadbalancenodeinfo.DataNodeModel.datanodepartition), exp);
Context.ProducterInfo.RemoveMQPathPartition(loadbalancenodeinfo.DataNodeModel.datanodepartition);//数据层出错视为数据节点异常，则移除。将在一定时间内尝试恢复
Context.ProducterInfo.LoadBalance.AddError(new ErrorLoadBalancePartitionInfo()
{
PartitionId = loadbalancenodeinfo.MQPathPartitionModel.partitionid, PartitionIndex = loadbalancenodeinfo.MQPathPartitionModel.partitionindex
});
//Context.IsNeedReload = true;
if (Context.SendMessageErrorTime == null)
{
Context.SendMessageErrorTime = DateTime.Now;
}
}
errortrycount++;
}
throw new BusinessMQException(string.Format("发送消息出现系统级错误,并超过重试次数,请检查。队列:{0}", MQPath));
}
catch (Exception exp)
{
ErrorLogHelper.WriteLine(Context.GetMQPathID(), Context.GetMQPath(), "SendMessage", "生产者发送消息出错", exp);
throw exp;
}
}
public override LoadBalancePartitionInfo GetLoadBalancePartitionInfo()
{
System.Threading.Interlocked.Increment(ref SendMessageCount);
if (SendMessageCount > 1000000000)
{
SendMessageCount = 1;
}
if (this.MQPathParitionModels.Count > 0)
{
LoadBalanceNodeInfo info = new LoadBalanceNodeInfo();
int index = (SendMessageCount - 1) % MQPathParitionModels.Count; var partion = MQPathParitionModels[index]; var partitionidinfo = PartitionRuleHelper.GetPartitionIDInfo(partion.partitionid);
return(new LoadBalancePartitionInfo()
{
PartitionId = partion.partitionid, PartitionIndex = partion.partitionindex, MQPathParitionModel = partion
});
}
else
{
ErrorLogHelper.WriteLine(-1, "", "SystemLoadBalance-LoadBalancePartitionInfo", "系统默认生产者负载均衡出错:当前可用分区数为0", new Exception());
return(null);
}
}
