/// <summary>
/// 取得已消费的数量
/// </summary>
/// <param name="conn"></param>
/// <param name="lastMqId"></param>
/// <returns></returns>
public long GetMsgCount(DbConn conn, long lastMqId)
{
return(SqlHelper.Visit((ps) =>
{
MQIDInfo info = PartitionRuleHelper.GetMQIDInfo(lastMqId);
string sql = "SELECT SUM(mqcount) FROM [tb_partition_messagequeue_report] WITH(NOLOCK) WHERE [day]<@day AND partitionid=@partitionid";
ps.Add("@day", info.Day);
ps.Add("@partitionid", PartitionRuleHelper.GetPartitionID(new PartitionIDInfo()
{
DataNodePartition = info.DataNodePartition, TablePartition = info.TablePartition
}));
string tableName = PartitionRuleHelper.GetTableName(info.TablePartition, info.Day);
object obj = conn.ExecuteScalar(sql, ps.ToParameters());
long msgCount = 0;
if (obj != DBNull.Value && obj != null)
{
msgCount = LibConvert.ObjToInt64(obj);
}
long lastCount = 0;
using (DbConn nodeConn = DbConfig.CreateConn(DataConfig.DataNodeParConn(PartitionRuleHelper.PartitionNameRule(info.DataNodePartition))))
{
nodeConn.Open();
var dal = new tb_messagequeue_dal(); dal.TableName = tableName;
lastCount = dal.GetLastDayMsgCount(nodeConn, lastMqId);
}
return msgCount + lastCount;
}));
}
private void CheckIfScanNewTable(tb_consumer_partition_model partition, MQIDInfo mqidinfo, string datanodeconnectstring, PartitionIDInfo partionidinfo)
{
var serverdate = Context.ManageServerTime.AddSeconds(-1); //此处延迟1秒，停顿消费者日分区表之间的切换
if (serverdate != null && serverdate.Date > mqidinfo.Day.Date) //服务器时间和消息时间对应不上
{
SqlHelper.ExcuteSql(datanodeconnectstring, (c) =>
{
DateTime t = mqidinfo.Day.Date.AddDays(1);
while (serverdate.Date >= t) //查找最后的可用消息表,然后跳出
{
string tablename = PartitionRuleHelper.GetTableName(partionidinfo.TablePartition, t);
if (exsittablenames.ContainsKey(tablename) || c.TableIsExist(tablename))
{
AddExsitTableNameCache(tablename);
_lastpullququeiddic[partition.partitionid] = PartitionRuleHelper.GetMQID(new MQIDInfo()
{
AutoID = 0,
DataNodePartition = mqidinfo.DataNodePartition,
TablePartition = mqidinfo.TablePartition,
Day = t
}); //重设第二天最小的id
break;
}
t = t.AddDays(1);
}
});
}
}
/// <summary>
/// 取得未消费的数量
/// </summary>
/// <param name="lastMqId"></param>
/// <returns></returns>
public long GetNonMsgCount(DbConn conn, long lastMqId)
{
return(SqlHelper.Visit((ps) =>
{
MQIDInfo info = PartitionRuleHelper.GetMQIDInfo(lastMqId);
var currentday = conn.GetServerDate().Date;
string sql = "SELECT SUM(mqcount) FROM [tb_partition_messagequeue_report] WITH(NOLOCK) WHERE [day]>@day AND partitionid=@partitionid AND [day]<>@currentday";
ps.Add("@day", info.Day); ps.Add("@currentday", currentday);
ps.Add("@partitionid", PartitionRuleHelper.GetPartitionID(new PartitionIDInfo()
{
DataNodePartition = info.DataNodePartition, TablePartition = info.TablePartition
}));
object obj = conn.ExecuteScalar(sql, ps.ToParameters());
long msgCount = 0;
if (obj != DBNull.Value && obj != null)
{
msgCount = LibConvert.ObjToInt64(obj);
}
long firstCount = 0; long lastCount = 0;
using (DbConn nodeConn = DbConfig.CreateConn(DataConfig.DataNodeParConn(PartitionRuleHelper.PartitionNameRule(info.DataNodePartition))))
{
nodeConn.Open();
string firsttableName = PartitionRuleHelper.GetTableName(info.TablePartition, info.Day);
var msgDal = new tb_messagequeue_dal(); msgDal.TableName = firsttableName;
firstCount = msgDal.GetLastDayNonMsgCount(nodeConn, lastMqId);
if (info.Day != currentday)//不是今天
{
string lasttableName = PartitionRuleHelper.GetTableName(info.TablePartition, currentday);
var dal = new tb_messagequeue_dal(); dal.TableName = lasttableName;
long maxmqid = dal.GetMaxID(nodeConn);
if (lastMqId == 0)
{
lastCount = 0;
}
else
{
lastCount = dal.GetLastDayMsgCount(nodeConn, maxmqid);
}
}
}
//最后一天剩余
return msgCount + firstCount + lastCount;
}));
}
public override void Run()
{
ConfigHelper.LoadConfig(this.AppConfig["BusinessMQManageConnectString"]);
List <tb_datanode_model> nodeList = new List <tb_datanode_model>();
using (DbConn conn = DbConfig.CreateConn(this.AppConfig["BusinessMQManageConnectString"]))
{
conn.Open();
List <tb_partition_model> createTimeList = new tb_consumer_dal().GetClientCreateTime(conn);
var serverdate = conn.GetServerDate().Date; var timenow = conn.GetServerDate();
nodeList = new tb_datanode_dal().List(conn);
List <Exception> exps = new List <Exception>();
foreach (var item in nodeList)
{
try
{
string dataNode = PartitionRuleHelper.GetDataNodeName(item.datanodepartition);
string nodeConn = string.Format("server={0};Initial Catalog={1};User ID={2};Password={3};", item.serverip, dataNode, item.username, item.password);
string partitionId = PartitionRuleHelper.PartitionNameRule(item.datanodepartition);
string t = PartitionRuleHelper.GetDataNodeName(item.datanodepartition);
using (DbConn dataNodeConn = DbConfig.CreateConn(nodeConn))
{
dataNodeConn.Open();
var tablesinfo = new tb_messagequeue_dal().GetDataNodeTable(dataNodeConn, serverdate.AddDays(-30));
foreach (var table in tablesinfo)
{
string tablename = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetTableName(table.TablePartition, table.Day);
TableInfo info = PartitionRuleHelper.GetTableInfo(tablename);
var beginId = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetMQID(new MQIDInfo()
{
AutoID = 0, DataNodePartition = item.datanodepartition, Day = table.Day, TablePartition = table.TablePartition
});
int mqPartitionId = PartitionRuleHelper.GetPartitionID(new PartitionIDInfo()
{
DataNodePartition = item.datanodepartition, TablePartition = table.TablePartition
});
tb_partition_model timeModel = createTimeList.Where(q => q.partitionid == mqPartitionId).FirstOrDefault();
DateTime time = DateTime.Parse("2015-01-01");
if (timeModel != null)
{
time = timeModel.createtime;
}
var dal = new tb_messagequeue_dal(); dal.TableName = tablename;
int count = dal.GetCount(dataNodeConn, time);
long maxId = dal.GetMaxID(dataNodeConn, time); maxId = (maxId == 0 ? beginId : maxId);
long minId = dal.GetMinID(dataNodeConn, time); minId = (minId == 0 ? beginId : minId);
tb_partition_messagequeue_report_model model = new tb_partition_messagequeue_report_model();
model.partitionid = XXF.BaseService.MessageQuque.BusinessMQ.SystemRuntime.PartitionRuleHelper.GetPartitionID(new PartitionIDInfo()
{
DataNodePartition = item.datanodepartition, TablePartition = table.TablePartition
});
MQIDInfo mqInfo = PartitionRuleHelper.GetMQIDInfo(maxId);
model.day = mqInfo.Day;
model.lastupdatetime = timenow;
model.createtime = timenow;
model.mqmaxid = maxId;
model.mqminid = minId;
model.mqcount = count;
new tb_partition_messagequeue_report_dal().AddReport(conn, model);
}
}
}
catch (Exception exp)
{
exps.Add(exp);
}
}
Error(this.AppConfig["BusinessMQManageConnectString"], "消息统计出错", exps);
}
}
public override void Run()
{
List <tb_datanode_model> nodeList = new List <tb_datanode_model>();
List <tb_partition_model> list = new List <tb_partition_model>();
using (DbConn conn = DbConfig.CreateConn(this.AppConfig["BusinessMQManageConnectString"]))
{
conn.Open();
nodeList = dataNodeDal.List(conn);
list = partitionDal.GetAllCanUsePartitionList(conn);
}
foreach (var item in nodeList)
{
string dataNode = PartitionRuleHelper.GetDataNodeName(item.datanodepartition);
string nodeConn = string.Format("server={0};Initial Catalog={1};User ID={2};Password={3};", item.serverip, dataNode, item.username, item.password);
try
{
using (DbConn dataNodeConn = DbConfig.CreateConn(nodeConn))
{
dataNodeConn.Open();
foreach (var value in list)
{
PartitionIDInfo info = PartitionRuleHelper.GetPartitionIDInfo(value.partitionid);
string partition = PartitionRuleHelper.PartitionNameRule(info.TablePartition);
ICollection <string> tableList = msgDal.GetTableNameListByPartition(dataNodeConn, partition);
if (tableList != null && tableList.Count > 0)
{
using (DbConn conn = DbConfig.CreateConn(this.AppConfig["BusinessMQManageConnectString"]))
{
foreach (var table in tableList)
{
if (!dataNodeConn.TableIsExist(table))
{
continue;
}
long maxId = msgDal.GetMaxID(dataNodeConn, table);
long minId = msgDal.GetMinID(dataNodeConn, table);
tb_partition_messagequeue_report_model model = new tb_partition_messagequeue_report_model();
model.partitionid = value.partitionid;
if (maxId > 0 && minId > 0)
{
MQIDInfo mqInfo = PartitionRuleHelper.GetMQIDInfo(maxId);
model.day = mqInfo.Day;
model.lastupdatetime = DateTime.Now;
model.createtime = DateTime.Now;
model.mqmaxid = maxId;
model.mqminid = minId;
reportDal.AddReport(conn, model);
}
}
}
}
}
}
}
catch (Exception ex)
{
base.OpenOperator.Error("节点不存在", ex);
continue;
}
}
}
private void ConsumeMessages(List <tb_messagequeue_model> messages, tb_consumer_partition_model partition, MQIDInfo mqidinfo)
{
long maxmqid = -1;
List <MQMessage> mqmessages = new List <MQMessage>();
foreach (var m in messages)
{
//若是第二天的数据或已迁移的数据则跳过
if ((m.sqlcreatetime < mqidinfo.Day.Date.AddDays(1)) && (m.state == (byte)EnumMessageState.CanRead))
{
mqmessages.Add(new MQMessage()
{
Model = m, Context = Context
});
}
maxmqid = m.id;
}
if (mqmessages.Count > 0)
{
Queue.Enqueue(partition.partitionid, mqmessages);
}
if (maxmqid > 0)
{
if (_lastpullququeiddic[partition.partitionid] <= maxmqid && !cancelSource.IsCancellationRequested)
{
_lastpullququeiddic[partition.partitionid] = maxmqid;
}
else
{
throw new BusinessMQException(string.Format("检测到消费者端拉去消息时出现消息乱序问题,partitionid:{0},maxmqid:{1},缓存maxmqid:{2}", partition.partitionid, maxmqid, _lastpullququeiddic[partition.partitionid]));
}
}
}
private List <tb_messagequeue_model> GetMessagesOfPatition(tb_consumer_partition_model partition, long scanlastmqid, MQIDInfo mqidinfo, string datanodeconnectstring, PartitionIDInfo partionidinfo)
{
//从数据库取数据,检查数据时间，和数据状态，无问题则插入队列，并修改上一次扫描的mqid
List <tb_messagequeue_model> messages = new List <tb_messagequeue_model>();
SqlHelper.ExcuteSql(datanodeconnectstring, (c) =>
{
string tablename = PartitionRuleHelper.GetTableName(partionidinfo.TablePartition, mqidinfo.Day);
if (exsittablenames.ContainsKey(tablename) || c.TableIsExist(tablename))
{
AddExsitTableNameCache(tablename);
tb_messagequeue_dal dal = new tb_messagequeue_dal(); dal.TableName = tablename;
messages = dal.GetMessages(c, scanlastmqid, SystemParamConfig.Consumer_ReceiveMessageQuque_EVERY_PULL_COUNT);
}
});
return(messages);
}
