private void CheckIfScanNewTable(tb_consumer_partition_model partition, MQIDInfo mqidinfo, string datanodeconnectstring, PartitionIDInfo partionidinfo) { var serverdate = Context.ManageServerTime.AddSeconds(-1);//此处延迟1秒,停顿消费者日分区表之间的切换 if (serverdate != null && serverdate.Date > mqidinfo.Day.Date)//服务器时间和消息时间对应不上 { SqlHelper.ExcuteSql(datanodeconnectstring, (c) => { DateTime t = mqidinfo.Day.Date.AddDays(1); while (serverdate.Date >= t)//查找最后的可用消息表,然后跳出 { string tablename = PartitionRuleHelper.GetTableName(partionidinfo.TablePartition, t); if (exsittablenames.ContainsKey(tablename) || c.TableIsExist(tablename)) { AddExsitTableNameCache(tablename); _lastpullququeiddic[partition.partitionid] = PartitionRuleHelper.GetMQID(new MQIDInfo() { AutoID = 0, DataNodePartition = mqidinfo.DataNodePartition, TablePartition = mqidinfo.TablePartition, Day = t });//重设第二天最小的id break; } t = t.AddDays(1); } }); } }
/// <summary> /// 获取MQID隐藏的信息 /// </summary> /// <param name="id"></param> /// <returns></returns> public static MQIDInfo GetMQIDInfo(long id) { MQIDInfo info = new MQIDInfo(); string example = "1010115062900000000"; string strid = id.ToString(); if (strid.Length != example.Length) { throw new Exception("消息Id格式不正确:" + id); } info.DataNodePartition = Convert.ToInt32(strid.Substring(1, 2)); info.TablePartition = Convert.ToInt32(strid.Substring(3, 2)); info.Day = DateTime.ParseExact(strid.Substring(5, 6), "yyMMdd", System.Globalization.CultureInfo.CurrentCulture); info.AutoID = Convert.ToInt32(strid.Substring(11, 8)); return(info); }
private void ConsumeMessages(List<tb_messagequeue_model> messages, tb_consumer_partition_model partition, MQIDInfo mqidinfo) { long maxmqid = -1; List<MQMessage> mqmessages = new List<MQMessage>(); foreach (var m in messages) { //若是第二天的数据或已迁移的数据则跳过 if ((m.sqlcreatetime < mqidinfo.Day.Date.AddDays(1)) && (m.state == (byte)EnumMessageState.CanRead)) { mqmessages.Add(new MQMessage() { Model = m, Context = Context }); } maxmqid = m.id; } if (mqmessages.Count > 0) { Queue.Enqueue(partition.partitionid, mqmessages); } if (maxmqid > 0) { if (_lastpullququeiddic[partition.partitionid] <= maxmqid && !cancelSource.IsCancellationRequested) _lastpullququeiddic[partition.partitionid] = maxmqid; else throw new BusinessMQException(string.Format("检测到消费者端拉去消息时出现消息乱序问题,partitionid:{0},maxmqid:{1},缓存maxmqid:{2}", partition.partitionid, maxmqid, _lastpullququeiddic[partition.partitionid])); } }
private List<tb_messagequeue_model> GetMessagesOfPatition(tb_consumer_partition_model partition, long scanlastmqid, MQIDInfo mqidinfo, string datanodeconnectstring, PartitionIDInfo partionidinfo) { //从数据库取数据,检查数据时间,和数据状态,无问题则插入队列,并修改上一次扫描的mqid List<tb_messagequeue_model> messages = new List<tb_messagequeue_model>(); SqlHelper.ExcuteSql(datanodeconnectstring, (c) => { string tablename = PartitionRuleHelper.GetTableName(partionidinfo.TablePartition, mqidinfo.Day); if (exsittablenames.ContainsKey(tablename) || c.TableIsExist(tablename)) { AddExsitTableNameCache(tablename); tb_messagequeue_dal dal = new tb_messagequeue_dal(); dal.TableName = tablename; messages = dal.GetMessages(c, scanlastmqid, SystemParamConfig.Consumer_ReceiveMessageQuque_EVERY_PULL_COUNT); } }); return messages; }
/// <summary> /// 获取MQID隐藏的信息 /// </summary> /// <param name="id"></param> /// <returns></returns> public static MQIDInfo GetMQIDInfo(long id) { MQIDInfo info = new MQIDInfo();string example = "1010115062900000000";string strid = id.ToString(); if(strid.Length!=example.Length) throw new Exception("消息Id格式不正确:"+id); info.DataNodePartition = Convert.ToInt32( strid.Substring(1,2)); info.TablePartition = Convert.ToInt32(strid.Substring(3,2)); info.Day = DateTime.ParseExact(strid.Substring(5, 6), "yyMMdd", System.Globalization.CultureInfo.CurrentCulture); info.AutoID = Convert.ToInt32(strid.Substring(11,8)); return info; }
/// <summary> /// 获取MQ消息ID 消息id号,规则1+数据节点编号+表分区编号+时间分区号+自增id /// </summary> /// <param name="info"></param> /// <returns></returns> public static long GetMQID(MQIDInfo info) { string id = "1" + PartitionNameRule(info.DataNodePartition) + PartitionNameRule(info.TablePartition) + info.Day.ToString("yyMMdd") + info.AutoID.ToString().PadLeft(8, '0'); return Convert.ToInt64(id); }
/// <summary> /// 获取MQ消息ID 消息id号,规则1+数据节点编号+表分区编号+时间分区号+自增id /// </summary> /// <param name="info"></param> /// <returns></returns> public static long GetMQID(MQIDInfo info) { string id = "1" + PartitionNameRule(info.DataNodePartition) + PartitionNameRule(info.TablePartition) + info.Day.ToString("yyMMdd") + info.AutoID.ToString().PadLeft(8, '0'); return(Convert.ToInt64(id)); }