private long StartMoveMQID(string fromtablename, PartitionIDInfo info, Dictionary <int, tb_datanode_model> datanodemodels, tb_partition_model p, DateTime tabletime)
{
long r = -1;
SqlHelper.ExcuteSql(this.GetDataNodeConnectString(datanodemodels[info.DataNodePartition]), (c) =>
{
bool exsit = c.TableIsExist(fromtablename);
if (exsit == true)
{
tb_messagequeue_dal dal = new tb_messagequeue_dal(); dal.TableName = fromtablename;
long maxid = dal.GetMaxID(c); long minid = dal.GetMinID(c);
int findcount = 0;
while (findcount <= 30)//二分查找算法,超过30次意味着算法若对，查询过一亿的数据，说明算法有问题
{
findcount++;
KeyValuePair <long, DateTime> mininfo = dal.GetIDCreateTime(c, minid); KeyValuePair <long, DateTime> maxinfo = dal.GetIDCreateTime(c, maxid);
if (mininfo.Value.Date <= tabletime.Date && maxinfo.Value.Date > tabletime.Date)//最小id小于等于当天，最大id>当天，说明中间有数据要迁移
{
var midid = (long)((maxinfo.Key + mininfo.Key) / 2); KeyValuePair <long, DateTime> midinfo = dal.GetIDCreateTime(c, midid);
if (midinfo.Value.Date <= tabletime.Date)//往大的查找
{
minid = midid;
}
else//往小的查找
{
maxid = minid;
}
System.Threading.Thread.Sleep(50); //避免给数据库大的压力
}
else if (mininfo.Value.Date > tabletime.Date) //最小id的时间超过表当天
{
r = mininfo.Key; return;
}
else if (maxinfo.Value.Date <= tabletime.Date)//最大id的时间小于或等同当天
{
r = maxinfo.Key; return;
}
}
if (findcount >= 30)
{
throw new Exception("二分查找算法错误，查找次数超过30次");
}
}
});
return(r);
}
public IList <ConsumerModel> GetPageList(DbConn conn, string name, int pageIndex, int pageSize, ref int count)
{
int tempCount = 0;
IList <ConsumerModel> list = new List <ConsumerModel>();
ConsumerModel cm = new ConsumerModel();
var result = SqlHelper.Visit((ps) =>
{
StringBuilder where = new StringBuilder(" WHERE 1=1 ");
if (!string.IsNullOrEmpty(name))
{
where.AppendFormat(" AND clientname LIKE '%{0}%'", name);
}
string sql = "SELECT ROW_NUMBER() OVER(ORDER BY Id DESC) AS rownum,* FROM tb_consumer WITH(NOLOCK)";
string countSql = "SELECT COUNT(1) FROM tb_consumer WITH(NOLOCK) " + where.ToString();
object obj = conn.ExecuteScalar(countSql, null);
if (obj != DBNull.Value && obj != null)
{
tempCount = LibConvert.ObjToInt(obj);
}
string sqlPage = string.Concat("SELECT * FROM (", sql.ToString(), where.ToString(), ") A WHERE rownum BETWEEN ", ((pageIndex - 1) * pageSize + 1), " AND ", pageSize * pageIndex);
DataTable dt = conn.SqlToDataTable(sqlPage, null);
if (dt != null && dt.Rows.Count > 0)
{
foreach (DataRow dr in dt.Rows)
{
ConsumerModel model = cm.CreateModel(dr);
IList <tb_consumer_partition_model> consumerList = partitionDal.GetPartitionByConsumerId(conn, model.consumerclientid);
if (consumerList != null && consumerList.Count > 0)
{
IList <ConsumerPartition> partitionList = new List <ConsumerPartition>();
foreach (var item in consumerList)
{
ConsumerPartition m = new ConsumerPartition();
m.PartitionId = item.partitionid;
PartitionIDInfo partitionInfo = PartitionRuleHelper.GetPartitionIDInfo(item.partitionid);
string node = string.Empty;
if (partitionInfo.DataNodePartition < 10)
{
node = "0" + partitionInfo.DataNodePartition.Tostring();
}
else
{
node = partitionInfo.DataNodePartition.Tostring();
}
using (DbConn nodeConn = DbConfig.CreateConn(DataConfig.DataNodeParConn(node)))
{
nodeConn.Open();
tb_partition_model partitionModel = new tb_partition_dal().Get(conn, item.partitionid);
if (partitionModel != null)
{
m.IsOnline = partitionModel.isused;
}
string table = msgDal.GetMaxMqTable(nodeConn, node);
m.Msg = msgDal.GetMsgCount(nodeConn, table, 0);
m.NonMsg = msgDal.GetMsgCount(nodeConn, table, 1);
partitionList.Add(m);
}
}
model.PartitionList = partitionList;
}
list.Add(model);
}
}
return(list);
});
count = tempCount;
return(result);
}
private void CheckIfScanNewTable(tb_consumer_partition_model partition, MQIDInfo mqidinfo, string datanodeconnectstring, PartitionIDInfo partionidinfo)
{
var serverdate = Context.ManageServerTime.AddSeconds(-1); //此处延迟1秒，停顿消费者日分区表之间的切换
if (serverdate != null && serverdate.Date > mqidinfo.Day.Date) //服务器时间和消息时间对应不上
{
SqlHelper.ExcuteSql(datanodeconnectstring, (c) =>
{
DateTime t = mqidinfo.Day.Date.AddDays(1);
while (serverdate.Date >= t) //查找最后的可用消息表,然后跳出
{
string tablename = PartitionRuleHelper.GetTableName(partionidinfo.TablePartition, t);
if (exsittablenames.ContainsKey(tablename) || c.TableIsExist(tablename))
{
AddExsitTableNameCache(tablename);
_lastpullququeiddic[partition.partitionid] = PartitionRuleHelper.GetMQID(new MQIDInfo()
{
AutoID = 0,
DataNodePartition = mqidinfo.DataNodePartition,
TablePartition = mqidinfo.TablePartition,
Day = t
}); //重设第二天最小的id
break;
}
t = t.AddDays(1);
}
});
}
}
public override void Run()
{
List <tb_datanode_model> nodeList = new List <tb_datanode_model>();
List <tb_partition_model> list = new List <tb_partition_model>();
using (DbConn conn = DbConfig.CreateConn(this.AppConfig["BusinessMQManageConnectString"]))
{
conn.Open();
nodeList = dataNodeDal.List(conn);
list = partitionDal.GetAllCanUsePartitionList(conn);
}
foreach (var item in nodeList)
{
string dataNode = PartitionRuleHelper.GetDataNodeName(item.datanodepartition);
string nodeConn = string.Format("server={0};Initial Catalog={1};User ID={2};Password={3};", item.serverip, dataNode, item.username, item.password);
try
{
using (DbConn dataNodeConn = DbConfig.CreateConn(nodeConn))
{
dataNodeConn.Open();
foreach (var value in list)
{
PartitionIDInfo info = PartitionRuleHelper.GetPartitionIDInfo(value.partitionid);
string partition = PartitionRuleHelper.PartitionNameRule(info.TablePartition);
ICollection <string> tableList = msgDal.GetTableNameListByPartition(dataNodeConn, partition);
if (tableList != null && tableList.Count > 0)
{
using (DbConn conn = DbConfig.CreateConn(this.AppConfig["BusinessMQManageConnectString"]))
{
foreach (var table in tableList)
{
if (!dataNodeConn.TableIsExist(table))
{
continue;
}
long maxId = msgDal.GetMaxID(dataNodeConn, table);
long minId = msgDal.GetMinID(dataNodeConn, table);
tb_partition_messagequeue_report_model model = new tb_partition_messagequeue_report_model();
model.partitionid = value.partitionid;
if (maxId > 0 && minId > 0)
{
MQIDInfo mqInfo = PartitionRuleHelper.GetMQIDInfo(maxId);
model.day = mqInfo.Day;
model.lastupdatetime = DateTime.Now;
model.createtime = DateTime.Now;
model.mqmaxid = maxId;
model.mqminid = minId;
reportDal.AddReport(conn, model);
}
}
}
}
}
}
}
catch (Exception ex)
{
base.OpenOperator.Error("节点不存在", ex);
continue;
}
}
}
private List <tb_messagequeue_model> GetMessagesOfPatition(tb_consumer_partition_model partition, long scanlastmqid, MQIDInfo mqidinfo, string datanodeconnectstring, PartitionIDInfo partionidinfo)
{
//从数据库取数据,检查数据时间，和数据状态，无问题则插入队列，并修改上一次扫描的mqid
List <tb_messagequeue_model> messages = new List <tb_messagequeue_model>();
SqlHelper.ExcuteSql(datanodeconnectstring, (c) =>
{
string tablename = PartitionRuleHelper.GetTableName(partionidinfo.TablePartition, mqidinfo.Day);
if (exsittablenames.ContainsKey(tablename) || c.TableIsExist(tablename))
{
AddExsitTableNameCache(tablename);
tb_messagequeue_dal dal = new tb_messagequeue_dal(); dal.TableName = tablename;
messages = dal.GetMessages(c, scanlastmqid, SystemParamConfig.Consumer_ReceiveMessageQuque_EVERY_PULL_COUNT);
}
});
return(messages);
}
