private void ProcessRedisQueue(RedisNode node)
{
using var span = _tracer?.NewSpan($"Alarm:{nameof(RedisMessageQueue)}");
// 所有队列
var list = RedisMessageQueue.FindAllByRedisId(node.Id);
foreach (var queue in list)
{
var groupName = !queue.Category.IsNullOrEmpty() ? queue.Category : node.Category;
var webhook = !queue.WebHook.IsNullOrEmpty() ? queue.WebHook : node.WebHook;
// 判断告警
if (queue.Enable && queue.MaxMessages > 0 && queue.Messages >= queue.MaxMessages && RobotHelper.CanAlarm(groupName, webhook))
{
// 一定时间内不要重复报错，除非错误翻倍
var error2 = _cache.Get <Int32>("alarm:RedisMessageQueue:" + queue.Id);
if (error2 == 0 || queue.Messages > error2 * 2)
{
_cache.Set("alarm:RedisMessageQueue:" + queue.Id, queue.Messages, 5 * 60);
var msg = GetMarkdown(node, queue, true);
RobotHelper.SendAlarm(groupName, webhook, "消息队列告警", msg);
}
}
}
}
public void TraceNode(RedisNode node)
{
if (!_servers.TryGetValue(node.Id, out var rds))
{
_servers[node.Id] = rds = new FullRedis();
}
// 可能后面更新了服务器地址和密码
rds.Server = node.Server;
rds.Password = node.Password;
rds.Tracer = DefaultTracer.Instance;
//var inf = rds.GetInfo(true);
var inf = rds.GetInfo(false);
node.Fill(inf);
node.SaveAsync();
var data = new RedisData
{
RedisId = node.Id,
Name = node.Name,
};
var dbs = data.Fill(inf);
data.Insert();
// 扫描队列
if (node.ScanQueue && dbs != null)
{
ScanQueue(node, dbs);
}
}
private static String GetMarkdown(RedisNode node, RedisMessageQueue queue, Boolean includeTitle)
{
var sb = new StringBuilder();
if (includeTitle)
{
sb.AppendLine($"### [{queue.Name}/{node}]消息队列告警");
}
sb.AppendLine($">**主题：**<font color=\"info\">{queue.Topic}</font>");
sb.AppendLine($">**积压：**<font color=\"info\">{queue.Messages:n0} > {queue.MaxMessages:n0}</font>");
sb.AppendLine($">**消费者：**<font color=\"info\">{queue.Consumers}</font>");
sb.AppendLine($">**总消费：**<font color=\"info\">{queue.Total:n0}</font>");
sb.AppendLine($">**服务器：**<font color=\"info\">{node.Server}</font>");
var str = sb.ToString();
if (str.Length > 2000)
{
str = str.Substring(0, 2000);
}
// 构造网址
var url = Setting.Current.WebUrl;
if (!url.IsNullOrEmpty())
{
url = url.EnsureEnd("/") + "Nodes/RedisMessageQueue?redisId=" + queue.RedisId + "&q=" + queue.Name;
str += Environment.NewLine + $"[更多信息]({url})";
}
return(str);
}
private void ScanQueue(RedisNode node, RedisDbEntry[] dbs)
{
var queues = RedisMessageQueue.FindAllByRedisId(node.Id);
for (var i = 0; i < dbs.Length; i++)
{
if (dbs[i] == null)
{
continue;
}
var rds2 = GetOrAdd(node, i);
// keys个数太大不支持扫描
if (rds2.Count < 10000)
{
foreach (var item in rds2.Search("*:Status:*", 1000))
{
var ss = item.Split(":");
var topic = ss.Take(ss.Length - 2).Join(":");
if (topic.IsNullOrEmpty())
{
continue;
}
var mq = queues.FirstOrDefault(e => e.Db == i && e.Topic == topic);
if (mq == null)
{
mq = new RedisMessageQueue
{
RedisId = node.Id,
Db = i,
Topic = topic,
Enable = true,
};
queues.Add(mq);
}
mq.Enable = true;
if (mq.Name.IsNullOrEmpty())
{
mq.Name = node.Name;
}
if (mq.Category.IsNullOrEmpty())
{
mq.Category = node.Category;
}
if (mq.Type.IsNullOrEmpty())
{
mq.Type = topic.EndsWithIgnoreCase(":Delay") ? "Delay" : "Queue";
}
mq.SaveAsync();
}
}
}
}
protected override IEnumerable <RedisData> Search(Pager p)
{
PageSetting.EnableAdd = false;
var redisId = p["redisId"].ToInt(-1);
var start = p["dtStart"].ToDateTime();
var end = p["dtEnd"].ToDateTime();
if (redisId > 0)
{
// 最近10小时
if (p.PageSize == 20 && redisId > 0)
{
p.PageSize = 24 * 60;
}
PageSetting.EnableNavbar = false;
}
if (p.Sort.IsNullOrEmpty())
{
p.OrderBy = _.Id.Desc();
}
var list = RedisData.Search(redisId, start, end, p["Q"], p);
if (list.Count > 0)
{
// 绘制日期曲线图
var node = RedisNode.FindById(redisId);
if (redisId >= 0 && node != null)
{
var list2 = list.OrderBy(e => e.Id).ToList();
var chart = new ECharts
{
Title = new ChartTitle {
Text = node.Name
},
Height = 400,
};
chart.SetX(list2, _.CreateTime, e => e.CreateTime.ToString("HH:mm"));
chart.SetY("指标");
chart.AddLine(list2, _.Speed, null, true);
chart.Add(list2, _.InputKbps);
chart.Add(list2, _.OutputKbps);
chart.Add(list2, _.ConnectedClients);
chart.Add(list2, _.UsedMemory);
chart.Add(list2, _.Keys);
chart.SetTooltip();
ViewBag.Charts = new[] { chart };
}
}
return(list);
}
private void ProcessRedisNode(RedisNode node)
{
if (node == null || !node.Enable || node.WebHook.IsNullOrEmpty())
{
return;
}
ProcessRedisData(node);
ProcessRedisQueue(node);
}
/// <summary>
/// Returns the node for the given id.
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public override Node GetNode(long id)
{
string nodeKey = PrimitiveExtensions.BuildNodeRedisKey(id);
RedisNode redisNode = _clientNode.GetValue(nodeKey);
Node node = null;
if (redisNode != null)
{
node = PrimitiveExtensions.ConvertFrom(redisNode);
}
return(node);
}
//public event Action<PooledSocket> SocketConnected;
/// <summary>
/// This will start the pool: initializes the nodelocator, warms up the socket pools, etc.
/// </summary>
public void Start()
{
// initialize the server list
foreach (IEndPointConfiguration ip in _configuration.Servers)
{
var node = new RedisNode(ip, _configuration.SocketPool, Authenticator);
_workingServers.Add(node);
}
// initializes the locator
RebuildIndexes();
}
private FullRedis GetOrAdd(RedisNode node, Int32 db)
{
var key = $"{node.Id}-{db}";
if (!_servers2.TryGetValue(key, out var rds))
{
_servers2[key] = rds = new FullRedis();
}
rds.Server = node.Server;
rds.Password = node.Password;
rds.Db = db;
return(rds);
}
private static String GetMarkdown(RedisNode node, RedisData data, String title, IList <Action <StringBuilder> > actions)
{
var sb = new StringBuilder();
if (!title.IsNullOrEmpty())
{
sb.AppendLine($"### [{node}]{title}");
}
sb.AppendLine($">**分类：**<font color=\"info\">{node.Category}</font>");
sb.AppendLine($">**版本：**<font color=\"info\">{node.Version}</font>");
sb.AppendLine($">**已用内存：**<font color=\"info\">{data.UsedMemory:n0}</font>");
sb.AppendLine($">**内存容量：**<font color=\"info\">{node.MaxMemory:n0}</font>");
sb.AppendLine($">**连接数：**<font color=\"info\">{data.ConnectedClients:n0}</font>");
sb.AppendLine($">**服务器：**<font color=\"info\">{node.Server}</font>");
//var rate = node.MaxMemory == 0 ? 0 : (data.UsedMemory * 100 / node.MaxMemory);
//if (rate >= node.AlarmMemoryRate && node.AlarmMemoryRate > 0)
//{
// sb.AppendLine($">**内存告警：**<font color=\"info\">{data.UsedMemory}/{node.MaxMemory} >= {node.AlarmMemoryRate:p0}</font>");
//}
//if (node.AlarmConnections > 0 && data.ConnectedClients >= node.AlarmConnections)
//{
// sb.AppendLine($">**连接告警：**<font color=\"info\">{data.ConnectedClients:n0} >= {node.AlarmConnections:n0}</font>");
//}
foreach (var item in actions)
{
item(sb);
}
var str = sb.ToString();
if (str.Length > 2000)
{
str = str.Substring(0, 2000);
}
// 构造网址
var url = Setting.Current.WebUrl;
if (!url.IsNullOrEmpty())
{
url = url.EnsureEnd("/") + "Nodes/RedisNode?id=" + node.Id;
str += Environment.NewLine + $"[更多信息]({url})";
}
return(str);
}
/// <summary>
/// Converts the given node to a redis node.
/// </summary>
/// <param name="node"></param>
/// <returns></returns>
public static RedisNode ConvertTo(Node node)
{
RedisNode redisNode = new RedisNode();
redisNode.Id = node.Id.Value;
redisNode.Latitude = node.Latitude.Value;
redisNode.Longitude = node.Longitude.Value;
redisNode.ChangeSetId = node.ChangeSetId;
redisNode.TimeStamp = node.TimeStamp;
redisNode.UserId = node.UserId;
redisNode.UserName = node.UserName;
redisNode.Version = node.Version;
redisNode.Visible = node.Visible;
redisNode.Tags = PrimitiveExtensions.ConvertTo(node.Tags);
return(redisNode);
}
/// <summary>
/// Converts the given node from a redis node.
/// </summary>
/// <param name="redisNode"></param>
/// <returns></returns>
public static Node ConvertFrom(RedisNode redisNode)
{
Node node = new Node();
node.Id = redisNode.Id.Value;
node.Latitude = redisNode.Latitude.Value;
node.Longitude = redisNode.Longitude.Value;
node.ChangeSetId = redisNode.ChangeSetId;
node.TimeStamp = redisNode.TimeStamp;
node.UserId = redisNode.UserId;
node.UserName = redisNode.UserName;
node.Version = redisNode.Version;
node.Visible = redisNode.Visible;
node.Tags = PrimitiveExtensions.ConvertFrom(redisNode.Tags);
return(node);
}
private void DoTraceNode(Object state)
{
var list = RedisNode.FindAllWithCache();
foreach (var item in list)
{
if (item.Enable)
{
// 捕获异常，不要影响后续操作
var key = $"DoTraceNode:{item.Id}";
var errors = _cache.Get <Int64>(key);
if (errors < 5)
{
try
{
TraceNode(item);
_cache.Remove(key);
}
catch (Exception ex)
{
errors = _cache.Increment(key, 1);
if (errors <= 1)
{
_cache.SetExpire(key, TimeSpan.FromMinutes(10));
}
XTrace.WriteException(ex);
}
}
else
{
item.Enable = false;
item.SaveAsync();
_cache.Remove(key);
}
}
}
}
private void DoAlarm(Object state)
{
while (_bag.TryTake(out var appId))
{
//Process(appId);
}
// 应用告警
var list = AppTracer.FindAllWithCache();
foreach (var item in list)
{
ProcessAppTracer(item);
}
// 节点告警
var nodes = Node.FindAllWithCache();
foreach (var item in nodes)
{
ProcessNode(item);
}
// Redis告警
var rnodes = RedisNode.FindAllWithCache();
foreach (var item in rnodes)
{
ProcessRedisNode(item);
}
if (Period > 0)
{
_timer.Period = Period * 1000;
}
}
/// <summary>
/// Reads an osmGeo object from disk.
/// </summary>
/// <param name="id"></param>
/// <param name="type"></param>
private OsmGeo Read(long id, OsmGeoType type)
{
switch (type)
{
case OsmGeoType.Node:
string nodeKey = PrimitiveExtensions.BuildNodeRedisKey(id);
RedisNode redisNode = _clientNode.GetValue(nodeKey);
Node node = null;
if (redisNode != null)
{
node = PrimitiveExtensions.ConvertFrom(redisNode);
}
return(node);
case OsmGeoType.Way:
string wayKey = PrimitiveExtensions.BuildWayRedisKey(id);
RedisWay redisWay = _clientWay.GetValue(wayKey);
Way way = null;
if (redisWay != null)
{
way = PrimitiveExtensions.ConvertFrom(redisWay);
}
return(way);
case OsmGeoType.Relation:
string relationKey = PrimitiveExtensions.BuildRelationRedisKey(id);
RedisRelation redisRelation = _clientRelation.GetValue(relationKey);
Relation relation = null;
if (redisRelation != null)
{
relation = PrimitiveExtensions.ConvertFrom(redisRelation);
}
return(relation);
}
throw new ArgumentOutOfRangeException("type");
}
private void ProcessRedisData(RedisNode node)
{
if (!RobotHelper.CanAlarm(node.Category, node.WebHook))
{
return;
}
if (node.AlarmMemoryRate <= 0 || node.AlarmConnections == 0)
{
return;
}
// 最新数据
var data = RedisData.FindLast(node.Id);
if (data == null)
{
return;
}
using var span = _tracer?.NewSpan($"Alarm:{nameof(RedisNode)}");
var actions = new List <Action <StringBuilder> >();
// 内存告警
var rate = data.UsedMemory * 100 / node.MaxMemory;
if (rate >= node.AlarmMemoryRate)
{
// 一定时间内不要重复报错，除非错误翻倍
var error2 = _cache.Get <Int32>("alarm:RedisMemory:" + node.Id);
if (error2 == 0 || rate > error2 * 2)
{
_cache.Set("alarm:RedisMemory:" + node.Id, rate, 5 * 60);
actions.Add(sb => sb.AppendLine($">**内存告警：**<font color=\"info\">{rate / 100:p0} >= {node.AlarmMemoryRate / 100:p0}</font>"));
}
}
// 连接数告警
var cs = data.ConnectedClients;
if (node.AlarmConnections > 0 && cs >= node.AlarmConnections)
{
// 一定时间内不要重复报错，除非错误翻倍
var error2 = _cache.Get <Int32>("alarm:RedisConnections:" + node.Id);
if (error2 == 0 || cs > error2 * 2)
{
_cache.Set("alarm:RedisConnections:" + node.Id, cs, 5 * 60);
actions.Add(sb => sb.AppendLine($">**连接数告警：**<font color=\"info\">{cs:n0} >= {node.AlarmConnections:n0}</font>"));
}
}
// 速度告警
var speed = data.Speed;
if (node.AlarmSpeed > 0 && speed >= node.AlarmSpeed)
{
// 一定时间内不要重复报错，除非错误翻倍
var error2 = _cache.Get <Int32>("alarm:RedisSpeed:" + node.Id);
if (error2 == 0 || speed > error2 * 2)
{
_cache.Set("alarm:RedisSpeed:" + node.Id, speed, 5 * 60);
actions.Add(sb => sb.AppendLine($">**速度告警：**<font color=\"info\">{speed:n0} >= {node.AlarmSpeed:n0}</font>"));
}
}
// 入流量告警
var input = data.InputKbps;
if (node.AlarmInputKbps > 0 && input >= node.AlarmInputKbps)
{
// 一定时间内不要重复报错，除非错误翻倍
var error2 = _cache.Get <Int32>("alarm:RedisInputKbps:" + node.Id);
if (error2 == 0 || input > error2 * 2)
{
_cache.Set("alarm:RedisInputKbps:" + node.Id, input, 5 * 60);
actions.Add(sb => sb.AppendLine($">**入流量告警：**<font color=\"info\">{input:n0} >= {node.AlarmInputKbps:n0}</font>"));
}
}
// 出流量告警
var output = data.OutputKbps;
if (node.AlarmOutputKbps > 0 && output >= node.AlarmOutputKbps)
{
// 一定时间内不要重复报错，除非错误翻倍
var error2 = _cache.Get <Int32>("alarm:RedisOutputKbps:" + node.Id);
if (error2 == 0 || output > error2 * 2)
{
_cache.Set("alarm:RedisOutputKbps:" + node.Id, output, 5 * 60);
actions.Add(sb => sb.AppendLine($">**出流量告警：**<font color=\"info\">{output:n0} >= {node.AlarmOutputKbps:n0}</font>"));
}
}
if (actions.Count > 0)
{
var msg = GetMarkdown(node, data, "Redis告警", actions);
RobotHelper.SendAlarm(node.Category, node.WebHook, "Redis告警", msg);
}
}
