Beispiel #1
0
        private void ProcessRedisQueue(RedisNode node)
        {
            using var span = _tracer?.NewSpan($"Alarm:{nameof(RedisMessageQueue)}");

            // 所有队列
            var list = RedisMessageQueue.FindAllByRedisId(node.Id);

            foreach (var queue in list)
            {
                var groupName = !queue.Category.IsNullOrEmpty() ? queue.Category : node.Category;
                var webhook   = !queue.WebHook.IsNullOrEmpty() ? queue.WebHook : node.WebHook;

                // 判断告警
                if (queue.Enable && queue.MaxMessages > 0 && queue.Messages >= queue.MaxMessages && RobotHelper.CanAlarm(groupName, webhook))
                {
                    // 一定时间内不要重复报错,除非错误翻倍
                    var error2 = _cache.Get <Int32>("alarm:RedisMessageQueue:" + queue.Id);
                    if (error2 == 0 || queue.Messages > error2 * 2)
                    {
                        _cache.Set("alarm:RedisMessageQueue:" + queue.Id, queue.Messages, 5 * 60);

                        var msg = GetMarkdown(node, queue, true);
                        RobotHelper.SendAlarm(groupName, webhook, "消息队列告警", msg);
                    }
                }
            }
        }
Beispiel #2
0
        private void ProcessAppTracer(AppTracer app)
        {
            // 应用是否需要告警
            if (app == null || !app.Enable || app.AlarmThreshold <= 0)
            {
                return;
            }

            var appId = app.ID;

            if (!RobotHelper.CanAlarm(app.Category, app.AlarmRobot))
            {
                return;
            }

            using var span = _tracer?.NewSpan($"Alarm:{nameof(AppTracer)}");

            // 最近一段时间的5分钟级数据
            var time   = DateTime.Now;
            var minute = time.Date.AddHours(time.Hour).AddMinutes(time.Minute / 5 * 5);
            var st     = AppMinuteStat.FindByAppIdAndTime(appId, minute);

            if (st == null)
            {
                return;
            }

            // 判断告警
            if (st.Errors >= app.AlarmThreshold)
            {
                // 一定时间内不要重复报错,除非错误翻倍
                var error2 = _cache.Get <Int32>("alarm:AppTracer:" + appId);
                if (error2 == 0 || st.Errors > error2 * 2)
                {
                    _cache.Set("alarm:AppTracer:" + appId, st.Errors, 5 * 60);

                    var msg = GetMarkdown(app, st, true);
                    RobotHelper.SendAlarm(app.Category, app.AlarmRobot, "系统告警", msg);
                }
            }
        }
Beispiel #3
0
        private void ProcessRedisData(RedisNode node)
        {
            if (!RobotHelper.CanAlarm(node.Category, node.WebHook))
            {
                return;
            }
            if (node.AlarmMemoryRate <= 0 || node.AlarmConnections == 0)
            {
                return;
            }

            // 最新数据
            var data = RedisData.FindLast(node.Id);

            if (data == null)
            {
                return;
            }

            using var span = _tracer?.NewSpan($"Alarm:{nameof(RedisNode)}");

            var actions = new List <Action <StringBuilder> >();

            // 内存告警
            var rate = data.UsedMemory * 100 / node.MaxMemory;

            if (rate >= node.AlarmMemoryRate)
            {
                // 一定时间内不要重复报错,除非错误翻倍
                var error2 = _cache.Get <Int32>("alarm:RedisMemory:" + node.Id);
                if (error2 == 0 || rate > error2 * 2)
                {
                    _cache.Set("alarm:RedisMemory:" + node.Id, rate, 5 * 60);

                    actions.Add(sb => sb.AppendLine($">**内存告警:**<font color=\"info\">{rate / 100:p0} >= {node.AlarmMemoryRate / 100:p0}</font>"));
                }
            }

            // 连接数告警
            var cs = data.ConnectedClients;

            if (node.AlarmConnections > 0 && cs >= node.AlarmConnections)
            {
                // 一定时间内不要重复报错,除非错误翻倍
                var error2 = _cache.Get <Int32>("alarm:RedisConnections:" + node.Id);
                if (error2 == 0 || cs > error2 * 2)
                {
                    _cache.Set("alarm:RedisConnections:" + node.Id, cs, 5 * 60);

                    actions.Add(sb => sb.AppendLine($">**连接数告警:**<font color=\"info\">{cs:n0} >= {node.AlarmConnections:n0}</font>"));
                }
            }

            // 速度告警
            var speed = data.Speed;

            if (node.AlarmSpeed > 0 && speed >= node.AlarmSpeed)
            {
                // 一定时间内不要重复报错,除非错误翻倍
                var error2 = _cache.Get <Int32>("alarm:RedisSpeed:" + node.Id);
                if (error2 == 0 || speed > error2 * 2)
                {
                    _cache.Set("alarm:RedisSpeed:" + node.Id, speed, 5 * 60);

                    actions.Add(sb => sb.AppendLine($">**速度告警:**<font color=\"info\">{speed:n0} >= {node.AlarmSpeed:n0}</font>"));
                }
            }

            // 入流量告警
            var input = data.InputKbps;

            if (node.AlarmInputKbps > 0 && input >= node.AlarmInputKbps)
            {
                // 一定时间内不要重复报错,除非错误翻倍
                var error2 = _cache.Get <Int32>("alarm:RedisInputKbps:" + node.Id);
                if (error2 == 0 || input > error2 * 2)
                {
                    _cache.Set("alarm:RedisInputKbps:" + node.Id, input, 5 * 60);

                    actions.Add(sb => sb.AppendLine($">**入流量告警:**<font color=\"info\">{input:n0} >= {node.AlarmInputKbps:n0}</font>"));
                }
            }

            // 出流量告警
            var output = data.OutputKbps;

            if (node.AlarmOutputKbps > 0 && output >= node.AlarmOutputKbps)
            {
                // 一定时间内不要重复报错,除非错误翻倍
                var error2 = _cache.Get <Int32>("alarm:RedisOutputKbps:" + node.Id);
                if (error2 == 0 || output > error2 * 2)
                {
                    _cache.Set("alarm:RedisOutputKbps:" + node.Id, output, 5 * 60);

                    actions.Add(sb => sb.AppendLine($">**出流量告警:**<font color=\"info\">{output:n0} >= {node.AlarmOutputKbps:n0}</font>"));
                }
            }

            if (actions.Count > 0)
            {
                var msg = GetMarkdown(node, data, "Redis告警", actions);
                RobotHelper.SendAlarm(node.Category, node.WebHook, "Redis告警", msg);
            }
        }
Beispiel #4
0
        private void SendAlarm(String kind, Node node, NodeData data, String title, String info = null)
        {
            var msg = GetMarkdown(kind, node, data, title, info);

            RobotHelper.SendAlarm(node.Category, node.WebHook, title, msg);
        }