public AppMinuteStat GetOrAdd(DateTime date, Int32 appId, out String key)
{
var model = new TraceStatModel {
Time = date, AppId = appId
};
key = model.Key;
return(GetOrAdd(key, k => AppMinuteStat.FindOrAdd(model)));
}
private static String GetMarkdown(AppTracer app, AppMinuteStat st, Boolean includeTitle)
{
var sb = new StringBuilder();
if (includeTitle)
{
sb.AppendLine($"### [{app}]系统告警");
}
sb.AppendLine($">**总数：**<font color=\"red\">{st.Errors}</font>");
var url = Setting.Current.WebUrl;
var appUrl = "";
var traceUrl = "";
if (!url.IsNullOrEmpty())
{
appUrl = url.EnsureEnd("/") + "Monitors/appMinuteStat?appId=" + st.AppId + "&minError=1";
traceUrl = url.EnsureEnd("/") + "Monitors/traceMinuteStat?appId=" + st.AppId + "&minError=1";
}
// 找找具体接口错误
var names = new List <String>();
var sts = TraceMinuteStat.FindAllByAppIdAndTime(st.AppId, st.StatTime).OrderByDescending(e => e.Errors).ToList();
foreach (var item in sts)
{
if (item.Errors > 0)
{
sb.AppendLine($">**错误：**<font color=\"red\">{item.StatTime.ToFullString()} 埋点[{item.Name}]共报错[{item.Errors:n0}]次</font>[更多]({traceUrl}&name={HttpUtility.UrlEncode(item.Name)})");
// 相同接口的错误，不要报多次
if (!names.Contains(item.Name))
{
var ds = TraceData.Search(st.AppId, item.Name, "minute", item.StatTime, 20);
if (ds.Count > 0)
{
var sms = SampleData.FindAllByDataIds(ds.Select(e => e.Id).ToArray(), item.StatTime).Where(e => !e.Error.IsNullOrEmpty()).ToList();
if (sms.Count > 0)
{
var msg = sms[0].Error?.Trim();
if (!msg.IsNullOrEmpty())
{
// 错误内容取第一行，详情看更多
var p = msg.IndexOfAny(new[] { '\r', '\n' });
if (p > 0)
{
msg = msg[..p];
private void ProcessAppTracer(AppTracer app)
{
// 应用是否需要告警
if (app == null || !app.Enable || app.AlarmThreshold <= 0)
{
return;
}
var appId = app.ID;
if (!RobotHelper.CanAlarm(app.Category, app.AlarmRobot))
{
return;
}
using var span = _tracer?.NewSpan($"Alarm:{nameof(AppTracer)}");
// 最近一段时间的5分钟级数据
var time = DateTime.Now;
var minute = time.Date.AddHours(time.Hour).AddMinutes(time.Minute / 5 * 5);
var st = AppMinuteStat.FindByAppIdAndTime(appId, minute);
if (st == null)
{
return;
}
// 判断告警
if (st.Errors >= app.AlarmThreshold)
{
// 一定时间内不要重复报错，除非错误翻倍
var error2 = _cache.Get <Int32>("alarm:AppTracer:" + appId);
if (error2 == 0 || st.Errors > error2 * 2)
{
_cache.Set("alarm:AppTracer:" + appId, st.Errors, 5 * 60);
var msg = GetMarkdown(app, st, true);
RobotHelper.SendAlarm(app.Category, app.AlarmRobot, "系统告警", msg);
}
}
}
