private void Timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { var notOk = tracker.Statuses.Where(s => !s.IsOk); foreach (var service in tracker.Statuses) { { var lastState = LastStates.GetOrAdd(service.InstanceId, id => service.heartbeatState); if (lastState != service.heartbeatState) { heartbeatLog.Log(new HeartbeatTrackerLogItem(LogLevel.Information, "State Changed", $"{service.Key} state {lastState} => {service.heartbeatState}")); } LastStates.TryUpdate(service.InstanceId, service.heartbeatState, lastState); } { var lastHealth = LastHealth.GetOrAdd(service.InstanceId, id => service.HealthStatus); if (lastHealth != service.HealthStatus) { heartbeatLog.Log(new HeartbeatTrackerLogItem(LogLevel.Information, "Health Changed", $"{service.Key} state {lastHealth} => {service.HealthStatus}")); } LastHealth.TryUpdate(service.InstanceId, service.HealthStatus, lastHealth); } } foreach (var service in notOk) { if (service.heartbeatState == HeartbeatState.Unspecified) { continue; // New? } var problem = ""; if (service.heartbeatState == HeartbeatState.Missing) { problem += " " + service.heartbeatState.ToString(); } if (service.HealthStatus != Microsoft.Extensions.Diagnostics.HealthChecks.HealthStatus.Healthy) { problem += " " + service.HealthStatus.ToString(); } string title = $"{problem}: {service.Key}"; heartbeatLog.Log(new HeartbeatTrackerLogItem(LogLevel.Warning, $"[ALERT - {problem}]", $"{title}")); //await Task.WhenAll(alerters.Select(async a => await a.Alert(title, problem, "", true))); } }
protected void Log(HeartbeatTrackerLogItem logItem) => heartbeatLog.Log(logItem);