private void Timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
var notOk = tracker.Statuses.Where(s => !s.IsOk);
foreach (var service in tracker.Statuses)
{
{
var lastState = LastStates.GetOrAdd(service.InstanceId, id => service.heartbeatState);
if (lastState != service.heartbeatState)
{
heartbeatLog.Log(new HeartbeatTrackerLogItem(LogLevel.Information, "State Changed", $"{service.Key} state {lastState} => {service.heartbeatState}"));
}
LastStates.TryUpdate(service.InstanceId, service.heartbeatState, lastState);
}
{
var lastHealth = LastHealth.GetOrAdd(service.InstanceId, id => service.HealthStatus);
if (lastHealth != service.HealthStatus)
{
heartbeatLog.Log(new HeartbeatTrackerLogItem(LogLevel.Information, "Health Changed", $"{service.Key} state {lastHealth} => {service.HealthStatus}"));
}
LastHealth.TryUpdate(service.InstanceId, service.HealthStatus, lastHealth);
}
}
foreach (var service in notOk)
{
if (service.heartbeatState == HeartbeatState.Unspecified)
{
continue; // New?
}
var problem = "";
if (service.heartbeatState == HeartbeatState.Missing)
{
problem += " " + service.heartbeatState.ToString();
}
if (service.HealthStatus != Microsoft.Extensions.Diagnostics.HealthChecks.HealthStatus.Healthy)
{
problem += " " + service.HealthStatus.ToString();
}
string title = $"{problem}: {service.Key}";
heartbeatLog.Log(new HeartbeatTrackerLogItem(LogLevel.Warning, $"[ALERT - {problem}]", $"{title}"));
//await Task.WhenAll(alerters.Select(async a => await a.Alert(title, problem, "", true)));
}
}
protected void Log(HeartbeatTrackerLogItem logItem) => heartbeatLog.Log(logItem);