public async Task CheckHealth(CancellationToken cancellationToken, IMonitorSettings settings)
{
Logger.DebugFormat("Checking health of {0}...", this);
var endpointHealth = await PerformHealthCheck(cancellationToken, settings);
LogHealthStatus(endpointHealth);
Health = endpointHealth;
}
public Endpoint UpdateHealth(EndpointHealth health)
{
if (Health != null && Health.CheckTimeUtc > health.CheckTimeUtc)
return this;
Health = health;
UpdateLastModifiedTime();
return this;
}
public EndpointMetricsForwarderCoordinatorTest()
{
var endpointId = Guid.NewGuid();
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.FromSeconds(1), EndpointStatus.Healthy);
_endpoint = new Endpoint(TimeCoordinatorMock.Get().Object, new EndpointIdentity(endpointId, "type", "Address"), new EndpointMetadata("Name", "Group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
_endpoint.UpdateHealth(health);
_forwarder = new Mock<IEndpointMetricsForwarder>();
_duplicateForwarder = new Mock<IEndpointMetricsForwarder>();
}
public void CheckHealth_should_update_the_endpoint_with_its_health_status()
{
var endpoint = new Endpoint(Guid.Empty, MonitorMock.Mock("monitor"), "address", "name", "group");
var sampler = new Mock<IHealthSampler>();
var token = new CancellationToken();
var result = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
sampler.Setup(s => s.CheckHealth(endpoint, token)).Returns(Task.FromResult(result));
endpoint.CheckHealth(sampler.Object, token).Wait();
Assert.Same(result, endpoint.Health);
}
public async Task CheckHealth_should_update_the_endpoint_with_its_health_status()
{
var endpoint = new MonitorableEndpoint(new EndpointIdentity(Guid.Empty, "monitor", "address"), MonitorMock.Mock("monitor"));
var sampler = new Mock<IHealthSampler>();
var token = new CancellationToken();
var result = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
sampler.Setup(s => s.CheckHealthAsync(endpoint, token)).Returns(Task.FromResult(result));
await endpoint.CheckHealth(sampler.Object, token);
Assert.Same(result, endpoint.Health);
}
public void InsertEndpointStatistics(Guid endpointId, EndpointHealth stats)
{
using (var conn = _db.OpenConnection())
using (var tx = conn.BeginTransaction())
{
conn.Execute(
"insert into EndpointStats (EndpointId, CheckTimeUtc, ResponseTime, Status) values (@endpointId, @checkTimeUtc, @responseTime, @status)",
new { endpointId, stats.CheckTimeUtc, responseTime = stats.ResponseTime.Ticks, stats.Status }, tx);
tx.Commit();
}
}
private void InsertStatistics(Guid id, EndpointHealth health)
{
try
{
_repository.InsertEndpointStatistics(id, health);
}
catch (Exception e)
{
Logger.Error($"Unable to insert endpoint statistics: {e}");
}
}
public void RecordEndpointStatistics_should_save_statistics()
{
var repository = new Mock<IEndpointStatsRepository>();
var monitorSettings = new Mock<IMonitorSettings>();
var endpointHealth = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Offline);
var endpointId = Guid.NewGuid();
using (var manager = new EndpointStatsManager(repository.Object, monitorSettings.Object))
manager.RecordEndpointStatistics(endpointId, endpointHealth);
repository.Verify(r => r.InsertEndpointStatistics(endpointId, endpointHealth));
}
public void HandleMetricsForwarding(EndpointIdentity identity, EndpointMetadata metadata, EndpointHealth health)
{
var ed = new EndpointDetails(identity.Id, metadata.Group, metadata.Name, identity.Address, identity.MonitorType);
var em = new EndpointMetrics(health.CheckTimeUtc, health.ResponseTime.Milliseconds, health.Status.ToString());
foreach (var f in _forwarders)
{
Logger.InfoFormat("Forwarding metrics using {0} forwarder, for endpoint id {1}", f.Key, identity.Id);
f.Value.ForwardEndpointMetrics(ed, em);
}
}
public void UpdateHealth_should_update_health_and_last_modified_time()
{
var endpoint = new Endpoint(_timeCoordinator.Object, new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), new EndpointMetadata("name", "group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
var expectedLastModifiedTime = DateTime.UtcNow;
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime);
endpoint.UpdateHealth(health);
Assert.Same(health, endpoint.Health);
Assert.Equal(expectedLastModifiedTime, endpoint.LastModifiedTimeUtc);
}
private void ReportEndpointTimeout(Endpoint endpoint)
{
try
{
var health = new EndpointHealth(_timeCoordinator.UtcNow, TimeSpan.Zero, EndpointStatus.TimedOut,
new Dictionary<string, string> { { "reason", "Endpoint health was not updated within specified period of time." } });
_endpointRegistry.UpdateHealth(endpoint.Identity.Id, health);
_logger.Warn($"Endpoint Id={endpoint.Identity.Id} health was not updated within specified period of time.");
}
catch (Exception e)
{
_logger.Error($"Unable to update endpoint Id={endpoint.Identity.Id} health.", e);
}
}
private void LogHealthStatus(EndpointHealth endpointHealth)
{
switch (endpointHealth.Status)
{
case EndpointStatus.TimedOut:
case EndpointStatus.Unhealthy:
Logger.WarnFormat("Status of {0}: Status={1}, ResponseTime={2}", this, endpointHealth.Status, endpointHealth.ResponseTime);
break;
case EndpointStatus.Faulty:
Logger.ErrorFormat("Status of {0}: Status={1}, ResponseTime={2}, Details={3}", this, endpointHealth.Status, endpointHealth.ResponseTime, endpointHealth.PrettyFormatDetails());
break;
default:
Logger.InfoFormat("Status of {0}: Status={1}, ResponseTime={2}", this, endpointHealth.Status, endpointHealth.ResponseTime);
break;
}
}
public void RecordEndpointStatistics_should_save_statistics()
{
var repository = new Mock<IEndpointStatsRepository>();
var monitorSettings = new Mock<IMonitorSettings>();
var endpointMetricsCoordinator = new Mock<IEndpointMetricsForwarderCoordinator>();
var endpointHealth = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Offline);
var endpointId = Guid.NewGuid();
var endpoint = new Endpoint(TimeCoordinatorMock.Get().Object, new EndpointIdentity(endpointId, "type", "address"), new EndpointMetadata("name", "group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
using (var manager = new EndpointStatsManager(repository.Object, monitorSettings.Object, TimeCoordinatorMock.Get().Object, endpointMetricsCoordinator.Object))
{
manager.RecordEndpointStatistics(endpoint.Identity, endpoint.Metadata, endpointHealth);
}
repository.Verify(r => r.InsertEndpointStatistics(endpointId, endpointHealth));
}
private static void LogHealthStatus(Endpoint endpoint, EndpointHealth endpointHealth)
{
switch (endpointHealth.Status)
{
case EndpointStatus.TimedOut:
case EndpointStatus.Unhealthy:
Logger.WarnFormat("Status of {0}: Status={1}, ResponseTime={2}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime);
break;
case EndpointStatus.Faulty:
Logger.ErrorFormat("Status of {0}: Status={1}, ResponseTime={2}, Details={3}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime, endpointHealth.PrettyFormatDetails());
break;
default:
Logger.InfoFormat("Status of {0}: Status={1}, ResponseTime={2}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime);
break;
}
}
public void UpdateHealth_should_not_update_health_if_already_have_more_recent_result()
{
var expectedLastModifiedTime1 = DateTime.UtcNow;
var expectedLastModifiedTime2 = DateTime.UtcNow.AddMinutes(1);
var endpoint = new Endpoint(_timeCoordinator.Object, new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), new EndpointMetadata("name", "group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
var newHealth = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
var oldHealth = new EndpointHealth(DateTime.UtcNow.AddSeconds(-1), TimeSpan.Zero, EndpointStatus.Healthy);
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime1);
endpoint.UpdateHealth(newHealth);
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime2);
endpoint.UpdateHealth(oldHealth);
Assert.Same(newHealth, endpoint.Health);
Assert.Equal(expectedLastModifiedTime1, endpoint.LastModifiedTimeUtc);
}
public void UpdateHealth_should_update_health_and_save_it_in_repository()
{
SetupMonitors("monitor");
var id = _registry.RegisterOrUpdate("monitor", "address", "group", "name", null, null, "password");
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
Assert.True(_registry.UpdateHealth(id, health));
_statsManager.Verify(r => r.RecordEndpointStatistics(It.IsAny<EndpointIdentity>(), It.IsAny<EndpointMetadata>(), health));
Assert.Same(_registry.GetById(id).Health, health);
}
public void UpdateHealth_should_return_false_for_unknown_endpoints()
{
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
var id = Guid.NewGuid();
Assert.False(_registry.UpdateHealth(id, health));
}
private static bool AssertHealth(EndpointHealth actual, HealthUpdate expected)
{
Assert.Equal(expected.CheckTimeUtc, actual.CheckTimeUtc);
Assert.Equal(expected.ResponseTime, actual.ResponseTime);
Assert.Equal(expected.Status, actual.Status);
Assert.Equal(expected.Details, actual.Details);
return true;
}
public void GetEndpoint_should_return_endpoint_information_with_details(EndpointStatus status)
{
var id = Guid.NewGuid();
var endpoint = new Endpoint(TimeCoordinatorMock.Get().Object, new EndpointIdentity(id, "monitor", "address"), new EndpointMetadata("name", "group", new[] { "t1", "t2" }, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
var endpointHealth = new EndpointHealth(_utcNow, TimeSpan.FromSeconds(5), status, new Dictionary<string, string> { { "abc", "def" } });
endpoint.UpdateHealth(endpointHealth);
_endpointRegistry.Setup(r => r.GetById(id)).Returns(endpoint);
var result = _controller.GetEndpoint(id) as OkNegotiatedContentResult<EndpointDetails>;
Assert.NotNull(result);
AssertEndpoint(endpoint, result.Content);
Assert.Equal(status, result.Content.Status);
Assert.Equal(endpointHealth.CheckTimeUtc, result.Content.LastCheckUtc);
Assert.Equal(endpointHealth.ResponseTime, result.Content.LastResponseTime);
Assert.Equal(endpointHealth.Details, result.Content.Details);
}
public void RecordEndpointStatistics(Guid endpointId, EndpointHealth stats)
{
_statsQueue.Add(Tuple.Create(endpointId, stats));
}
public async Task CheckHealth(IHealthSampler sampler, CancellationToken cancellationToken)
{
Health = await sampler.CheckHealth(this, cancellationToken);
}
public void RecordEndpointStatistics(EndpointIdentity identity, EndpointMetadata metadata, EndpointHealth stats)
{
_statsQueue.Enqueue(Tuple.Create(identity, metadata, stats));
}
public void UpdateHealth(Guid endpointId, EndpointHealth endpointHealth)
{
_outgoingQueue.Enqueue(new EndpointHealthUpdate(endpointId, endpointHealth));
}
private static void AssertCheckTime(EndpointHealth result)
{
var now = DateTime.UtcNow;
var delta = TimeSpan.FromMilliseconds(100);
var dateFormat = "yyyy-MM-dd HH:mm:ss.fff";
var difference = (now - result.CheckTimeUtc) - result.ResponseTime;
Assert.True(difference.Duration() < delta, string.Format("Expected CheckTimeUtc being {0} ~ {1}ms, got: {2}", (now - result.ResponseTime).ToString(dateFormat), delta.TotalMilliseconds, result.CheckTimeUtc.ToString(dateFormat)));
}
public async Task CheckHealth(IHealthSampler sampler, CancellationToken cancellationToken)
{
Health = await sampler.CheckHealth(this, cancellationToken);
}
private void AssertResponseTime(EndpointHealth result, TimeSpan expectedTime)
{
var delta = TimeSpan.FromMilliseconds(100);
var difference = result.ResponseTime - expectedTime;
Assert.True(difference.Duration() < delta, string.Format("Expected ResponseTime being {0} ~ {1}ms, got: {2}", expectedTime, delta.TotalMilliseconds, result.ResponseTime));
}
public EndpointHealthUpdate(Guid endpointId, EndpointHealth health)
{
EndpointId = endpointId;
Health = health;
}
public void GetEndpoint_should_return_endpoint_information_with_details(EndpointStatus status)
{
Guid id = Guid.NewGuid();
var healthSampler = new Mock<IHealthSampler>();
var endpoint = new Endpoint(id, MonitorMock.GetMock("monitor").Object, "address", "name", "group");
var token = new CancellationToken();
var endpointHealth = new EndpointHealth(DateTime.UtcNow, TimeSpan.FromSeconds(1), status, new Dictionary<string, string> { { "a", "b" }, { "c", "d" } });
healthSampler.Setup(s => s.CheckHealth(endpoint, token)).Returns(Task.FromResult(endpointHealth));
endpoint.CheckHealth(healthSampler.Object, token).Wait();
_endpointRegistry.Setup(r => r.GetById(id)).Returns(endpoint);
var result = _controller.GetEndpoint(id) as OkNegotiatedContentResult<EndpointDetails>;
Assert.NotNull(result);
AssertEndpoint(endpoint, result.Content);
Assert.Equal(status, result.Content.Status);
Assert.Equal(endpointHealth.CheckTimeUtc, result.Content.LastCheckUtc);
Assert.Equal(endpointHealth.ResponseTime, result.Content.LastResponseTime);
Assert.Equal(endpointHealth.Details, result.Content.Details);
}
