private void AssertResponseTime(EndpointHealth result, TimeSpan expectedTime)
{
var delta = TimeSpan.FromMilliseconds(100);
var difference = result.ResponseTime - expectedTime;
Assert.True(difference.Duration() < delta, string.Format("Expected ResponseTime being {0} ~ {1}ms, got: {2}", expectedTime, delta.TotalMilliseconds, result.ResponseTime));
}
public async Task Notifier_should_send_current_health_to_the_API()
{
var endpointId = Guid.NewGuid();
var expectedHealth = new EndpointHealth(HealthStatus.Offline, new Dictionary <string, string> {
{ "key", "value" }
});
HealthUpdate lastCaptured = null;
var countdown = new AsyncCountdown("update", 5);
SetupHealthCheckInterval(TimeSpan.FromMilliseconds(1));
SetupEndpointRegistration(endpointId);
_mockClient.Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny <HealthUpdate>(), It.IsAny <CancellationToken>()))
.Returns((Guid id, string authToken, HealthUpdate upd, CancellationToken token) => _awaitableFactory
.Execute(() => lastCaptured = upd)
.WithCountdown(countdown)
.RunAsync());
using (CreateNotifier(token => Task.FromResult(expectedHealth)))
await countdown.WaitAsync(TestMaxTime);
Assert.NotNull(lastCaptured);
Assert.Equal(expectedHealth.Status, lastCaptured.Status);
Assert.Equal(expectedHealth.Details, lastCaptured.Details);
Assert.True(lastCaptured.CheckTimeUtc > DateTime.UtcNow.AddMinutes(-1) && lastCaptured.CheckTimeUtc < DateTime.UtcNow.AddMinutes(1));
}
public HealthUpdate(DateTime checkTimeUtc, TimeSpan responseTime, EndpointHealth info)
{
CheckTimeUtc = checkTimeUtc;
Status = info.Status;
ResponseTime = responseTime;
Details = info.Details;
}
public void UpdateHealth_should_return_false_for_unknown_endpoints()
{
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
var id = Guid.NewGuid();
Assert.False(_registry.UpdateHealth(id, health));
}
public void GetEndpoint_should_return_endpoint_information_with_details(EndpointStatus status)
{
Guid id = Guid.NewGuid();
var healthSampler = new Mock <IHealthSampler>();
var endpoint = new Endpoint(id, MonitorMock.GetMock("monitor").Object, "address", "name", "group");
var token = new CancellationToken();
var endpointHealth = new EndpointHealth(DateTime.UtcNow, TimeSpan.FromSeconds(1), status, new Dictionary <string, string> {
{ "a", "b" }, { "c", "d" }
});
healthSampler.Setup(s => s.CheckHealth(endpoint, token)).Returns(Task.FromResult(endpointHealth));
endpoint.CheckHealth(healthSampler.Object, token).Wait();
_endpointRegistry.Setup(r => r.GetById(id)).Returns(endpoint);
var result = _controller.GetEndpoint(id) as OkNegotiatedContentResult <EndpointDetails>;
Assert.NotNull(result);
AssertEndpoint(endpoint, result.Content);
Assert.Equal(status, result.Content.Status);
Assert.Equal(endpointHealth.CheckTimeUtc, result.Content.LastCheckUtc);
Assert.Equal(endpointHealth.ResponseTime, result.Content.LastResponseTime);
Assert.Equal(endpointHealth.Details, result.Content.Details);
}
public HealthUpdate(DateTime checkTimeUtc, TimeSpan responseTime, EndpointHealth info)
{
CheckTimeUtc = checkTimeUtc;
Status = info.Status;
ResponseTime = responseTime;
Details = info.Details;
}
private async Task PerformHealthCheckAsync()
{
EndpointHealth endpointHealth;
var checkTimeUtc = DateTime.UtcNow;
var watch = Stopwatch.StartNew();
try
{
endpointHealth = await _healthChecker.CheckHealthAsync(_cancelationTokenSource.Token);
if (endpointHealth == null)
{
throw new InvalidOperationException("Health information not provided");
}
}
catch (OperationCanceledException) when(_cancelationTokenSource.IsCancellationRequested)
{
return;
}
catch (Exception e)
{
_logger.Error("Unable to collect health information", e);
endpointHealth = new EndpointHealth(
HealthStatus.Faulty,
new Dictionary <string, string> {
{ "reason", "Unable to collect health information" }, { "exception", e.ToString() }
});
}
await EnsureSendHealthUpdateAsync(new HealthUpdate(checkTimeUtc, watch.Elapsed, endpointHealth));
}
private static bool AssertHealth(EndpointHealth actual, HealthUpdate expected)
{
Assert.Equal(expected.CheckTimeUtc, actual.CheckTimeUtc);
Assert.Equal(expected.ResponseTime, actual.ResponseTime);
Assert.Equal(expected.Status, actual.Status);
Assert.Equal(expected.Details, actual.Details);
return(true);
}
public void UpdateHealth_should_ignore_unknown_edpoints()
{
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
var id = Guid.NewGuid();
_registry.UpdateHealth(id, health);
_statsRepository.Verify(r => r.InsertEndpointStatistics(id, health), Times.Never);
}
private static void AssertCheckTime(EndpointHealth result)
{
var now = DateTime.UtcNow;
var delta = TimeSpan.FromMilliseconds(100);
var dateFormat = "yyyy-MM-dd HH:mm:ss.fff";
var difference = (now - result.CheckTimeUtc) - result.ResponseTime;
Assert.True(difference.Duration() < delta, string.Format("Expected CheckTimeUtc being {0} ~ {1}ms, got: {2}", (now - result.ResponseTime).ToString(dateFormat), delta.TotalMilliseconds, result.CheckTimeUtc.ToString(dateFormat)));
}
public void UpdateHealth_should_update_health_and_save_it_in_repository()
{
SetupMonitors("monitor");
var id = _registry.RegisterOrUpdate("monitor", "address", "group", "name", null);
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
_registry.UpdateHealth(id, health);
_statsRepository.Verify(r => r.InsertEndpointStatistics(id, health));
Assert.Same(_registry.GetById(id).Health, health);
}
public void UpdateHealth_should_update_health_and_save_it_in_repository()
{
SetupMonitors("monitor");
var id = _registry.RegisterOrUpdate("monitor", "address", "group", "name", null, null, "password");
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
Assert.True(_registry.UpdateHealth(id, health));
_statsManager.Verify(r => r.RecordEndpointStatistics(It.IsAny <EndpointIdentity>(), It.IsAny <EndpointMetadata>(), health));
Assert.Same(_registry.GetById(id).Health, health);
}
public void UpdateHealth_should_not_update_health_if_already_have_more_recent_result()
{
var endpoint = new Endpoint(new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), new EndpointMetadata("name", "group", null));
var newHealth = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
var oldHealth = new EndpointHealth(DateTime.UtcNow.AddSeconds(-1), TimeSpan.Zero, EndpointStatus.Healthy);
endpoint.UpdateHealth(newHealth);
endpoint.UpdateHealth(oldHealth);
Assert.Same(newHealth, endpoint.Health);
}
private void InsertStatistics(Guid id, EndpointHealth health)
{
try
{
_repository.InsertEndpointStatistics(id, health);
}
catch (Exception e)
{
Logger.Error($"Unable to insert endpoint statistics: {e}");
}
}
private void InsertStatistics(Guid id, EndpointHealth health)
{
try
{
_repository.InsertEndpointStatistics(id, health);
}
catch (Exception e)
{
_logger.ErrorFormat("Unable to insert endpoint statistics: {0}", e.ToString());
}
}
public void UpdateHealth(Guid endpointId, EndpointHealth health)
{
var endpoint = GetById(endpointId);
if (endpoint == null)
{
return;
}
endpoint.UpdateHealth(health);
_endpointStatsManager.RecordEndpointStatistics(endpoint.Identity, endpoint.Metadata, health);
}
public void CheckHealth_should_update_the_endpoint_with_its_health_status()
{
var endpoint = new Endpoint(Guid.Empty, MonitorMock.Mock("monitor"), "address", "name", "group");
var sampler = new Mock <IHealthSampler>();
var token = new CancellationToken();
var result = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
sampler.Setup(s => s.CheckHealth(endpoint, token)).Returns(Task.FromResult(result));
endpoint.CheckHealth(sampler.Object, token).Wait();
Assert.Same(result, endpoint.Health);
}
public void UpdateHealth(Guid endpointId, EndpointHealth health)
{
var endpoint = GetById(endpointId);
if (endpoint == null)
{
return;
}
endpoint.UpdateHealth(health);
_statsRepository.InsertEndpointStatistics(endpointId, health);
}
public Endpoint UpdateHealth(EndpointHealth health)
{
if (Health != null && Health.CheckTimeUtc > health.CheckTimeUtc)
{
return(this);
}
Health = health;
UpdateLastModifiedTime();
return(this);
}
public EndpointMetricsForwarderCoordinatorTest()
{
var endpointId = Guid.NewGuid();
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.FromSeconds(1), EndpointStatus.Healthy);
_endpoint = new Endpoint(TimeCoordinatorMock.Get().Object, new EndpointIdentity(endpointId, "type", "Address"), new EndpointMetadata("Name", "Group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
_endpoint.UpdateHealth(health);
_forwarder = new Mock <IEndpointMetricsForwarder>();
_duplicateForwarder = new Mock <IEndpointMetricsForwarder>();
}
public void InsertEndpointStatistics(Guid endpointId, EndpointHealth stats)
{
using (var conn = _db.OpenConnection())
using (var tx = conn.BeginTransaction())
{
conn.Execute(
"insert into EndpointStats (EndpointId, CheckTimeUtc, ResponseTime, Status) values (@endpointId, @checkTimeUtc, @responseTime, @status)",
new { endpointId, stats.CheckTimeUtc, responseTime = stats.ResponseTime.Ticks, stats.Status }, tx);
tx.Commit();
}
}
public async Task UpdateHealth_should_update_health_and_last_modified_time()
{
var endpoint = new Endpoint(new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), new EndpointMetadata("name", "group", null));
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
await Task.Delay(TimeSpan.FromMilliseconds(200));
endpoint.UpdateHealth(health);
Assert.Same(health, endpoint.Health);
AssertTime(endpoint, DateTimeOffset.UtcNow, endpoint.LastModifiedTime, TimeSpan.FromMilliseconds(50));
}
public async Task CheckHealth_should_update_the_endpoint_with_its_health_status()
{
var endpoint = new MonitorableEndpoint(new EndpointIdentity(Guid.Empty, "monitor", "address"), MonitorMock.Mock("monitor"));
var sampler = new Mock <IHealthSampler>();
var token = new CancellationToken();
var result = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
sampler.Setup(s => s.CheckHealthAsync(endpoint, token)).Returns(Task.FromResult(result));
await endpoint.CheckHealth(sampler.Object, token);
Assert.Same(result, endpoint.Health);
}
public void RecordEndpointStatistics_should_save_statistics()
{
var repository = new Mock <IEndpointStatsRepository>();
var monitorSettings = new Mock <IMonitorSettings>();
var endpointHealth = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Offline);
var endpointId = Guid.NewGuid();
using (var manager = new EndpointStatsManager(repository.Object, monitorSettings.Object))
manager.RecordEndpointStatistics(endpointId, endpointHealth);
repository.Verify(r => r.InsertEndpointStatistics(endpointId, endpointHealth));
}
public void UpdateHealth_should_update_health_and_last_modified_time()
{
var endpoint = new Endpoint(_timeCoordinator.Object, new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), new EndpointMetadata("name", "group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
var expectedLastModifiedTime = DateTime.UtcNow;
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime);
endpoint.UpdateHealth(health);
Assert.Same(health, endpoint.Health);
Assert.Equal(expectedLastModifiedTime, endpoint.LastModifiedTimeUtc);
}
private void ReportEndpointTimeout(Endpoint endpoint)
{
try
{
var health = new EndpointHealth(_timeCoordinator.UtcNow, TimeSpan.Zero, EndpointStatus.TimedOut,
new Dictionary <string, string> {
{ "reason", "Endpoint health was not updated within specified period of time." }
});
_endpointRegistry.UpdateHealth(endpoint.Identity.Id, health);
_logger.Warn($"Endpoint Id={endpoint.Identity.Id} health was not updated within specified period of time.");
}
catch (Exception e)
{
_logger.Error($"Unable to update endpoint Id={endpoint.Identity.Id} health.", e);
}
}
public void RecordEndpointStatistics_should_save_statistics()
{
var repository = new Mock <IEndpointStatsRepository>();
var monitorSettings = new Mock <IMonitorSettings>();
var endpointMetricsCoordinator = new Mock <IEndpointMetricsForwarderCoordinator>();
var endpointHealth = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Offline);
var endpointId = Guid.NewGuid();
var endpoint = new Endpoint(TimeCoordinatorMock.Get().Object, new EndpointIdentity(endpointId, "type", "address"), new EndpointMetadata("name", "group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
using (var manager = new EndpointStatsManager(repository.Object, monitorSettings.Object, TimeCoordinatorMock.Get().Object, endpointMetricsCoordinator.Object))
{
manager.RecordEndpointStatistics(endpoint.Identity, endpoint.Metadata, endpointHealth);
}
repository.Verify(r => r.InsertEndpointStatistics(endpointId, endpointHealth));
}
public void UpdateHealth_should_not_update_health_if_already_have_more_recent_result()
{
var expectedLastModifiedTime1 = DateTime.UtcNow;
var expectedLastModifiedTime2 = DateTime.UtcNow.AddMinutes(1);
var endpoint = new Endpoint(_timeCoordinator.Object, new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), new EndpointMetadata("name", "group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
var newHealth = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
var oldHealth = new EndpointHealth(DateTime.UtcNow.AddSeconds(-1), TimeSpan.Zero, EndpointStatus.Healthy);
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime1);
endpoint.UpdateHealth(newHealth);
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime2);
endpoint.UpdateHealth(oldHealth);
Assert.Same(newHealth, endpoint.Health);
Assert.Equal(expectedLastModifiedTime1, endpoint.LastModifiedTimeUtc);
}
private static void LogHealthStatus(MonitorableEndpoint endpoint, EndpointHealth endpointHealth)
{
switch (endpointHealth.Status)
{
case EndpointStatus.TimedOut:
case EndpointStatus.Unhealthy:
Logger.WarnFormat("Status of {0}: Status={1}, ResponseTime={2}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime);
break;
case EndpointStatus.Faulty:
Logger.ErrorFormat("Status of {0}: Status={1}, ResponseTime={2}, Details={3}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime, endpointHealth.PrettyFormatDetails());
break;
default:
Logger.InfoFormat("Status of {0}: Status={1}, ResponseTime={2}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime);
break;
}
}
public void UpdateHealth_should_update_health_and_last_modified_time_if_newer_health_is_provided()
{
var expectedLastModifiedTime1 = DateTime.UtcNow;
var expectedLastModifiedTime2 = DateTime.UtcNow.AddMinutes(1);
var endpoint = new Endpoint(_timeCoordinator.Object, new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), new EndpointMetadata("name", "group", null));
var oldHealth = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime1);
endpoint.UpdateHealth(oldHealth);
Assert.Same(oldHealth, endpoint.Health);
Assert.Equal(expectedLastModifiedTime1, endpoint.LastModifiedTimeUtc);
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime2);
endpoint.UpdateHealth(health);
Assert.Same(health, endpoint.Health);
Assert.Equal(expectedLastModifiedTime2, endpoint.LastModifiedTimeUtc);
}
public void GetEndpoint_should_return_endpoint_information_with_details(EndpointStatus status)
{
var id = Guid.NewGuid();
var endpoint = new Endpoint(TimeCoordinatorMock.Get().Object, new EndpointIdentity(id, "monitor", "address"), new EndpointMetadata("name", "group", new[] { "t1", "t2" }, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
var endpointHealth = new EndpointHealth(_utcNow, TimeSpan.FromSeconds(5), status, new Dictionary <string, string> {
{ "abc", "def" }
});
endpoint.UpdateHealth(endpointHealth);
_endpointRegistry.Setup(r => r.GetById(id)).Returns(endpoint);
var result = _controller.GetEndpoint(id) as OkNegotiatedContentResult <EndpointDetails>;
Assert.NotNull(result);
AssertEndpoint(endpoint, result.Content);
Assert.Equal(status, result.Content.Status);
Assert.Equal(endpointHealth.CheckTimeUtc, result.Content.LastCheckUtc);
Assert.Equal(endpointHealth.ResponseTime, result.Content.LastResponseTime);
Assert.Equal(endpointHealth.Details, result.Content.Details);
}
public async Task Notifier_should_send_current_health_to_the_API()
{
var endpointId = Guid.NewGuid();
var expectedHealth = new EndpointHealth(HealthStatus.Offline, new Dictionary<string, string> { { "key", "value" } });
HealthUpdate lastCaptured = null;
var countdown = new AsyncCountdown("update", 5);
SetupHealthCheckInterval(TimeSpan.FromMilliseconds(1));
SetupEndpointRegistration(endpointId);
_mockClient.Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny<HealthUpdate>(), It.IsAny<CancellationToken>()))
.Returns((Guid id, string authToken, HealthUpdate upd, CancellationToken token) => _awaitableFactory
.Execute(() => lastCaptured = upd)
.WithCountdown(countdown)
.RunAsync());
using (CreateNotifier(token => Task.FromResult(expectedHealth)))
await countdown.WaitAsync(TestMaxTime);
Assert.NotNull(lastCaptured);
Assert.Equal(expectedHealth.Status, lastCaptured.Status);
Assert.Equal(expectedHealth.Details, lastCaptured.Details);
Assert.True(lastCaptured.CheckTimeUtc > DateTime.UtcNow.AddMinutes(-1) && lastCaptured.CheckTimeUtc < DateTime.UtcNow.AddMinutes(1));
}
private async Task PerformHealthCheckAsync()
{
EndpointHealth endpointHealth;
var checkTimeUtc = DateTime.UtcNow;
var watch = Stopwatch.StartNew();
try
{
endpointHealth = await _healthChecker.CheckHealthAsync(_cancelationTokenSource.Token);
if (endpointHealth == null)
throw new InvalidOperationException("Health information not provided");
}
catch (OperationCanceledException) when (_cancelationTokenSource.IsCancellationRequested)
{
return;
}
catch (Exception e)
{
_logger.Error("Unable to collect health information", e);
endpointHealth = new EndpointHealth(HealthStatus.Faulty, new Dictionary<string, string> { { "reason", "Unable to collect health information" }, { "exception", e.ToString() } });
}
await EnsureSendHealthUpdateAsync(new HealthUpdate(checkTimeUtc, watch.Elapsed, endpointHealth));
}
