public void Dispose_should_mark_endpoint_disposed()
{
var endpoint = new Endpoint(Guid.Empty, MonitorMock.Mock("monitor"), "address", "name", "group");
Assert.False(endpoint.IsDisposed);
endpoint.Dispose();
Assert.True(endpoint.IsDisposed);
}
private async Task<EndpointHealth> PerformHealthCheck(CancellationToken cancellationToken, Endpoint endpoint)
{
var checkTimeUtc = DateTime.UtcNow;
var timer = new Stopwatch();
try
{
using (var timeoutToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
timer.Start();
var timeoutTask = ConfigureTimeoutTask(endpoint, cancellationToken);
var healthTask = endpoint.Monitor.CheckHealthAsync(endpoint.Address, timeoutToken.Token);
var healthResult = await Task.WhenAny(healthTask, timeoutTask);
timer.Stop();
await CancelHealthTaskIfNeeded(healthTask, timeoutToken);
return FromResult(checkTimeUtc, timer.Elapsed, healthResult.Result);
}
}
catch (OperationCanceledException)
{
throw;
}
catch (AggregateException e)
{
if (e.InnerExceptions.First() is OperationCanceledException)
throw;
return FromException(checkTimeUtc, timer.Elapsed, e.InnerExceptions.First());
}
catch (Exception e)
{
return FromException(checkTimeUtc, timer.Elapsed, e);
}
}
public async Task<EndpointHealth> CheckHealth(Endpoint endpoint, CancellationToken cancellationToken)
{
Logger.DebugFormat("Checking health of {0}...", endpoint);
var endpointHealth = await PerformHealthCheck(cancellationToken, endpoint);
_statsManager.RecordEndpointStatistics(endpoint.Id, endpointHealth);
LogHealthStatus(endpoint, endpointHealth);
return endpointHealth;
}
public HealthSamplerTests()
{
_settings = new Mock<IMonitorSettings>();
_statsManager = new Mock<IEndpointStatsManager>();
_sampler = new HealthSampler(_settings.Object, _statsManager.Object);
_monitor = new Mock<IHealthMonitor>();
_endpoint = new Endpoint(Guid.NewGuid(), _monitor.Object, "address", "name", "group");
}
public void Endpoint_should_be_created_with_last_modified_time_set()
{
var expectedLastModifiedTime = DateTime.UtcNow;
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime);
var endpoint = new Endpoint(_timeCoordinator.Object, new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), new EndpointMetadata("name", "group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
Assert.Equal(expectedLastModifiedTime, endpoint.LastModifiedTimeUtc);
}
public void EndpointRegistry_should_load_endpoints_from_repository()
{
var endpoint = new Endpoint(_timeCoordinator.Object, new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), new EndpointMetadata("name", "group", new[] { "t1", "t2" }, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
_configurationStore.Setup(s => s.LoadEndpoints()).Returns(new[] { endpoint });
var registry = new EndpointRegistry(_healthMonitorTypeRegistry.Object, _configurationStore.Object, _statsManager.Object, _timeCoordinator.Object);
Assert.Same(endpoint, registry.GetById(endpoint.Identity.Id));
}
public void EndpointRegistry_should_load_endpoints_from_store()
{
var endpoint = new Endpoint(Guid.NewGuid(), MonitorMock.Mock("monitor"), "address", "name", "group");
_configurationStore.Setup(s => s.LoadEndpoints(_monitorRegistry.Object)).Returns(new[] { endpoint });
var registry = new EndpointRegistry(_monitorRegistry.Object, _configurationStore.Object,_statsRepository.Object);
Assert.Same(endpoint, registry.GetById(endpoint.Id));
}
private async Task<Endpoint> CreateTaskFor(Endpoint endpoint)
{
await Task.Yield();
while (!_cancellation.IsCancellationRequested && !endpoint.IsDisposed)
{
await endpoint.CheckHealth(_cancellation.Token,_settings);
await Task.Delay(_settings.HealthCheckInterval);
}
return endpoint;
}
public EndpointMetricsForwarderCoordinatorTest()
{
var endpointId = Guid.NewGuid();
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.FromSeconds(1), EndpointStatus.Healthy);
_endpoint = new Endpoint(TimeCoordinatorMock.Get().Object, new EndpointIdentity(endpointId, "type", "Address"), new EndpointMetadata("Name", "Group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
_endpoint.UpdateHealth(health);
_forwarder = new Mock<IEndpointMetricsForwarder>();
_duplicateForwarder = new Mock<IEndpointMetricsForwarder>();
}
public void CheckHealth_should_update_the_endpoint_with_its_health_status()
{
var endpoint = new Endpoint(Guid.Empty, MonitorMock.Mock("monitor"), "address", "name", "group");
var sampler = new Mock<IHealthSampler>();
var token = new CancellationToken();
var result = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
sampler.Setup(s => s.CheckHealth(endpoint, token)).Returns(Task.FromResult(result));
endpoint.CheckHealth(sampler.Object, token).Wait();
Assert.Same(result, endpoint.Health);
}
public void SaveEndpoint(Endpoint endpoint)
{
using (var conn = _db.OpenConnection())
using (var tx = conn.BeginTransaction())
{
if (conn.Execute("update EndpointConfig set MonitorType=@MonitorType, Address=@Address, GroupName=@Group, Name=@Name where Id=@Id", new { endpoint.MonitorType, endpoint.Address, endpoint.Group, endpoint.Name, endpoint.Id }, tx) == 0)
conn.Execute("insert into EndpointConfig (MonitorType, Address, GroupName, Name, Id) values(@MonitorType,@Address,@Group,@Name,@Id)", new { endpoint.MonitorType, endpoint.Address, endpoint.Group, endpoint.Name, endpoint.Id }, tx);
tx.Commit();
}
}
private static Mock<IHealthSampler> SetupSampler(Endpoint endpoint, ManualResetEventSlim manualReset)
{
var sampler = new Mock<IHealthSampler>();
sampler
.Setup(s => s.CheckHealth(endpoint, It.IsAny<CancellationToken>()))
.Returns(async (Endpoint e, CancellationToken ct) =>
{
await Task.Run(() => manualReset.Wait(ct));
return null;
});
return sampler;
}
public void UpdateHealth_should_update_health_and_last_modified_time()
{
var endpoint = new Endpoint(_timeCoordinator.Object, new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), new EndpointMetadata("name", "group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
var health = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
var expectedLastModifiedTime = DateTime.UtcNow;
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime);
endpoint.UpdateHealth(health);
Assert.Same(health, endpoint.Health);
Assert.Equal(expectedLastModifiedTime, endpoint.LastModifiedTimeUtc);
}
public void GetEndpoint_should_return_endpoint_information()
{
Guid id = Guid.NewGuid();
var endpoint = new Endpoint(id, MonitorMock.Mock("monitor"), "address", "name", "group");
_endpointRegistry.Setup(r => r.GetById(id)).Returns(endpoint);
var result = _controller.GetEndpoint(id) as OkNegotiatedContentResult<EndpointDetails>;
Assert.NotNull(result);
AssertEndpoint(endpoint, result.Content);
Assert.Equal(EndpointStatus.NotRun, result.Content.Status);
Assert.Equal(null, result.Content.LastCheckUtc);
Assert.Equal(null, result.Content.LastResponseTime);
Assert.Equal(new Dictionary<string, string>(), result.Content.Details);
}
public void UpdateMetadata_should_update_metadata_and_last_modified_time()
{
var endpoint = new Endpoint(_timeCoordinator.Object, new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), new EndpointMetadata("name", "group", new[] { "t1" }, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
var expectedLastModifiedTime = DateTime.UtcNow;
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime);
endpoint.UpdateMetadata("new group", "new name", new[] { "t1", "t2" }, EndpointMetadata.DefaultMonitorTag);
Assert.Equal("new group", endpoint.Metadata.Group);
Assert.Equal("new name", endpoint.Metadata.Name);
Assert.Equal(new[] { "t1", "t2" }, endpoint.Metadata.Tags);
Assert.Equal(expectedLastModifiedTime, endpoint.LastModifiedTimeUtc);
}
public void RecordEndpointStatistics_should_save_statistics()
{
var repository = new Mock<IEndpointStatsRepository>();
var monitorSettings = new Mock<IMonitorSettings>();
var endpointMetricsCoordinator = new Mock<IEndpointMetricsForwarderCoordinator>();
var endpointHealth = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Offline);
var endpointId = Guid.NewGuid();
var endpoint = new Endpoint(TimeCoordinatorMock.Get().Object, new EndpointIdentity(endpointId, "type", "address"), new EndpointMetadata("name", "group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
using (var manager = new EndpointStatsManager(repository.Object, monitorSettings.Object, TimeCoordinatorMock.Get().Object, endpointMetricsCoordinator.Object))
{
manager.RecordEndpointStatistics(endpoint.Identity, endpoint.Metadata, endpointHealth);
}
repository.Verify(r => r.InsertEndpointStatistics(endpointId, endpointHealth));
}
public void UpdateHealth_should_not_update_health_if_already_have_more_recent_result()
{
var expectedLastModifiedTime1 = DateTime.UtcNow;
var expectedLastModifiedTime2 = DateTime.UtcNow.AddMinutes(1);
var endpoint = new Endpoint(_timeCoordinator.Object, new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), new EndpointMetadata("name", "group", null, EndpointMetadata.DefaultMonitorTag, DateTime.UtcNow, DateTime.UtcNow));
var newHealth = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
var oldHealth = new EndpointHealth(DateTime.UtcNow.AddSeconds(-1), TimeSpan.Zero, EndpointStatus.Healthy);
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime1);
endpoint.UpdateHealth(newHealth);
_timeCoordinator.Setup(c => c.UtcNow).Returns(expectedLastModifiedTime2);
endpoint.UpdateHealth(oldHealth);
Assert.Same(newHealth, endpoint.Health);
Assert.Equal(expectedLastModifiedTime1, endpoint.LastModifiedTimeUtc);
}
public void CheckHealth_should_update_the_endpoint_with_its_health_status(HealthStatus healthStatus)
{
var tokenSource = new CancellationTokenSource();
var healthInfo = new HealthInfo(healthStatus, TimeSpan.FromSeconds(1), new Dictionary<string, string> { { "property1", "value1" }, { "property2", "value2" } });
var monitor = MonitorMock.GetMock("monitor");
monitor.Setup(x => x.CheckHealthAsync("address", It.IsAny<CancellationToken>())).Returns(Task.FromResult(healthInfo));
var endpoint = new Endpoint(Guid.NewGuid(), monitor.Object, "address", "name", "group");
endpoint.CheckHealth(tokenSource.Token, MonitorSettingsHelper.ConfigureDefaultSettings()).Wait();
var health = endpoint.Health;
Assert.NotNull(health);
Assert.Equal(healthInfo.ResponseTime, health.ResponseTime);
Assert.Equal(healthInfo.Status.ToString(), health.Status.ToString());
Assert.True(DateTime.UtcNow - health.CheckTimeUtc < TimeSpan.FromMilliseconds(500), "CheckTimeUtc should be captured");
Assert.Equal(healthInfo.Details, health.Details);
}
public EndpointDetails(Endpoint endpoint)
{
Id = endpoint.Id;
Name = endpoint.Name;
Address = endpoint.Address;
MonitorType = endpoint.MonitorType;
Group = endpoint.Group;
if (endpoint.Health != null)
{
Status = endpoint.Health.Status;
LastCheckUtc = endpoint.Health.CheckTimeUtc;
LastResponseTime = endpoint.Health.ResponseTime;
Details = endpoint.Health.Details.ToDictionary(p => p.Key, p => p.Value);
}
else
{
Details = new Dictionary<string, string>();
Status = EndpointStatus.NotRun;
}
}
public void CheckHealth_should_update_the_endpoint_with_Faulty_status_if_monitor_fails()
{
var tokenSource = new CancellationTokenSource();
var monitor = MonitorMock.GetMock("monitor");
var exception = new Exception("some error");
monitor.Setup(x => x.CheckHealthAsync("address", It.IsAny<CancellationToken>())).Throws(exception);
var endpoint = new Endpoint(Guid.NewGuid(), monitor.Object, "address", "name", "group");
endpoint.CheckHealth(tokenSource.Token, MonitorSettingsHelper.ConfigureDefaultSettings()).Wait();
var expectedDetails = new Dictionary<string, string>
{
{ "reason", exception.Message },
{ "exception", exception.ToString() }
};
var health = endpoint.Health;
Assert.NotNull(health);
Assert.Equal(EndpointStatus.Faulty, health.Status);
Assert.True(DateTime.UtcNow - health.CheckTimeUtc < TimeSpan.FromMilliseconds(500), "CheckTimeUtc should be captured");
Assert.Equal(TimeSpan.Zero, health.ResponseTime);
Assert.Equal(expectedDetails, health.Details);
}
private async Task<HealthInfo> ConfigureTimeoutTask(Endpoint endpoint, CancellationToken cancellationToken)
{
if (RequiresShortTimeout(endpoint))
{
await Task.Delay(_settings.ShortTimeOut, cancellationToken);
return new HealthInfo(HealthStatus.TimedOut, TimeoutDetails);
}
await Task.Delay(_settings.FailureTimeOut, cancellationToken);
return new HealthInfo(HealthStatus.Faulty, TimeoutDetails);
}
private async Task<Endpoint> CreateTaskFor(Endpoint endpoint)
{
await Task.Delay(GetRandomizedDelay());
while (!_cancellation.IsCancellationRequested && !endpoint.IsDisposed)
{
var delay = Task.Delay(_settings.HealthCheckInterval);
await endpoint.CheckHealth(_sampler, _cancellation.Token);
await delay;
}
return endpoint;
}
private void HandleNewEndpoint(Endpoint endpoint)
{
if (_tasks.TryAdd(endpoint, CreateTaskFor(endpoint)))
_onNewTask.Set();
}
public void GetEndpoint_should_return_endpoint_information_with_details(HealthStatus status)
{
Guid id = Guid.NewGuid();
var monitor = MonitorMock.GetMock("monitor");
var healthInfo = new HealthInfo(status, TimeSpan.FromSeconds(2), new Dictionary<string, string> { { "a", "b" }, { "c", "d" } });
monitor.Setup(p => p.CheckHealthAsync("address", It.IsAny<CancellationToken>())).Returns(Task.FromResult(healthInfo));
var endpoint = new Endpoint(id, monitor.Object, "address", "name", "group");
endpoint.CheckHealth(new CancellationToken(), MonitorSettingsHelper.ConfigureDefaultSettings()).Wait();
_endpointRegistry.Setup(r => r.GetById(id)).Returns(endpoint);
var result = _controller.GetEndpoint(id) as OkNegotiatedContentResult<EndpointDetails>;
Assert.NotNull(result);
AssertEndpoint(endpoint, result.Content);
Assert.Equal(status.ToString(), result.Content.Status.ToString());
Assert.NotNull(result.Content.LastCheckUtc);
Assert.Equal(healthInfo.ResponseTime, result.Content.LastResponseTime);
Assert.Equal(healthInfo.Details, result.Content.Details);
}
private static bool RequiresShortTimeout(Endpoint endpoint)
{
return endpoint.Health == null ||
(endpoint.Health.Status == EndpointStatus.Healthy || endpoint.Health.Status == EndpointStatus.NotRun ||
endpoint.Health.Status == EndpointStatus.Offline || endpoint.Health.Status == EndpointStatus.NotExists);
}
private static void AssertEndpoint(Endpoint expected, EndpointDetails actual)
{
Assert.Equal(expected.MonitorType, actual.MonitorType);
Assert.Equal(expected.Address, actual.Address);
Assert.Equal(expected.Name, actual.Name);
Assert.Equal(expected.Group, actual.Group);
Assert.Equal(expected.Id, actual.Id);
}
private static void LogHealthStatus(Endpoint endpoint, EndpointHealth endpointHealth)
{
switch (endpointHealth.Status)
{
case EndpointStatus.TimedOut:
case EndpointStatus.Unhealthy:
Logger.WarnFormat("Status of {0}: Status={1}, ResponseTime={2}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime);
break;
case EndpointStatus.Faulty:
Logger.ErrorFormat("Status of {0}: Status={1}, ResponseTime={2}, Details={3}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime, endpointHealth.PrettyFormatDetails());
break;
default:
Logger.InfoFormat("Status of {0}: Status={1}, ResponseTime={2}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime);
break;
}
}
private Func<Endpoint, CancellationToken, Task> CaptureGuardTask(Endpoint endpoint)
{
_endpointRegistry.Setup(r => r.Endpoints).Returns(Enumerable.Repeat(endpoint, 1));
Func<Endpoint, CancellationToken, Task> capturedTaskFactory = null;
_taskExecutor.Setup(e => e.TryRegisterTaskFor(endpoint, It.IsAny<Func<Endpoint, CancellationToken, Task>>()))
.Returns((Endpoint e, Func<Endpoint, CancellationToken, Task> taskFactory) =>
{
capturedTaskFactory = taskFactory;
return true;
});
using (CreateGuard()) { }
Assert.True(capturedTaskFactory != null, "It should capture monitor task");
return capturedTaskFactory;
}
private void VerifyTaskWasRegisteredForEndpointInGuard(Endpoint endpoint, Func<Times> times)
{
_taskExecutor.Verify(e => e.TryRegisterTaskFor(endpoint, It.IsAny<Func<Endpoint, CancellationToken, Task>>()), times);
}
private TimeSpan GetTimeSpanSinceLastEndpointUpdate(Endpoint endpoint)
{
return _timeCoordinator.Object.UtcNow - (endpoint.Health?.CheckTimeUtc ?? endpoint.LastModifiedTimeUtc);
}