public void Dispose_should_mark_endpoint_disposed()
{
var endpoint = new MonitorableEndpoint(new EndpointIdentity(Guid.Empty, "monitor", "address"), MonitorMock.Mock("monitor"));
Assert.False(endpoint.IsDisposed);
endpoint.Dispose();
Assert.True(endpoint.IsDisposed);
}
public void Dispose_should_mark_endpoint_disposed()
{
var endpoint = new MonitorableEndpoint(new EndpointIdentity(Guid.Empty, "monitor", "address"), MonitorMock.Mock("monitor"));
Assert.False(endpoint.IsDisposed);
endpoint.Dispose();
Assert.True(endpoint.IsDisposed);
}
public async Task<EndpointHealth> CheckHealthAsync(MonitorableEndpoint endpoint, CancellationToken cancellationToken)
{
Logger.DebugFormat("Checking health of {0}...", endpoint);
var endpointHealth = await PerformHealthCheckAsync(cancellationToken, endpoint);
LogHealthStatus(endpoint, endpointHealth);
_healthUpdateListener.UpdateHealth(endpoint.Identity.Id, endpointHealth);
return endpointHealth;
}
public async Task <EndpointHealth> CheckHealthAsync(MonitorableEndpoint endpoint, CancellationToken cancellationToken)
{
Logger.DebugFormat("Checking health of {0}...", endpoint);
var endpointHealth = await PerformHealthCheckAsync(cancellationToken, endpoint);
LogHealthStatus(endpoint, endpointHealth);
_healthUpdateListener.UpdateHealth(endpoint.Identity.Id, endpointHealth);
return(endpointHealth);
}
public async Task CheckHealth_should_update_the_endpoint_with_its_health_status()
{
var endpoint = new MonitorableEndpoint(new EndpointIdentity(Guid.Empty, "monitor", "address"), MonitorMock.Mock("monitor"));
var sampler = new Mock<IHealthSampler>();
var token = new CancellationToken();
var result = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
sampler.Setup(s => s.CheckHealthAsync(endpoint, token)).Returns(Task.FromResult(result));
await endpoint.CheckHealth(sampler.Object, token);
Assert.Same(result, endpoint.Health);
}
public async Task CheckHealth_should_update_the_endpoint_with_its_health_status()
{
var endpoint = new MonitorableEndpoint(new EndpointIdentity(Guid.Empty, "monitor", "address"), MonitorMock.Mock("monitor"));
var sampler = new Mock <IHealthSampler>();
var token = new CancellationToken();
var result = new EndpointHealth(DateTime.UtcNow, TimeSpan.Zero, EndpointStatus.Healthy);
sampler.Setup(s => s.CheckHealthAsync(endpoint, token)).Returns(Task.FromResult(result));
await endpoint.CheckHealth(sampler.Object, token);
Assert.Same(result, endpoint.Health);
}
private static Mock<IHealthSampler> SetupSampler(MonitorableEndpoint endpoint, ManualResetEventSlim manualReset)
{
var sampler = new Mock<IHealthSampler>();
sampler
.Setup(s => s.CheckHealthAsync(endpoint, It.IsAny<CancellationToken>()))
.Returns(async (MonitorableEndpoint e, CancellationToken ct) =>
{
await Task.Run(() => manualReset.Wait(ct));
return null;
});
return sampler;
}
private static Mock <IHealthSampler> SetupSampler(MonitorableEndpoint endpoint, ManualResetEventSlim manualReset)
{
var sampler = new Mock <IHealthSampler>();
sampler
.Setup(s => s.CheckHealthAsync(endpoint, It.IsAny <CancellationToken>()))
.Returns(async(MonitorableEndpoint e, CancellationToken ct) =>
{
await Task.Run(() => manualReset.Wait(ct));
return(null);
});
return(sampler);
}
private async Task <HealthInfo> ConfigureTimeoutTaskAsync(MonitorableEndpoint endpoint, CancellationToken cancellationToken)
{
if (RequiresShortTimeout(endpoint))
{
await _timeCoordinator.Delay(_settings.ShortTimeOut, cancellationToken);
return(new HealthInfo(HealthStatus.TimedOut, TimeoutDetails));
}
await _timeCoordinator.Delay(_settings.FailureTimeOut, cancellationToken);
return(new HealthInfo(HealthStatus.Faulty, TimeoutDetails));
}
public HealthSamplerTests()
{
var settings = SetupSettings();
_healthUpdateListener = new Mock<IEndpointHealthUpdateListener>();
_timeCoordinator = new Mock<ITimeCoordinator>();
_timeCoordinator.Setup(c => c.UtcNow).Returns(_utcNow);
_sampler = new HealthSampler(settings.Object, _healthUpdateListener.Object, _timeCoordinator.Object);
_monitor = new Mock<IHealthMonitor>();
_monitor.Setup(m => m.Name).Returns("monitor");
_endpoint = new MonitorableEndpoint(new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), _monitor.Object);
_awaitableFactory = new AwaitableFactory();
}
private static async Task<HealthInfo> PerformHealthCheckAsync(MonitorableEndpoint endpoint, CancellationTokenSource timeoutToken)
{
try
{
return await endpoint.Monitor.CheckHealthAsync(endpoint.Identity.Address, timeoutToken.Token);
}
catch (OperationCanceledException) when (timeoutToken.IsCancellationRequested)
{
throw;
}
catch (Exception e)
{
throw new AggregateException("Health check failed", e);
}
}
public HealthSamplerTests()
{
var settings = SetupSettings();
_healthUpdateListener = new Mock <IEndpointHealthUpdateListener>();
_timeCoordinator = new Mock <ITimeCoordinator>();
_timeCoordinator.Setup(c => c.UtcNow).Returns(_utcNow);
_sampler = new HealthSampler(settings.Object, _healthUpdateListener.Object, _timeCoordinator.Object);
_monitor = new Mock <IHealthMonitor>();
_monitor.Setup(m => m.Name).Returns("monitor");
_endpoint = new MonitorableEndpoint(new EndpointIdentity(Guid.NewGuid(), "monitor", "address"), _monitor.Object);
_awaitableFactory = new AwaitableFactory();
}
private static async Task <HealthInfo> PerformHealthCheckAsync(MonitorableEndpoint endpoint, CancellationTokenSource timeoutToken)
{
try
{
return(await endpoint.Monitor.CheckHealthAsync(endpoint.Identity.Address, timeoutToken.Token));
}
catch (OperationCanceledException) when(timeoutToken.IsCancellationRequested)
{
throw;
}
catch (Exception e)
{
throw new AggregateException("Health check failed", e);
}
}
public void TryRegister_should_return_existing_endpoint_and_not_emit_NewEndpointAdded_event_if_endpoint_of_given_id_is_already_registered()
{
MockMonitor("monitor");
var endpointId = Guid.NewGuid();
var endpoint1 = _registry.TryRegister(new EndpointIdentity(endpointId, "monitor", "address"));
MonitorableEndpoint newEndpointCapture = null;
_registry.NewEndpointAdded += e => { newEndpointCapture = e; };
var endpoint2 = _registry.TryRegister(new EndpointIdentity(endpointId, "monitor", "address"));
Assert.Same(endpoint1, endpoint2);
Assert.Null(newEndpointCapture);
}
private void SetupEndpointMonitorOnTimeLineToRunNTimes(MonitorableEndpoint endpoint, string eventName, int noOfHealthChecks, CancellationToken cancellationToken)
{
int repeats = 0;
_mockHealthMonitor.Setup(m => m.CheckHealthAsync(endpoint.Identity.Address, cancellationToken))
.Returns(() => _awaitableFactory.Execute(() =>
{
if (++repeats >= noOfHealthChecks)
{
endpoint.Dispose();
}
return(new HealthInfo(HealthStatus.Healthy));
})
.WithTimeline(eventName).RunAsync());
}
public MonitorableEndpoint TryRegister(EndpointIdentity identity)
{
var monitor = _healthMonitorRegistry.FindByName(identity.MonitorType);
if (monitor == null)
return null;
var monitorableEndpoint = new MonitorableEndpoint(identity, monitor);
var current = _endpoints.GetOrAdd(identity.Id, monitorableEndpoint);
if (ReferenceEquals(current, monitorableEndpoint))
{
Logger.Info($"Starting monitoring of: {monitorableEndpoint}");
NewEndpointAdded?.Invoke(monitorableEndpoint);
}
return current;
}
public void TryRegister_should_register_new_endpoint_and_emit_NewEndpointAdded_event()
{
MockMonitor("monitor");
MonitorableEndpoint capturedEndpoint = null;
_registry.NewEndpointAdded += e => { capturedEndpoint = e; };
var endpointIdentity = new EndpointIdentity(Guid.NewGuid(), "monitor", "address");
var endpoint = _registry.TryRegister(endpointIdentity);
Assert.NotNull(endpoint);
Assert.Same(endpoint, capturedEndpoint);
Assert.Equal("monitor", endpoint.Identity.MonitorType);
Assert.Equal("address", endpoint.Identity.Address);
Assert.Equal(endpointIdentity.Id, endpoint.Identity.Id);
}
private static void LogHealthStatus(MonitorableEndpoint endpoint, EndpointHealth endpointHealth)
{
switch (endpointHealth.Status)
{
case EndpointStatus.TimedOut:
case EndpointStatus.Unhealthy:
Logger.WarnFormat("Status of {0}: Status={1}, ResponseTime={2}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime);
break;
case EndpointStatus.Faulty:
Logger.ErrorFormat("Status of {0}: Status={1}, ResponseTime={2}, Details={3}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime, endpointHealth.PrettyFormatDetails());
break;
default:
Logger.InfoFormat("Status of {0}: Status={1}, ResponseTime={2}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime);
break;
}
}
public MonitorableEndpoint TryRegister(EndpointIdentity identity)
{
var monitor = _healthMonitorRegistry.FindByName(identity.MonitorType);
if (monitor == null)
{
return(null);
}
var monitorableEndpoint = new MonitorableEndpoint(identity, monitor);
var current = _endpoints.GetOrAdd(identity.Id, monitorableEndpoint);
if (ReferenceEquals(current, monitorableEndpoint))
{
Logger.Info($"Starting monitoring of: {monitorableEndpoint}");
NewEndpointAdded?.Invoke(monitorableEndpoint);
}
return(current);
}
private async Task<EndpointHealth> PerformHealthCheckAsync(CancellationToken cancellationToken, MonitorableEndpoint endpoint)
{
var checkTimeUtc = _timeCoordinator.UtcNow;
var timer = _timeCoordinator.CreateStopWatch();
using (var timeoutToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
try
{
timer.Start();
var timeoutTask = ConfigureTimeoutTaskAsync(endpoint, cancellationToken);
var healthTask = PerformHealthCheckAsync(endpoint, timeoutToken);
var healthResult = await Task.WhenAny(healthTask, timeoutTask);
timer.Stop();
await CancelHealthTaskIfNeededAsync(healthTask, timeoutToken);
return FromResult(checkTimeUtc, timer.Elapsed, healthResult.Result);
}
catch (OperationCanceledException) when (timeoutToken.IsCancellationRequested)
{
throw;
}
catch (AggregateException e) when (timeoutToken.IsCancellationRequested && e.Flatten().InnerExceptions.All(ex => ex is OperationCanceledException))
{
throw;
}
catch (AggregateException e)
{
return FromException(checkTimeUtc, timer.Elapsed, e.Flatten().InnerExceptions.First());
}
catch (Exception e)
{
return FromException(checkTimeUtc, timer.Elapsed, e);
}
}
}
private static bool RequiresShortTimeout(MonitorableEndpoint endpoint)
{
return endpoint.Health == null ||
(endpoint.Health.Status == EndpointStatus.Healthy || endpoint.Health.Status == EndpointStatus.NotRun ||
endpoint.Health.Status == EndpointStatus.Offline || endpoint.Health.Status == EndpointStatus.NotExists);
}
private void SetupEndpointMonitorOnTimeLineToRunNTimes(MonitorableEndpoint endpoint, string eventName, int noOfHealthChecks, CancellationToken cancellationToken)
{
int repeats = 0;
_mockHealthMonitor.Setup(m => m.CheckHealthAsync(endpoint.Identity.Address, cancellationToken))
.Returns(() => _awaitableFactory.Execute(() =>
{
if (++repeats >= noOfHealthChecks)
endpoint.Dispose();
return new HealthInfo(HealthStatus.Healthy);
})
.WithTimeline(eventName).RunAsync());
}
private async Task <EndpointHealth> PerformHealthCheckAsync(CancellationToken cancellationToken, MonitorableEndpoint endpoint)
{
var checkTimeUtc = _timeCoordinator.UtcNow;
var timer = _timeCoordinator.CreateStopWatch();
using (var timeoutToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
try
{
timer.Start();
var timeoutTask = ConfigureTimeoutTaskAsync(endpoint, cancellationToken);
var healthTask = PerformHealthCheckAsync(endpoint, timeoutToken);
var healthResult = await Task.WhenAny(healthTask, timeoutTask);
timer.Stop();
await CancelHealthTaskIfNeededAsync(healthTask, timeoutToken);
return(FromResult(checkTimeUtc, timer.Elapsed, healthResult.Result));
}
catch (OperationCanceledException) when(timeoutToken.IsCancellationRequested)
{
throw;
}
catch (AggregateException e) when(timeoutToken.IsCancellationRequested && e.Flatten().InnerExceptions.All(ex => ex is OperationCanceledException))
{
throw;
}
catch (AggregateException e)
{
return(FromException(checkTimeUtc, timer.Elapsed, e.Flatten().InnerExceptions.First()));
}
catch (Exception e)
{
return(FromException(checkTimeUtc, timer.Elapsed, e));
}
}
}
private Func <MonitorableEndpoint, CancellationToken, Task> CaptureMonitorTask(MonitorableEndpoint endpoint)
{
Func <MonitorableEndpoint, CancellationToken, Task> capturedTaskFactory = null;
_mockExecutor.Setup(e => e.TryRegisterTaskFor(endpoint, It.IsAny <Func <MonitorableEndpoint, CancellationToken, Task> >()))
.Returns((MonitorableEndpoint e, Func <MonitorableEndpoint, CancellationToken, Task> taskFactory) =>
{
capturedTaskFactory = taskFactory;
return(true);
});
using (CreateEndpointMonitor()) { }
Assert.True(capturedTaskFactory != null, "It should capture monitor task");
return(capturedTaskFactory);
}
private static bool RequiresShortTimeout(MonitorableEndpoint endpoint)
{
return(endpoint.Health == null ||
(endpoint.Health.Status == EndpointStatus.Healthy || endpoint.Health.Status == EndpointStatus.NotRun ||
endpoint.Health.Status == EndpointStatus.Offline || endpoint.Health.Status == EndpointStatus.NotExists));
}
private Func<MonitorableEndpoint, CancellationToken, Task> CaptureMonitorTask(MonitorableEndpoint endpoint)
{
Func<MonitorableEndpoint, CancellationToken, Task> capturedTaskFactory = null;
_mockExecutor.Setup(e => e.TryRegisterTaskFor(endpoint, It.IsAny<Func<MonitorableEndpoint, CancellationToken, Task>>()))
.Returns((MonitorableEndpoint e, Func<MonitorableEndpoint, CancellationToken, Task> taskFactory) =>
{
capturedTaskFactory = taskFactory;
return true;
});
using (CreateEndpointMonitor()) { }
Assert.True(capturedTaskFactory != null, "It should capture monitor task");
return capturedTaskFactory;
}
private async Task<HealthInfo> ConfigureTimeoutTaskAsync(MonitorableEndpoint endpoint, CancellationToken cancellationToken)
{
if (RequiresShortTimeout(endpoint))
{
await _timeCoordinator.Delay(_settings.ShortTimeOut, cancellationToken);
return new HealthInfo(HealthStatus.TimedOut, TimeoutDetails);
}
await _timeCoordinator.Delay(_settings.FailureTimeOut, cancellationToken);
return new HealthInfo(HealthStatus.Faulty, TimeoutDetails);
}
private void VerifyTaskWasRegisteredForEndpointInMonitor(MonitorableEndpoint endpoint, Func <Times> times)
{
_mockExecutor.Verify(e => e.TryRegisterTaskFor(endpoint, It.IsAny <Func <MonitorableEndpoint, CancellationToken, Task> >()), times);
}
private static void LogHealthStatus(MonitorableEndpoint endpoint, EndpointHealth endpointHealth)
{
switch (endpointHealth.Status)
{
case EndpointStatus.TimedOut:
case EndpointStatus.Unhealthy:
Logger.WarnFormat("Status of {0}: Status={1}, ResponseTime={2}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime);
break;
case EndpointStatus.Faulty:
Logger.ErrorFormat("Status of {0}: Status={1}, ResponseTime={2}, Details={3}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime, endpointHealth.PrettyFormatDetails());
break;
default:
Logger.InfoFormat("Status of {0}: Status={1}, ResponseTime={2}", endpoint, endpointHealth.Status, endpointHealth.ResponseTime);
break;
}
}
public async Task<EndpointHealth> CheckHealthAsync(MonitorableEndpoint endpoint, CancellationToken cancellationToken)
{
using (await Begin(endpoint.Monitor.Name, cancellationToken))
return await _sampler.CheckHealthAsync(endpoint, cancellationToken);
}
public async Task <EndpointHealth> CheckHealthAsync(MonitorableEndpoint endpoint, CancellationToken cancellationToken)
{
await endpoint.Monitor.CheckHealthAsync(endpoint.Identity.Address, cancellationToken);
return(new EndpointHealth(DateTime.MinValue, TimeSpan.Zero, EndpointStatus.Healthy));
}
private void VerifyTaskWasRegisteredForEndpointInMonitor(MonitorableEndpoint endpoint, Func<Times> times)
{
_mockExecutor.Verify(e => e.TryRegisterTaskFor(endpoint, It.IsAny<Func<MonitorableEndpoint, CancellationToken, Task>>()), times);
}
private async Task <EndpointHealth> PerformHealthCheckAsync(CancellationToken cancellationToken, MonitorableEndpoint endpoint)
{
var checkTimeUtc = _timeCoordinator.UtcNow;
var timer = _timeCoordinator.CreateStopWatch();
try
{
using (var timeoutToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
timer.Start();
var timeoutTask = ConfigureTimeoutTaskAsync(endpoint, cancellationToken);
var healthTask = endpoint.Monitor.CheckHealthAsync(endpoint.Identity.Address, timeoutToken.Token);
var healthResult = await Task.WhenAny(healthTask, timeoutTask);
timer.Stop();
await CancelHealthTaskIfNeededAsync(healthTask, timeoutToken);
return(FromResult(checkTimeUtc, timer.Elapsed, healthResult.Result));
}
}
catch (OperationCanceledException)
{
throw;
}
catch (AggregateException e)
{
if (e.InnerExceptions.First() is OperationCanceledException)
{
throw;
}
return(FromException(checkTimeUtc, timer.Elapsed, e.InnerExceptions.First()));
}
catch (Exception e)
{
return(FromException(checkTimeUtc, timer.Elapsed, e));
}
}
public async Task<EndpointHealth> CheckHealthAsync(MonitorableEndpoint endpoint, CancellationToken cancellationToken)
{
await endpoint.Monitor.CheckHealthAsync(endpoint.Identity.Address, cancellationToken);
return new EndpointHealth(DateTime.MinValue, TimeSpan.Zero, EndpointStatus.Healthy);
}
public async Task <EndpointHealth> CheckHealthAsync(MonitorableEndpoint endpoint, CancellationToken cancellationToken)
{
using (await Begin(endpoint.Monitor.Name, cancellationToken))
return(await _sampler.CheckHealthAsync(endpoint, cancellationToken));
}
