public async Task Notifier_should_send_current_health_to_the_API()
{
var endpointId = Guid.NewGuid();
var expectedHealth = new EndpointHealth(HealthStatus.Offline, new Dictionary <string, string> {
{ "key", "value" }
});
HealthUpdate lastCaptured = null;
var countdown = new AsyncCountdown("update", 5);
SetupHealthCheckInterval(TimeSpan.FromMilliseconds(1));
SetupEndpointRegistration(endpointId);
_mockClient.Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny <HealthUpdate>(), It.IsAny <CancellationToken>()))
.Returns((Guid id, string authToken, HealthUpdate upd, CancellationToken token) => _awaitableFactory
.Execute(() => lastCaptured = upd)
.WithCountdown(countdown)
.RunAsync());
using (CreateNotifier(token => Task.FromResult(expectedHealth)))
await countdown.WaitAsync(TestMaxTime);
Assert.NotNull(lastCaptured);
Assert.Equal(expectedHealth.Status, lastCaptured.Status);
Assert.Equal(expectedHealth.Details, lastCaptured.Details);
Assert.True(lastCaptured.CheckTimeUtc > DateTime.UtcNow.AddMinutes(-1) && lastCaptured.CheckTimeUtc < DateTime.UtcNow.AddMinutes(1));
}
public async Task Notifier_should_register_endpoint_with_all_details_and_specified_host()
{
EndpointDefinition captured = null;
var countdown = new AsyncCountdown("register", 1);
var endpointId = Guid.NewGuid();
_mockClient
.Setup(c => c.RegisterEndpointAsync(It.IsAny <EndpointDefinition>(), It.IsAny <CancellationToken>()))
.Returns((EndpointDefinition def, CancellationToken token) => _awaitableFactory
.Execute(() => { captured = def; return(endpointId); })
.WithCountdown(countdown)
.RunAsync());
SetupHealthCheckInterval(TimeSpan.FromMilliseconds(1));
Action <IEndpointDefintionBuilder> builder = b => b.DefineName("endpointName")
.DefineGroup("endpointGroup")
.DefineTags("t1")
.DefineAddress("host", "uniqueName")
.DefinePassword(AuthenticationToken);
using (CreateNotifier(builder, token => Task.FromResult(new EndpointHealth(HealthStatus.Offline))))
await countdown.WaitAsync(TestMaxTime);
Assert.NotNull(captured);
Assert.Equal("endpointName", captured.EndpointName);
Assert.Equal("endpointGroup", captured.GroupName);
Assert.Equal(new[] { "t1" }, captured.Tags);
Assert.Equal("host:uniqueName", captured.Address);
}
public async Task Exchange_should_try_download_list_of_endpoints_periodically()
{
SetupDefaultRegisterEndpointsMock();
var endpointIdentities1 = new[] { new EndpointIdentity(Guid.NewGuid(), MonitorTypeName, "address1"), new EndpointIdentity(Guid.NewGuid(), MonitorTypeName, "address2") };
var endpointIdentities2 = new[] { new EndpointIdentity(Guid.NewGuid(), MonitorTypeName, "address1"), new EndpointIdentity(Guid.NewGuid(), MonitorTypeName, "address3") };
var countdown1 = new AsyncCountdown("endpointIdentities1", 1);
var countdown2 = new AsyncCountdown("endpointIdentities2", 1);
var monitorTag = "some_tag";
SetupExchangeClient(c => c.GetEndpointIdentitiesAsync(monitorTag, It.IsAny <CancellationToken>()),
() => { throw new InvalidOperationException(); },
() => _awaitableFactory.Return(endpointIdentities1).WithCountdown(countdown1).RunAsync(),
() => { throw new InvalidOperationException(); },
() => _awaitableFactory.Return(endpointIdentities2).WithCountdown(countdown2).RunAsync());
using (CreateExchange(new DataExchangeConfig(100, 10, TimeSpan.FromSeconds(1), TimeSpan.FromMilliseconds(1), monitorTag)))
{
await countdown1.WaitAsync(TestMaxWaitTime);
await countdown2.WaitAsync(TestMaxWaitTime);
}
}
public async Task Exchange_should_upload_monitor_healths_in_buckets()
{
SetupDefaultRegisterEndpointsMock();
var identity = SetupDefaultEndpointIdentitiesMock().First();
int bucketCount = 10;
int bucketSize = 10;
var uploads = new List <int>();
var countdown = new AsyncCountdown(nameof(IHealthMonitorExchangeClient.UploadHealthAsync), bucketCount);
SetupDefaultHealthUploadMock(x => { uploads.Add(x.Length); countdown.Decrement(); });
var healths = PrepareEndpointHealths(bucketSize * bucketCount);
using (var ex = CreateExchange(new DataExchangeConfig(bucketSize * bucketCount, bucketSize, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1), EndpointMetadata.DefaultMonitorTag)))
{
foreach (var health in healths)
{
ex.UpdateHealth(identity.Id, health);
}
await countdown.WaitAsync(TestMaxWaitTime);
}
Assert.Equal(new[] { bucketSize }, uploads.Distinct());
}
public async Task Notifier_should_register_endpoint_again_if_notification_failed_with_EndpointNotFoundException()
{
var oldEndpointId = Guid.NewGuid();
var newEndpointId = Guid.NewGuid();
var oldEndpointCountdown = new AsyncCountdown("oldEndpoint", 1);
var newEndpointCountdown = new AsyncCountdown("newEndpoint", 1);
_mockClient
.SetupSequence(c => c.RegisterEndpointAsync(It.IsAny <EndpointDefinition>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(oldEndpointId))
.Returns(Task.FromResult(newEndpointId));
_mockClient
.Setup(c => c.SendHealthUpdateAsync(oldEndpointId, AuthenticationToken, It.IsAny <HealthUpdate>(), It.IsAny <CancellationToken>()))
.Returns(() => _awaitableFactory
.Throw(new EndpointNotFoundException())
.WithCountdown(oldEndpointCountdown)
.RunAsync());
_mockClient
.Setup(c => c.SendHealthUpdateAsync(newEndpointId, AuthenticationToken, It.IsAny <HealthUpdate>(), It.IsAny <CancellationToken>()))
.Returns(() => _awaitableFactory
.Return()
.WithCountdown(newEndpointCountdown)
.RunAsync());
using (CreateNotifier())
{
await oldEndpointCountdown.WaitAsync(TestMaxTime);
await newEndpointCountdown.WaitAsync(TestMaxTime);
}
}
public async Task Executor_should_execute_tasks_until_disposal()
{
var task1Name = "task1";
var task2Name = "task2";
var countdown1 = new AsyncCountdown(task1Name, 10);
var counter1 = new AsyncCounter();
var countdown2 = new AsyncCountdown(task2Name, 10);
var counter2 = new AsyncCounter();
using (var executor = ContinuousTaskExecutor <string> .StartExecutor(Mock.Of <ITimeCoordinator>()))
{
Assert.True(executor.TryRegisterTaskFor(task1Name, (item, token) => StartTaskAsync(countdown1, counter1, token)));
await countdown1.WaitAsync(_testTimeout);
Assert.True(executor.TryRegisterTaskFor(task2Name, (item, token) => StartTaskAsync(countdown2, counter2, token)));
await countdown2.WaitAsync(_testTimeout);
// check that task 1 still works
await countdown1.ResetTo(10).WaitAsync(_testTimeout);
}
var expected1 = counter1.Value;
var expected2 = counter2.Value;
await Task.Delay(200);
Assert.Equal(expected1, counter1.Value);
Assert.Equal(expected2, counter2.Value);
}
public async Task Executor_should_continue_processing_other_tasks_when_one_finish()
{
var task1Name = "task1";
var task2Name = "task2";
var countdown1 = new AsyncCountdown(task1Name, 10);
var task2Finished = new SemaphoreSlim(0);
using (var executor = ContinuousTaskExecutor <string> .StartExecutor(Mock.Of <ITimeCoordinator>()))
{
executor.FinishedTaskFor += item =>
{
if (item == task2Name)
{
task2Finished.Release();
}
};
Assert.True(executor.TryRegisterTaskFor(task1Name, (item, token) => StartTaskAsync(token, b => b.WithCountdown(countdown1))));
await countdown1.WaitAsync(_testTimeout);
Assert.True(executor.TryRegisterTaskFor(task2Name, (item, token) => Task.Delay(25, token)));
await task2Finished.WaitAsync(_testTimeout);
// check that task 1 still works
await countdown1.ResetTo(10).WaitAsync(_testTimeout);
}
}
public async Task Notifier_should_survive_communication_errors_and_eventually_restore_connectivity()
{
var healthUpdateCountdown = new AsyncCountdown("update", 10);
var healthUpdateCountdown2 = new AsyncCountdown("update2", 10);
var registrationCountdown = new AsyncCountdown("registration", 10);
var intervalCountdown = new AsyncCountdown("interval", 10);
var delayCountdown = new AsyncCountdown("delay", 10);
var healthCheckInterval = TimeSpan.FromMilliseconds(127);
var endpointId = Guid.NewGuid();
var workingHealthUpdate = _awaitableFactory.Return().WithCountdown(healthUpdateCountdown2);
var notWorkingHealthUpdate = _awaitableFactory.Throw(new TaskCanceledException()).WithCountdown(healthUpdateCountdown);
var currentHealthUpdate = notWorkingHealthUpdate;
var notWorkingRegistration = _awaitableFactory.Throw <Guid>(new TaskCanceledException()).WithCountdown(registrationCountdown);
var workingRegistration = _awaitableFactory.Return(endpointId);
var currentRegistration = notWorkingRegistration;
var notWorkingHealthCheckInterval = _awaitableFactory.Throw <TimeSpan>(new TaskCanceledException()).WithCountdown(intervalCountdown);
var workingHealthCheckInterval = _awaitableFactory.Return(healthCheckInterval);
var currentHealthCheckInterval = notWorkingHealthCheckInterval;
_mockClient.Setup(c => c.SendHealthUpdateAsync(It.IsAny <Guid>(), AuthenticationToken, It.IsAny <HealthUpdate>(), It.IsAny <CancellationToken>()))
.Returns(() => currentHealthUpdate.RunAsync());
_mockClient.Setup(c => c.RegisterEndpointAsync(It.IsAny <EndpointDefinition>(), It.IsAny <CancellationToken>()))
.Returns(() => currentRegistration.RunAsync());
_mockClient.Setup(c => c.GetHealthCheckIntervalAsync(It.IsAny <CancellationToken>()))
.Returns(() => currentHealthCheckInterval.RunAsync());
_mockTimeCoordinator.Setup(c => c.Delay(healthCheckInterval, It.IsAny <CancellationToken>()))
.Returns(() => _awaitableFactory.Return().WithCountdown(delayCountdown).RunAsync());
using (CreateNotifier())
{
await registrationCountdown.WaitAsync(TestMaxTime);
currentRegistration = workingRegistration;
await healthUpdateCountdown.WaitAsync(TestMaxTime);
currentHealthUpdate = workingHealthUpdate;
await healthUpdateCountdown2.WaitAsync(TestMaxTime);
await intervalCountdown.WaitAsync(TestMaxTime);
currentHealthCheckInterval = workingHealthCheckInterval;
await delayCountdown.WaitAsync(TestMaxTime);
}
}
public async Task Exchange_should_download_list_of_endpoints()
{
SetupDefaultRegisterEndpointsMock();
var endpointIdentitiesGetCountdown = new AsyncCountdown(nameof(IHealthMonitorExchangeClient.GetEndpointIdentitiesAsync), 1);
var ids = SetupDefaultEndpointIdentitiesMock(cfg => cfg.WithCountdown(endpointIdentitiesGetCountdown));
using (CreateExchange())
await endpointIdentitiesGetCountdown.WaitAsync(TimeSpan.FromSeconds(5));
_endpointRegistry.Verify(r => r.UpdateEndpoints(ids));
}
public async Task Notifier_should_retry_sending_health_updates_according_to_recommened_backOff_strategy_in_case_of_exceptions()
{
var endpointId = Guid.NewGuid();
SetupEndpointRegistration(endpointId);
var checkInterval = TimeSpan.FromMilliseconds(127);
SetupHealthCheckInterval(checkInterval);
var minRepeats = 10;
var countdown = new AsyncCountdown("update", minRepeats);
var updates = new ConcurrentQueue <HealthUpdate>();
var backOffPlan = new BackOffPlan(null, false);
_mockClient
.Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny <HealthUpdate>(), It.IsAny <CancellationToken>()))
.Returns((Guid id, string authToken, HealthUpdate upd, CancellationToken token) =>
{
updates.Enqueue(upd);
return(_awaitableFactory
.Throw(new InvalidOperationException())
.WithCountdown(countdown)
.RunAsync());
});
using (CreateNotifier())
await countdown.WaitAsync(TestMaxTime);
int expectedSeconds = 1;
for (int i = 0; i < minRepeats; ++i)
{
backOffPlan = new BackOffPlan(TimeSpan.FromSeconds(Math.Min(expectedSeconds, MaxEndpointNotifierRetryDelayInSecs)), true);
_mockBackOffStategy
.Setup(b => b.GetCurrent(TimeSpan.FromSeconds(expectedSeconds), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(backOffPlan));
_mockTimeCoordinator.Verify(c => c.Delay(TimeSpan.FromSeconds(expectedSeconds), It.IsAny <CancellationToken>()));
expectedSeconds = Math.Min(expectedSeconds *= 2, MaxEndpointNotifierRetryDelayInSecs);
}
_mockTimeCoordinator.Verify(c => c.Delay(checkInterval, It.IsAny <CancellationToken>()), Times.Once);
Assert.Equal(1, updates.Distinct().Count());
}
public async Task Exchange_should_try_upload_supported_monitor_types_on_start_until_they_succeeds()
{
var countdown = new AsyncCountdown(nameof(IHealthMonitorExchangeClient.RegisterMonitorsAsync), 1);
SetupExchangeClient(
c => c.RegisterMonitorsAsync(It.IsAny <IEnumerable <string> >(), It.IsAny <CancellationToken>()),
() => { throw new InvalidOperationException(); },
() => { throw new InvalidOperationException(); },
() => _awaitableFactory.Return().WithCountdown(countdown).RunAsync()
);
SetupDefaultEndpointIdentitiesMock();
using (CreateExchange(new DataExchangeConfig(100, 10, TimeSpan.FromMilliseconds(1), TimeSpan.FromSeconds(1), EndpointMetadata.DefaultMonitorTag)))
await countdown.WaitAsync(TestMaxWaitTime);
}
public async Task Exchange_should_try_download_list_of_endpoints_periodically_until_disposal()
{
var countdown = new AsyncCountdown(nameof(IHealthMonitorExchangeClient.RegisterMonitorsAsync), 10);
var counter = new AsyncCounter();
SetupDefaultRegisterEndpointsMock();
_exchangeClient.Setup(c => c.GetEndpointIdentitiesAsync(EndpointMetadata.DefaultMonitorTag, It.IsAny <CancellationToken>()))
.Returns(() => _awaitableFactory.Throw <EndpointIdentity[]>(new InvalidOperationException()).WithCountdown(countdown).WithCounter(counter).RunAsync());
using (CreateExchange(new DataExchangeConfig(100, 10, TimeSpan.FromSeconds(1), TimeSpan.FromMilliseconds(1), EndpointMetadata.DefaultMonitorTag)))
await countdown.WaitAsync(TestMaxWaitTime);
var capturedCalls = counter.Value;
Assert.True(capturedCalls > 0, "capturedCalls>0");
await Task.Delay(PostRunDelay);
Assert.Equal(capturedCalls, counter.Value);
}
public async Task Executor_should_immediately_break_the_loop_on_cancelation()
{
var timeCoordinator = new Mock <ITimeCoordinator>();
var countdown = new AsyncCountdown("task", 1);
var taskNotCancelled = false;
using (var executor = ContinuousTaskExecutor <string> .StartExecutor(timeCoordinator.Object))
{
executor.TryRegisterTaskFor("task", async(item, token) =>
{
countdown.Decrement();
await Task.Delay(_testTimeout, token);
taskNotCancelled = true;
});
await countdown.WaitAsync(_testTimeout);
}
Assert.False(taskNotCancelled, "Task was not cancelled");
timeCoordinator.Verify(c => c.Delay(It.IsAny <TimeSpan>(), It.IsAny <CancellationToken>()), Times.Never, "Executor should not trigger any delay");
}
public async Task Notifier_should_cancel_health_check_on_dispose()
{
SetupEndpointRegistration(Guid.NewGuid());
SetupHealthCheckInterval(TimeSpan.FromMilliseconds(1));
var notCancelled = false;
var countdown = new AsyncCountdown("healthCheck", 1);
Func <CancellationToken, Task <EndpointHealth> > healthCheck = async token =>
{
countdown.Decrement();
await Task.Delay(TestMaxTime, token);
notCancelled = true;
return(new EndpointHealth(HealthStatus.Healthy));
};
using (CreateNotifier(healthCheck))
await countdown.WaitAsync(TestMaxTime);
Assert.False(notCancelled);
}
public async Task Monitor_should_start_checking_the_health_of_endpoints_until_disposed()
{
var endpoint1Countdown = new AsyncCountdown("endpoint1", 2);
var endpoint2Countdown = new AsyncCountdown("endpoint2", 1);
var endpoint3Countdown = new AsyncCountdown("endpoint3", 2);
var counters = new[] { new AsyncCounter(), new AsyncCounter(), new AsyncCounter() };
var healthCheck1 = _awaitableFactory.Return(new HealthInfo(HealthStatus.Healthy)).WithCountdown(endpoint1Countdown).WithCounter(counters[0]);
var healthCheck2 = _awaitableFactory.Return(new HealthInfo(HealthStatus.Healthy)).WithCountdown(endpoint2Countdown).WithCounter(counters[1]);
var healthCheck3 = _awaitableFactory.Return(new HealthInfo(HealthStatus.Healthy)).WithCountdown(endpoint3Countdown).WithCounter(counters[2]);
_mockHealthMonitor.Setup(cfg => cfg.CheckHealthAsync("address1", It.IsAny <CancellationToken>())).Returns(healthCheck1.RunAsync);
_mockHealthMonitor.Setup(cfg => cfg.CheckHealthAsync("address2", It.IsAny <CancellationToken>())).Returns(healthCheck2.RunAsync);
_mockHealthMonitor.Setup(cfg => cfg.CheckHealthAsync("address3", It.IsAny <CancellationToken>())).Returns(healthCheck3.RunAsync);
var endpoint1 = _endpointRegistry.TryRegister(CreateEndpointIdentity("address1"));
_endpointRegistry.TryRegister(CreateEndpointIdentity("address2"));
using (CreateEndpointMonitor(TimeSpan.FromMilliseconds(50)))
{
await endpoint1Countdown.WaitAsync(TestMaxWaitTime);
await endpoint2Countdown.WaitAsync(TestMaxWaitTime);
_endpointRegistry.TryRegister(CreateEndpointIdentity("address3"));
await endpoint3Countdown.WaitAsync(TestMaxWaitTime);
_endpointRegistry.TryUnregister(endpoint1.Identity);
// ensure that endpoint 2 still is running, and give time for endpoint 1 to shutdown
await endpoint2Countdown.ResetTo(10).WaitAsync(TestMaxWaitTime);
// ensure that endpoint 1 calls does not change, while endpoint 2 still runs
await AssertValueDidNotChangedAfterOperationAsync(
() => counters[0].Value,
() => endpoint2Countdown.ResetTo(10).WaitAsync(TestMaxWaitTime));
}
await AssertValueDidNotChangedAfterOperationAsync(() => counters.Sum(c => c.Value), () => Task.Delay(200));
}
public async Task Exchange_should_retry_uploading_supported_monitor_types_until_disposal()
{
var countdown = new AsyncCountdown(nameof(IHealthMonitorExchangeClient.RegisterMonitorsAsync), 10);
var counter = new AsyncCounter();
_exchangeClient
.Setup(c => c.RegisterMonitorsAsync(It.IsAny <IEnumerable <string> >(), It.IsAny <CancellationToken>()))
.Returns(() => _awaitableFactory.Throw(new InvalidOperationException()).WithCountdown(countdown).WithCounter(counter).RunAsync());
SetupDefaultEndpointIdentitiesMock();
using (CreateExchange(new DataExchangeConfig(100, 10, TimeSpan.FromMilliseconds(1), TimeSpan.FromSeconds(1))))
await countdown.WaitAsync(TestMaxWaitTime);
var capturedCalls = counter.Value;
Assert.True(capturedCalls > 0, "capturedCalls>0");
await Task.Delay(PostRunDelay);
Assert.Equal(capturedCalls, counter.Value);
}
public async Task Manager_should_delete_old_stats()
{
const int expectedRepeats = 3;
var age = TimeSpan.FromHours(1);
var utcNow = DateTime.UtcNow;
var asyncCountdown = new AsyncCountdown("delete old stats", expectedRepeats);
var asyncCounter = new AsyncCounter();
var monitorSettings = new Mock <IMonitorSettings>();
monitorSettings.Setup(s => s.StatsHistoryMaxAge).Returns(age);
var timeCoordinator = new Mock <ITimeCoordinator>();
timeCoordinator.Setup(c => c.UtcNow).Returns(utcNow);
var endpointMetricsCoordinator = new Mock <IEndpointMetricsForwarderCoordinator>();
var repository = new Mock <IEndpointStatsRepository>();
repository.Setup(r => r.DeleteStatisticsOlderThan(utcNow - age, It.IsAny <int>())).Returns(() =>
{
asyncCountdown.Decrement();
asyncCounter.Increment();
const int deletedItemsCount = 127;
return(asyncCounter.Value >= expectedRepeats ? 0 : deletedItemsCount);
});
using (new EndpointStatsManager(repository.Object, monitorSettings.Object, timeCoordinator.Object, endpointMetricsCoordinator.Object))
await asyncCountdown.WaitAsync(MaxTestTime);
await Task.Delay(TimeSpan.FromMilliseconds(500));
//it should stop calling cleanup when there are no more items to be cleaned
Assert.Equal(expectedRepeats, asyncCounter.Value);
}
public async Task Exchange_should_retry_health_update_send_in_case_of_errors()
{
var count = 10;
var countdown = new AsyncCountdown(nameof(IHealthMonitorExchangeClient.UploadHealthAsync), count);
SetupDefaultRegisterEndpointsMock();
var identity = SetupDefaultEndpointIdentitiesMock().First();
var attempts = 10;
SetupDefaultHealthUploadMock(updates =>
{
if (attempts-- > 0)
{
throw new Exception();
}
foreach (var update in updates)
{
countdown.Decrement();
}
});
var config = new DataExchangeConfig(100, count, TimeSpan.FromMilliseconds(1), TimeSpan.FromSeconds(1), EndpointMetadata.DefaultMonitorTag);
var healths = PrepareEndpointHealths(count);
using (var ex = CreateExchange(config))
{
foreach (var health in healths)
{
ex.UpdateHealth(identity.Id, health);
}
await countdown.WaitAsync(TestMaxWaitTime);
}
}
public async Task Exchange_should_retry_health_update_send_until_disposal()
{
var countdown = new AsyncCountdown(nameof(IHealthMonitorExchangeClient.UploadHealthAsync), 10);
SetupDefaultRegisterEndpointsMock();
var identity = SetupDefaultEndpointIdentitiesMock().First();
int calls = 0;
SetupDefaultHealthUploadMock(x =>
{
Interlocked.Increment(ref calls);
countdown.Decrement();
if (calls > 1)
{
throw new InvalidOperationException();
}
});
var healths = PrepareEndpointHealths(2);
using (var ex = CreateExchange(new DataExchangeConfig(100, 1, TimeSpan.FromMilliseconds(1), TimeSpan.FromSeconds(1), EndpointMetadata.DefaultMonitorTag)))
{
foreach (var health in healths)
{
ex.UpdateHealth(identity.Id, health);
}
await countdown.WaitAsync(TestMaxWaitTime);
}
var capturedCalls = calls;
Assert.True(capturedCalls > 0, "capturedCalls>0");
await Task.Delay(PostRunDelay);
Assert.Equal(capturedCalls, calls);
}
public async Task Notifier_should_send_notifications_within_specified_time_span()
{
var minimumChecks = 3;
var countdown = new AsyncCountdown("notifications", minimumChecks);
var endpointId = Guid.NewGuid();
var interval = TimeSpan.FromMilliseconds(300);
SetupHealthCheckInterval(interval);
SetupEndpointRegistration(endpointId);
_mockClient
.Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny <HealthUpdate>(), It.IsAny <CancellationToken>()))
.Returns(() => _awaitableFactory.Return().WithDelay(TimeSpan.FromMilliseconds(100)).WithTimeline("updateHealth").WithCountdown(countdown).RunAsync());
_mockTimeCoordinator
.Setup(c => c.Delay(interval, It.IsAny <CancellationToken>()))
.Returns(() => _awaitableFactory.Return().WithDelay(TimeSpan.FromMilliseconds(50)).WithTimeline("delay").RunAsync());
using (CreateNotifier())
await countdown.WaitAsync(TestMaxTime);
var actualEventTimeline = _awaitableFactory.GetOrderedTimelineEvents()
.Select(eventName => eventName == "updateHealth_start" || eventName == "delay_start" ? "update_start" : eventName)
.Select(eventName => eventName == "updateHealth_finish" || eventName == "delay_finish" ? "update_finish" : eventName)
.Take(minimumChecks * 4)
.ToArray();
for (int i = 0; i < minimumChecks; i++)
{
var actualSequence = actualEventTimeline.Skip(i * 4).Take(4).ToList();
Assert.Equal("update_start", actualSequence.First());
Assert.Equal("update_finish", actualSequence.Last());
Assert.Equal(2, actualSequence.Count(s => s == "update_start"));
Assert.Equal(2, actualSequence.Count(s => s == "update_finish"));
}
}
public async Task Manager_should_delete_old_stats()
{
var age = TimeSpan.FromHours(1);
var utcNow = DateTime.UtcNow;
var asyncCountdown = new AsyncCountdown("delete old stats", 1);
var monitorSettings = new Mock <IMonitorSettings>();
monitorSettings.Setup(s => s.StatsHistoryMaxAge).Returns(age);
var timeCoordinator = new Mock <ITimeCoordinator>();
timeCoordinator.Setup(c => c.UtcNow).Returns(utcNow);
var endpointMetricsCoordinator = new Mock <IEndpointMetricsForwarderCoordinator>();
var repository = new Mock <IEndpointStatsRepository>();
repository.Setup(r => r.DeleteStatisticsOlderThan(utcNow - age)).Callback(() => asyncCountdown.Decrement());
using (new EndpointStatsManager(repository.Object, monitorSettings.Object, timeCoordinator.Object, endpointMetricsCoordinator.Object))
await asyncCountdown.WaitAsync(MaxTestTime);
}
public async Task Executor_should_cancel_all_tasks_on_disposal_and_report_all_finished()
{
var task1NotCancelled = false;
var task2NotCancelled = false;
var task1 = "task1";
var task2 = "task2";
var task1Ran = new AsyncCountdown(task1, 1);
var task2Ran = new AsyncCountdown(task2, 1);
var completed = new ConcurrentQueue <string>();
using (var executor = ContinuousTaskExecutor <string> .StartExecutor(Mock.Of <ITimeCoordinator>()))
{
executor.FinishedTaskFor += item => completed.Enqueue(item);
executor.TryRegisterTaskFor(task1, async(item, token) =>
{
task1Ran.Decrement();
await Task.Delay(_testTimeout, token);
task1NotCancelled = true;
});
executor.TryRegisterTaskFor(task2, async(item, token) =>
{
task2Ran.Decrement();
await Task.Delay(_testTimeout, token);
task2NotCancelled = true;
});
await task1Ran.WaitAsync(_testTimeout);
await task2Ran.WaitAsync(_testTimeout);
}
Assert.False(task1NotCancelled, "task1NotCancelled");
Assert.False(task2NotCancelled, "task2NotCancelled");
CollectionAssert.AreEquivalent(new[] { task1, task2 }, completed);
}
public async Task Monitor_should_check_endpoint_health_with_regular_intervals()
{
var interval = TimeSpan.FromMilliseconds(200);
var healthCounter = new AsyncCountdown("health", 1);
var delayCounter = new AsyncCountdown("delay", 1);
_mockHealthMonitor.Setup(cfg => cfg.CheckHealthAsync("address", It.IsAny <CancellationToken>())).Returns(() => _awaitableFactory.Return(new HealthInfo(HealthStatus.Healthy)).WithTimeline("health").WithCountdown(healthCounter).RunAsync());
_mockTimeCoordinator.Setup(cfg => cfg.Delay(interval, It.IsAny <CancellationToken>())).Returns(() => _awaitableFactory.Return().WithDelay(interval).WithTimeline("delay").WithCountdown(delayCounter).RunAsync());
_endpointRegistry.TryRegister(CreateEndpointIdentity("address"));
using (CreateEndpointMonitor(interval))
await Task.WhenAll(healthCounter.WaitAsync(TestMaxWaitTime), delayCounter.WaitAsync(TestMaxWaitTime));
var results = _awaitableFactory.GetTimeline();
Assert.True(results.Length > 1, "results.Length>1");
var delayTask = results[0];
var healthTask = results[1];
Assert.Equal(delayTask.Tag, "delay");
Assert.Equal(healthTask.Tag, "health");
AssertTimeOrder(delayTask.Started, healthTask.Started, healthTask.Finished, delayTask.Finished);
}
public async Task Notifier_should_send_faulty_health_to_the_API_if_provided_health_method_throws()
{
var endpointId = Guid.NewGuid();
HealthUpdate lastCaptured = null;
var countdown = new AsyncCountdown("update", 5);
SetupHealthCheckInterval(TimeSpan.FromMilliseconds(1));
SetupEndpointRegistration(endpointId);
_mockClient.Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny <HealthUpdate>(), It.IsAny <CancellationToken>()))
.Returns((Guid id, string authToken, HealthUpdate upd, CancellationToken token) => _awaitableFactory
.Execute(() => lastCaptured = upd)
.WithCountdown(countdown)
.RunAsync());
using (CreateNotifier(async token => { await Task.Delay(50, token); throw new InvalidOperationException("some reason"); }))
await countdown.WaitAsync(TestMaxTime);
Assert.NotNull(lastCaptured);
Assert.Equal(HealthStatus.Faulty, lastCaptured.Status);
Assert.Equal("Unable to collect health information", lastCaptured.Details["reason"]);
Assert.True(lastCaptured.Details["exception"].StartsWith("System.InvalidOperationException: some reason"));
Assert.True(lastCaptured.CheckTimeUtc > DateTime.UtcNow.AddMinutes(-1) && lastCaptured.CheckTimeUtc < DateTime.UtcNow.AddMinutes(1));
}
public async Task Notifier_should_send_notifications_within_specified_time_span()
{
var minimumChecks = 3;
var expectedEventTimeline = new List <string>();
for (var i = 0; i < minimumChecks; ++i)
{
expectedEventTimeline.AddRange(new[] { "updateHealth_start", "delay_start", "update_finish", "update_finish" });
}
var countdown = new AsyncCountdown("notifications", minimumChecks);
var endpointId = Guid.NewGuid();
var interval = TimeSpan.FromMilliseconds(300);
SetupHealthCheckInterval(interval);
SetupEndpointRegistration(endpointId);
_mockClient
.Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny <HealthUpdate>(), It.IsAny <CancellationToken>()))
.Returns(() => _awaitableFactory.Return().WithDelay(TimeSpan.FromMilliseconds(100)).WithTimeline("updateHealth").WithCountdown(countdown).RunAsync());
_mockTimeCoordinator
.Setup(c => c.Delay(interval, It.IsAny <CancellationToken>()))
.Returns(() => _awaitableFactory.Return().WithDelay(TimeSpan.FromMilliseconds(50)).WithTimeline("delay").RunAsync());
using (CreateNotifier())
await countdown.WaitAsync(TestMaxTime);
var actualEventTimeline = _awaitableFactory.GetOrderedTimelineEvents()
.Select(eventName => (eventName == "updateHealth_finish" || eventName == "delay_finish") ? "update_finish" : eventName)
.Take(minimumChecks * 4)
.ToArray();
Assert.True(expectedEventTimeline.SequenceEqual(actualEventTimeline), $"Expected:\n{string.Join(",", expectedEventTimeline)}\nGot:\n{string.Join(",", actualEventTimeline)}");
}