public async Task Manager_should_delete_old_stats()
{
const int expectedRepeats = 3;
var age = TimeSpan.FromHours(1);
var utcNow = DateTime.UtcNow;
var asyncCountdown = new AsyncCountdown("delete old stats", expectedRepeats);
var asyncCounter = new AsyncCounter();
var monitorSettings = new Mock<IMonitorSettings>();
monitorSettings.Setup(s => s.StatsHistoryMaxAge).Returns(age);
var timeCoordinator = new Mock<ITimeCoordinator>();
timeCoordinator.Setup(c => c.UtcNow).Returns(utcNow);
var endpointMetricsCoordinator = new Mock<IEndpointMetricsForwarderCoordinator>();
var repository = new Mock<IEndpointStatsRepository>();
repository.Setup(r => r.DeleteStatisticsOlderThan(utcNow - age, It.IsAny<int>())).Returns(() =>
{
asyncCountdown.Decrement();
asyncCounter.Increment();
const int deletedItemsCount = 127;
return asyncCounter.Value >= expectedRepeats ? 0 : deletedItemsCount;
});
using (new EndpointStatsManager(repository.Object, monitorSettings.Object, timeCoordinator.Object, endpointMetricsCoordinator.Object))
await asyncCountdown.WaitAsync(MaxTestTime);
await Task.Delay(TimeSpan.FromMilliseconds(500));
//it should stop calling cleanup when there are no more items to be cleaned
Assert.Equal(expectedRepeats, asyncCounter.Value);
}
public async Task Notifier_should_send_notifications_within_specified_time_span()
{
var minimumChecks = 3;
var expectedEventTimeline = new List<string>();
for (var i = 0; i < minimumChecks; ++i)
expectedEventTimeline.AddRange(new[] { "updateHealth_start", "delay_start", "update_finish", "update_finish" });
var countdown = new AsyncCountdown("notifications", minimumChecks);
var endpointId = Guid.NewGuid();
var interval = TimeSpan.FromMilliseconds(300);
SetupHealthCheckInterval(interval);
SetupEndpointRegistration(endpointId);
_mockClient
.Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny<HealthUpdate>(), It.IsAny<CancellationToken>()))
.Returns(() => _awaitableFactory.Return().WithDelay(TimeSpan.FromMilliseconds(100)).WithTimeline("updateHealth").WithCountdown(countdown).RunAsync());
_mockTimeCoordinator
.Setup(c => c.Delay(interval, It.IsAny<CancellationToken>()))
.Returns(() => _awaitableFactory.Return().WithDelay(TimeSpan.FromMilliseconds(50)).WithTimeline("delay").RunAsync());
using (CreateNotifier())
await countdown.WaitAsync(TestMaxTime);
var actualEventTimeline = _awaitableFactory.GetOrderedTimelineEvents()
.Select(eventName => (eventName == "updateHealth_finish" || eventName == "delay_finish") ? "update_finish" : eventName)
.Take(minimumChecks * 4)
.ToArray();
Assert.True(expectedEventTimeline.SequenceEqual(actualEventTimeline), $"Expected:\n{string.Join(",", expectedEventTimeline)}\nGot:\n{string.Join(",", actualEventTimeline)}");
}
public async Task Executor_should_execute_tasks_until_disposal()
{
var task1Name = "task1";
var task2Name = "task2";
var countdown1 = new AsyncCountdown(task1Name, 10);
var counter1 = new AsyncCounter();
var countdown2 = new AsyncCountdown(task2Name, 10);
var counter2 = new AsyncCounter();
using (var executor = ContinuousTaskExecutor<string>.StartExecutor(Mock.Of<ITimeCoordinator>()))
{
Assert.True(executor.TryRegisterTaskFor(task1Name, (item, token) => StartTaskAsync(countdown1, counter1, token)));
await countdown1.WaitAsync(_testTimeout);
Assert.True(executor.TryRegisterTaskFor(task2Name, (item, token) => StartTaskAsync(countdown2, counter2, token)));
await countdown2.WaitAsync(_testTimeout);
// check that task 1 still works
await countdown1.ResetTo(10).WaitAsync(_testTimeout);
}
var expected1 = counter1.Value;
var expected2 = counter2.Value;
await Task.Delay(200);
Assert.Equal(expected1, counter1.Value);
Assert.Equal(expected2, counter2.Value);
}
public async Task Notifier_should_send_current_health_to_the_API()
{
var endpointId = Guid.NewGuid();
var expectedHealth = new EndpointHealth(HealthStatus.Offline, new Dictionary<string, string> { { "key", "value" } });
HealthUpdate lastCaptured = null;
var countdown = new AsyncCountdown("update", 5);
SetupHealthCheckInterval(TimeSpan.FromMilliseconds(1));
SetupEndpointRegistration(endpointId);
_mockClient.Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny<HealthUpdate>(), It.IsAny<CancellationToken>()))
.Returns((Guid id, string authToken, HealthUpdate upd, CancellationToken token) => _awaitableFactory
.Execute(() => lastCaptured = upd)
.WithCountdown(countdown)
.RunAsync());
using (CreateNotifier(token => Task.FromResult(expectedHealth)))
await countdown.WaitAsync(TestMaxTime);
Assert.NotNull(lastCaptured);
Assert.Equal(expectedHealth.Status, lastCaptured.Status);
Assert.Equal(expectedHealth.Details, lastCaptured.Details);
Assert.True(lastCaptured.CheckTimeUtc > DateTime.UtcNow.AddMinutes(-1) && lastCaptured.CheckTimeUtc < DateTime.UtcNow.AddMinutes(1));
}
public async Task Notifier_should_send_faulty_health_to_the_API_if_provided_health_method_throws()
{
var endpointId = Guid.NewGuid();
HealthUpdate lastCaptured = null;
var countdown = new AsyncCountdown("update", 5);
SetupHealthCheckInterval(TimeSpan.FromMilliseconds(1));
SetupEndpointRegistration(endpointId);
_mockClient.Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny<HealthUpdate>(), It.IsAny<CancellationToken>()))
.Returns((Guid id, string authToken, HealthUpdate upd, CancellationToken token) => _awaitableFactory
.Execute(() => lastCaptured = upd)
.WithCountdown(countdown)
.RunAsync());
using (CreateNotifier(async token => { await Task.Delay(50, token); throw new InvalidOperationException("some reason"); }))
await countdown.WaitAsync(TestMaxTime);
Assert.NotNull(lastCaptured);
Assert.Equal(HealthStatus.Faulty, lastCaptured.Status);
Assert.Equal("Unable to collect health information", lastCaptured.Details["reason"]);
Assert.True(lastCaptured.Details["exception"].StartsWith("System.InvalidOperationException: some reason"));
Assert.True(lastCaptured.CheckTimeUtc > DateTime.UtcNow.AddMinutes(-1) && lastCaptured.CheckTimeUtc < DateTime.UtcNow.AddMinutes(1));
}
public async Task Notifier_should_survive_communication_errors_and_eventually_restore_connectivity()
{
var healthUpdateCountdown = new AsyncCountdown("update", 10);
var healthUpdateCountdown2 = new AsyncCountdown("update2", 10);
var registrationCountdown = new AsyncCountdown("registration", 10);
var intervalCountdown = new AsyncCountdown("interval", 10);
var delayCountdown = new AsyncCountdown("delay", 10);
var healthCheckInterval = TimeSpan.FromMilliseconds(127);
var endpointId = Guid.NewGuid();
var workingHealthUpdate = _awaitableFactory.Return().WithCountdown(healthUpdateCountdown2);
var notWorkingHealthUpdate = _awaitableFactory.Throw(new TaskCanceledException()).WithCountdown(healthUpdateCountdown);
var currentHealthUpdate = notWorkingHealthUpdate;
var notWorkingRegistration = _awaitableFactory.Throw<Guid>(new TaskCanceledException()).WithCountdown(registrationCountdown);
var workingRegistration = _awaitableFactory.Return(endpointId);
var currentRegistration = notWorkingRegistration;
var notWorkingHealthCheckInterval = _awaitableFactory.Throw<TimeSpan>(new TaskCanceledException()).WithCountdown(intervalCountdown);
var workingHealthCheckInterval = _awaitableFactory.Return(healthCheckInterval);
var currentHealthCheckInterval = notWorkingHealthCheckInterval;
_mockClient.Setup(c => c.SendHealthUpdateAsync(It.IsAny<Guid>(), AuthenticationToken, It.IsAny<HealthUpdate>(), It.IsAny<CancellationToken>()))
.Returns(() => currentHealthUpdate.RunAsync());
_mockClient.Setup(c => c.RegisterEndpointAsync(It.IsAny<EndpointDefinition>(), It.IsAny<CancellationToken>()))
.Returns(() => currentRegistration.RunAsync());
_mockClient.Setup(c => c.GetHealthCheckIntervalAsync(It.IsAny<CancellationToken>()))
.Returns(() => currentHealthCheckInterval.RunAsync());
_mockTimeCoordinator.Setup(c => c.Delay(healthCheckInterval, It.IsAny<CancellationToken>()))
.Returns(() => _awaitableFactory.Return().WithCountdown(delayCountdown).RunAsync());
using (CreateNotifier())
{
await registrationCountdown.WaitAsync(TestMaxTime);
currentRegistration = workingRegistration;
await healthUpdateCountdown.WaitAsync(TestMaxTime);
currentHealthUpdate = workingHealthUpdate;
await healthUpdateCountdown2.WaitAsync(TestMaxTime);
await intervalCountdown.WaitAsync(TestMaxTime);
currentHealthCheckInterval = workingHealthCheckInterval;
await delayCountdown.WaitAsync(TestMaxTime);
}
}
public async Task Notifier_should_retry_sending_health_updates_in_case_of_exceptions()
{
var endpointId = Guid.NewGuid();
SetupEndpointRegistration(endpointId);
var checkInterval = TimeSpan.FromMilliseconds(127);
SetupHealthCheckInterval(checkInterval);
var minRepeats = 10;
var countdown = new AsyncCountdown("update", minRepeats);
var updates = new ConcurrentQueue<HealthUpdate>();
_mockClient
.Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny<HealthUpdate>(), It.IsAny<CancellationToken>()))
.Returns((Guid id, string authToken, HealthUpdate upd, CancellationToken token) =>
{
updates.Enqueue(upd);
return _awaitableFactory
.Throw(new InvalidOperationException())
.WithCountdown(countdown)
.RunAsync();
});
using (CreateNotifier())
await countdown.WaitAsync(TestMaxTime);
int expectedSeconds = 1;
for (int i = 0; i < minRepeats; ++i)
{
_mockTimeCoordinator.Verify(c => c.Delay(TimeSpan.FromSeconds(expectedSeconds), It.IsAny<CancellationToken>()));
expectedSeconds = Math.Min(expectedSeconds *= 2, MaxEndpointNotifierRetryDelayInSecs);
}
_mockTimeCoordinator.Verify(c => c.Delay(checkInterval, It.IsAny<CancellationToken>()), Times.Once);
Assert.Equal(1, updates.Distinct().Count());
}
public async Task Notifier_should_cancel_health_check_on_dispose()
{
SetupEndpointRegistration(Guid.NewGuid());
SetupHealthCheckInterval(TimeSpan.FromMilliseconds(1));
var notCancelled = false;
var countdown = new AsyncCountdown("healthCheck", 1);
Func<CancellationToken, Task<EndpointHealth>> healthCheck = async token =>
{
countdown.Decrement();
await Task.Delay(TestMaxTime, token);
notCancelled = true;
return new EndpointHealth(HealthStatus.Healthy);
};
using (CreateNotifier(healthCheck))
await countdown.WaitAsync(TestMaxTime);
Assert.False(notCancelled);
}
public async Task Notifier_should_register_endpoint_with_all_details_and_specified_host()
{
EndpointDefinition captured = null;
var countdown = new AsyncCountdown("register", 1);
var endpointId = Guid.NewGuid();
_mockClient
.Setup(c => c.RegisterEndpointAsync(It.IsAny<EndpointDefinition>(), It.IsAny<CancellationToken>()))
.Returns((EndpointDefinition def, CancellationToken token) => _awaitableFactory
.Execute(() => { captured = def; return endpointId; })
.WithCountdown(countdown)
.RunAsync());
SetupHealthCheckInterval(TimeSpan.FromMilliseconds(1));
Action<IEndpointDefintionBuilder> builder = b => b.DefineName("endpointName")
.DefineGroup("endpointGroup")
.DefineTags("t1")
.DefineAddress("host", "uniqueName")
.DefinePassword(AuthenticationToken);
using (CreateNotifier(builder, token => Task.FromResult(new EndpointHealth(HealthStatus.Offline))))
await countdown.WaitAsync(TestMaxTime);
Assert.NotNull(captured);
Assert.Equal("endpointName", captured.EndpointName);
Assert.Equal("endpointGroup", captured.GroupName);
Assert.Equal(new[] { "t1" }, captured.Tags);
Assert.Equal("host:uniqueName", captured.Address);
}
public async Task Notifier_should_register_endpoint_again_if_notification_failed_with_EndpointNotFoundException()
{
var oldEndpointId = Guid.NewGuid();
var newEndpointId = Guid.NewGuid();
var oldEndpointCountdown = new AsyncCountdown("oldEndpoint", 1);
var newEndpointCountdown = new AsyncCountdown("newEndpoint", 1);
_mockClient
.SetupSequence(c => c.RegisterEndpointAsync(It.IsAny<EndpointDefinition>(), It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(oldEndpointId))
.Returns(Task.FromResult(newEndpointId));
_mockClient
.Setup(c => c.SendHealthUpdateAsync(oldEndpointId, AuthenticationToken, It.IsAny<HealthUpdate>(), It.IsAny<CancellationToken>()))
.Returns(() => _awaitableFactory
.Throw(new EndpointNotFoundException())
.WithCountdown(oldEndpointCountdown)
.RunAsync());
_mockClient
.Setup(c => c.SendHealthUpdateAsync(newEndpointId, AuthenticationToken, It.IsAny<HealthUpdate>(), It.IsAny<CancellationToken>()))
.Returns(() => _awaitableFactory
.Return()
.WithCountdown(newEndpointCountdown)
.RunAsync());
using (CreateNotifier())
{
await oldEndpointCountdown.WaitAsync(TestMaxTime);
await newEndpointCountdown.WaitAsync(TestMaxTime);
}
}
public async Task Executor_should_continue_processing_other_tasks_when_one_finish()
{
var task1Name = "task1";
var task2Name = "task2";
var countdown1 = new AsyncCountdown(task1Name, 10);
var task2Finished = new SemaphoreSlim(0);
using (var executor = ContinuousTaskExecutor<string>.StartExecutor(Mock.Of<ITimeCoordinator>()))
{
executor.FinishedTaskFor += item =>
{
if (item == task2Name) task2Finished.Release();
};
Assert.True(executor.TryRegisterTaskFor(task1Name, (item, token) => StartTaskAsync(token, b => b.WithCountdown(countdown1))));
await countdown1.WaitAsync(_testTimeout);
Assert.True(executor.TryRegisterTaskFor(task2Name, (item, token) => Task.Delay(25, token)));
await task2Finished.WaitAsync(_testTimeout);
// check that task 1 still works
await countdown1.ResetTo(10).WaitAsync(_testTimeout);
}
}
private Task StartTaskAsync(AsyncCountdown countdown, AsyncCounter counter, CancellationToken token)
{
return StartTaskAsync(token, c => c.WithCountdown(countdown).WithCounter(counter));
}
public async Task Executor_should_immediately_break_the_loop_on_cancelation()
{
var timeCoordinator = new Mock<ITimeCoordinator>();
var countdown = new AsyncCountdown("task", 1);
var taskNotCancelled = false;
using (var executor = ContinuousTaskExecutor<string>.StartExecutor(timeCoordinator.Object))
{
executor.TryRegisterTaskFor("task", async (item, token) =>
{
countdown.Decrement();
await Task.Delay(_testTimeout, token);
taskNotCancelled = true;
});
await countdown.WaitAsync(_testTimeout);
}
Assert.False(taskNotCancelled, "Task was not cancelled");
timeCoordinator.Verify(c => c.Delay(It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()), Times.Never, "Executor should not trigger any delay");
}
public async Task Executor_should_cancel_all_tasks_on_disposal_and_report_all_finished()
{
var task1NotCancelled = false;
var task2NotCancelled = false;
var task1 = "task1";
var task2 = "task2";
var task1Ran = new AsyncCountdown(task1, 1);
var task2Ran = new AsyncCountdown(task2, 1);
var completed = new ConcurrentQueue<string>();
using (var executor = ContinuousTaskExecutor<string>.StartExecutor(Mock.Of<ITimeCoordinator>()))
{
executor.FinishedTaskFor += item => completed.Enqueue(item);
executor.TryRegisterTaskFor(task1, async (item, token) =>
{
task1Ran.Decrement();
await Task.Delay(_testTimeout, token);
task1NotCancelled = true;
});
executor.TryRegisterTaskFor(task2, async (item, token) =>
{
task2Ran.Decrement();
await Task.Delay(_testTimeout, token);
task2NotCancelled = true;
});
await task1Ran.WaitAsync(_testTimeout);
await task2Ran.WaitAsync(_testTimeout);
}
Assert.False(task1NotCancelled, "task1NotCancelled");
Assert.False(task2NotCancelled, "task2NotCancelled");
CollectionAssert.AreEquivalent(new[] { task1, task2 }, completed);
}