private async Task EnsureSendHealthUpdateAsync(HealthUpdate update)
{
while (!_cancelationTokenSource.IsCancellationRequested)
{
try
{
await SendHealthUpdateAsync(update);
_retryInterval = null;
return;
}
catch (Exception e) when(!_cancelationTokenSource.IsCancellationRequested)
{
BackOffPlan backOffPlan = await _backOffStategy.GetCurrent(_retryInterval, _cancelationTokenSource.Token);
if (backOffPlan.ShouldLog)
{
_logger.Error("Unable to send health update", e);
}
_retryInterval = backOffPlan.RetryInterval;
if (_retryInterval.HasValue)
{
await SafeDelay(_retryInterval.Value);
}
}
}
}
public async Task Notifier_should_retry_sending_health_updates_according_to_recommened_backOff_strategy_in_case_of_exceptions()
{
var endpointId = Guid.NewGuid();
SetupEndpointRegistration(endpointId);
var checkInterval = TimeSpan.FromMilliseconds(127);
SetupHealthCheckInterval(checkInterval);
var minRepeats = 10;
var countdown = new AsyncCountdown("update", minRepeats);
var updates = new ConcurrentQueue <HealthUpdate>();
var backOffPlan = new BackOffPlan(null, false);
_mockClient
.Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny <HealthUpdate>(), It.IsAny <CancellationToken>()))
.Returns((Guid id, string authToken, HealthUpdate upd, CancellationToken token) =>
{
updates.Enqueue(upd);
return(_awaitableFactory
.Throw(new InvalidOperationException())
.WithCountdown(countdown)
.RunAsync());
});
using (CreateNotifier())
await countdown.WaitAsync(TestMaxTime);
int expectedSeconds = 1;
for (int i = 0; i < minRepeats; ++i)
{
backOffPlan = new BackOffPlan(TimeSpan.FromSeconds(Math.Min(expectedSeconds, MaxEndpointNotifierRetryDelayInSecs)), true);
_mockBackOffStategy
.Setup(b => b.GetCurrent(TimeSpan.FromSeconds(expectedSeconds), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(backOffPlan));
_mockTimeCoordinator.Verify(c => c.Delay(TimeSpan.FromSeconds(expectedSeconds), It.IsAny <CancellationToken>()));
expectedSeconds = Math.Min(expectedSeconds *= 2, MaxEndpointNotifierRetryDelayInSecs);
}
_mockTimeCoordinator.Verify(c => c.Delay(checkInterval, It.IsAny <CancellationToken>()), Times.Once);
Assert.Equal(1, updates.Distinct().Count());
}
