private async Task EnsureSendHealthUpdateAsync(HealthUpdate update) { while (!_cancelationTokenSource.IsCancellationRequested) { try { await SendHealthUpdateAsync(update); _retryInterval = null; return; } catch (Exception e) when(!_cancelationTokenSource.IsCancellationRequested) { BackOffPlan backOffPlan = await _backOffStategy.GetCurrent(_retryInterval, _cancelationTokenSource.Token); if (backOffPlan.ShouldLog) { _logger.Error("Unable to send health update", e); } _retryInterval = backOffPlan.RetryInterval; if (_retryInterval.HasValue) { await SafeDelay(_retryInterval.Value); } } } }
public async Task Notifier_should_retry_sending_health_updates_according_to_recommened_backOff_strategy_in_case_of_exceptions() { var endpointId = Guid.NewGuid(); SetupEndpointRegistration(endpointId); var checkInterval = TimeSpan.FromMilliseconds(127); SetupHealthCheckInterval(checkInterval); var minRepeats = 10; var countdown = new AsyncCountdown("update", minRepeats); var updates = new ConcurrentQueue <HealthUpdate>(); var backOffPlan = new BackOffPlan(null, false); _mockClient .Setup(c => c.SendHealthUpdateAsync(endpointId, AuthenticationToken, It.IsAny <HealthUpdate>(), It.IsAny <CancellationToken>())) .Returns((Guid id, string authToken, HealthUpdate upd, CancellationToken token) => { updates.Enqueue(upd); return(_awaitableFactory .Throw(new InvalidOperationException()) .WithCountdown(countdown) .RunAsync()); }); using (CreateNotifier()) await countdown.WaitAsync(TestMaxTime); int expectedSeconds = 1; for (int i = 0; i < minRepeats; ++i) { backOffPlan = new BackOffPlan(TimeSpan.FromSeconds(Math.Min(expectedSeconds, MaxEndpointNotifierRetryDelayInSecs)), true); _mockBackOffStategy .Setup(b => b.GetCurrent(TimeSpan.FromSeconds(expectedSeconds), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(backOffPlan)); _mockTimeCoordinator.Verify(c => c.Delay(TimeSpan.FromSeconds(expectedSeconds), It.IsAny <CancellationToken>())); expectedSeconds = Math.Min(expectedSeconds *= 2, MaxEndpointNotifierRetryDelayInSecs); } _mockTimeCoordinator.Verify(c => c.Delay(checkInterval, It.IsAny <CancellationToken>()), Times.Once); Assert.Equal(1, updates.Distinct().Count()); }