/// <summary> /// Async methods that conduct http request to query the health state from the health controller of every endpoints. /// </summary> private async Task ProbeEndpointsAsync(AsyncSemaphore semaphore, CancellationToken cancellationToken) { // Always continue probing until receives cancellation token. try { while (true) { var probeTasks = new List <Task>(); cancellationToken.ThrowIfCancellationRequested(); // try catch to prevent while loop crashed by any nonfatal exception try { // Submit probe requests. foreach (var endpoint in _endpointManager.GetItems()) { // Start a single probing attempt. probeTasks.Add(ProbeEndpointAsync(endpoint, semaphore, cancellationToken)); } await Task.WhenAll(probeTasks); } catch (OperationCanceledException) when(_cts.IsCancellationRequested) { // If the cancel requested by our StopAsync method. It is a expected graceful shut down. throw; } catch (Exception ex) when(!ex.IsFatal()) { // Swallow the nonfatal exception, we don not want the health check to break. _logger.LogError(ex, $"Prober for '{BackendId}' encounters unexpected exception."); } _logger.LogInformation($"The backend prober for '{BackendId}' has checked all endpoints with time interval {_healthCheckInterval.TotalSeconds} second."); // Wait for next probe cycle. await _timer.Delay(_healthCheckInterval, cancellationToken); } } catch (OperationCanceledException) when(_cts.IsCancellationRequested) { // If the cancel requested by our StopAsync method. It is a expected graceful shut down. _logger.LogInformation($"Prober for backend '{BackendId}' has gracefully shutdown."); } catch (Exception ex) { // Swallow the exception, we want the health check continuously running like the heartbeat. _logger.LogError(ex, $"Prober for '{BackendId}' encounters unexpected exception."); } }
/// <summary> /// Operates like <see cref="CancellationTokenSource.CancelAfter(TimeSpan)"/> but supporting specifying a custom timer. /// </summary> /// <param name="cancellationTokenSource">Token to cancel after expiration is complete.</param> /// <param name="timeout">Timeout after which the cancellationTokenSource will be canceled.</param> /// <param name="timer">Timer to perform the measurement of time for determining when to cancel.</param> public static async void CancelAfter(this CancellationTokenSource cancellationTokenSource, TimeSpan timeout, IMonotonicTimer timer) { if (timer == null) { throw new ArgumentNullException(nameof(timer)); } try { await timer.Delay(timeout, cancellationTokenSource.Token); cancellationTokenSource.Cancel(); } catch (ObjectDisposedException) { // Ignore disposed cancellation tokens. Indicates cancellation is no longer needed. Unfortunately CTS's don't give a good // way to safely check async disposal, so must rely on exception handling instead. } catch (OperationCanceledException) { // It cts was canceled, then there's no need for us to cancel the token. Return successfully. // Note that we can't avoid this situation in advance as we strongly desire here to retain the 'void' returning // interface that cts.CancelAfter(ts) has. } }