/// <summary>
/// Async methods that conduct http request to query the health state from the health controller of every endpoints.
/// </summary>
private async Task ProbeEndpointsAsync(AsyncSemaphore semaphore, CancellationToken cancellationToken)
{
// Always continue probing until receives cancellation token.
try
{
while (true)
{
var probeTasks = new List <Task>();
cancellationToken.ThrowIfCancellationRequested();
// try catch to prevent while loop crashed by any nonfatal exception
try
{
// Submit probe requests.
foreach (var endpoint in _endpointManager.GetItems())
{
// Start a single probing attempt.
probeTasks.Add(ProbeEndpointAsync(endpoint, semaphore, cancellationToken));
}
await Task.WhenAll(probeTasks);
}
catch (OperationCanceledException) when(_cts.IsCancellationRequested)
{
// If the cancel requested by our StopAsync method. It is a expected graceful shut down.
throw;
}
catch (Exception ex) when(!ex.IsFatal())
{
// Swallow the nonfatal exception, we don not want the health check to break.
_logger.LogError(ex, $"Prober for '{BackendId}' encounters unexpected exception.");
}
_logger.LogInformation($"The backend prober for '{BackendId}' has checked all endpoints with time interval {_healthCheckInterval.TotalSeconds} second.");
// Wait for next probe cycle.
await _timer.Delay(_healthCheckInterval, cancellationToken);
}
}
catch (OperationCanceledException) when(_cts.IsCancellationRequested)
{
// If the cancel requested by our StopAsync method. It is a expected graceful shut down.
_logger.LogInformation($"Prober for backend '{BackendId}' has gracefully shutdown.");
}
catch (Exception ex)
{
// Swallow the exception, we want the health check continuously running like the heartbeat.
_logger.LogError(ex, $"Prober for '{BackendId}' encounters unexpected exception.");
}
}
/// <summary>
/// Operates like <see cref="CancellationTokenSource.CancelAfter(TimeSpan)"/> but supporting specifying a custom timer.
/// </summary>
/// <param name="cancellationTokenSource">Token to cancel after expiration is complete.</param>
/// <param name="timeout">Timeout after which the cancellationTokenSource will be canceled.</param>
/// <param name="timer">Timer to perform the measurement of time for determining when to cancel.</param>
public static async void CancelAfter(this CancellationTokenSource cancellationTokenSource, TimeSpan timeout, IMonotonicTimer timer)
{
if (timer == null)
{
throw new ArgumentNullException(nameof(timer));
}
try
{
await timer.Delay(timeout, cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
}
catch (ObjectDisposedException)
{
// Ignore disposed cancellation tokens. Indicates cancellation is no longer needed. Unfortunately CTS's don't give a good
// way to safely check async disposal, so must rely on exception handling instead.
}
catch (OperationCanceledException)
{
// It cts was canceled, then there's no need for us to cancel the token. Return successfully.
// Note that we can't avoid this situation in advance as we strongly desire here to retain the 'void' returning
// interface that cts.CancelAfter(ts) has.
}
}
