private static async Task <HealthCheckResult> ExecuteHealthCheckWithRetriesAsync(
Uri uri,
int retries,
TimeSpan delayBetweenRetries,
TimeSpan timeoutPerRequest,
bool degradedOnError,
CancellationToken cancellationToken)
{
var sw = new Stopwatch();
var attempts = 0;
try
{
sw.Start();
do
{
try
{
attempts++;
using (var tokenWithTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
tokenWithTimeout.CancelAfter(timeoutPerRequest);
var response = await HttpClient.GetAsync(uri, tokenWithTimeout.Token);
if (attempts == retries || response.IsSuccessStatusCode)
{
return(response.IsSuccessStatusCode
? HealthCheckResult.Healthy(
$"OK. '{uri}' success. Total Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.")
: HealthCheckResultOnError(
$"FAILED. '{uri}' status code was {response.StatusCode}. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
degradedOnError));
}
if (response.StatusCode == HttpStatusCode.GatewayTimeout ||
response.StatusCode == HttpStatusCode.ServiceUnavailable)
{
Logger.Error(
$"HTTP Health Check '{uri}' failed with status code {response.StatusCode}. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.");
Logger.Info(
$"Retrying HTTP Health Check '{uri}' in {delayBetweenRetries.TotalMilliseconds}ms. {attempts} / {retries} retries.");
await Task.Delay(delayBetweenRetries, cancellationToken);
}
}
}
catch (Exception ex) when(ex is TaskCanceledException)
{
Logger.ErrorException(
$"HTTP Health Check '{uri}' did not respond within '{timeoutPerRequest.TotalMilliseconds}'ms. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
ex);
if (attempts == retries)
{
return(HealthCheckResultOnError(
$"FAILED. '{uri}' did not respond within {timeoutPerRequest.TotalMilliseconds}ms. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
degradedOnError));
}
await Retry(uri, retries, delayBetweenRetries, attempts, ex, cancellationToken);
}
catch (Exception ex) when(ex is TimeoutException)
{
Logger.ErrorException(
$"HTTP Health Check '{uri}' timed out. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
ex);
if (attempts == retries)
{
return(HealthCheckResultOnError(
$"FAILED. '{uri}' timed out. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
degradedOnError));
}
await Retry(uri, retries, delayBetweenRetries, attempts, ex, cancellationToken);
}
catch (Exception ex) when(ex is HttpRequestException)
{
Logger.ErrorException(
$"HTTP Health Check '{uri}' failed. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
ex);
if (attempts == retries)
{
return(HealthCheckResultOnError(
$"FAILED. '{uri}' request failed with an unexpected error. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
degradedOnError));
}
await Retry(uri, retries, delayBetweenRetries, attempts, ex, cancellationToken);
}
catch (Exception ex)
{
var message =
$"HTTP Health Check failed to request '{uri}'. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.";
if (attempts == retries)
{
Logger.ErrorException(message, ex);
return(HealthCheckResultOnError(
$"FAILED. {message}.",
degradedOnError));
}
await Retry(uri, retries, delayBetweenRetries, attempts, ex, cancellationToken);
}
}while (true);
}
catch (Exception ex) when(ex is TaskCanceledException)
{
Logger.ErrorException(
$"HTTP Health Check '{uri}' did not respond within '{timeoutPerRequest.TotalMilliseconds}'ms. Attempts: {attempts}.",
ex);
return(HealthCheckResultOnError(
$"FAILED. '{uri}' did not respond within {timeoutPerRequest.TotalMilliseconds}ms. Attempts: {attempts}.",
degradedOnError));
}
catch (Exception ex)
{
var message = $"HTTP Health Check failed to request '{uri}'. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.";
Logger.ErrorException(message, ex);
return(HealthCheckResultOnError(
$"FAILED. {message}",
degradedOnError));
}
finally
{
sw.Stop();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="HealthCheck" /> class.
/// </summary>
/// <param name="name">A descriptive name for the health check.</param>
/// <param name="check">A function returning a message that is a healthy result.</param>
public HealthCheck(string name, Func <Task <string> > check)
: this(name, async() => HealthCheckResult.Healthy(await check()))
{
}
private static async Task <HealthCheckResult> ExecuteHttpCheckNoRetriesAsync(
Uri uri,
HttpContent content,
TimeSpan timeout,
bool degradedOnError,
CancellationToken cancellationToken,
Dictionary <string, string> requestHeaders = null)
{
var sw = new Stopwatch();
try
{
using (var tokenWithTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
tokenWithTimeout.CancelAfter(timeout);
sw.Start();
if (requestHeaders != null)
{
foreach (var header in requestHeaders)
{
HttpClient.DefaultRequestHeaders.Add(header.Key, header.Value);
}
}
var response = await HttpClient.PostAsync(uri, content, tokenWithTimeout.Token)
.ConfigureAwait(false);
return(response.IsSuccessStatusCode
? HealthCheckResult.Healthy($"OK. '{uri}' success. Time taken: {sw.ElapsedMilliseconds}ms.")
: HealthCheckResultOnError(
$"FAILED. '{uri}' status code was {response.StatusCode}. Time taken: {sw.ElapsedMilliseconds}ms.",
degradedOnError));
}
}
catch (Exception ex) when(ex is TaskCanceledException)
{
Logger.ErrorException(
$"HTTP Health Check '{uri}' did not respond within '{timeout.TotalMilliseconds}'ms.", ex);
return(HealthCheckResultOnError($"FAILED. '{uri}' did not respond within {timeout.TotalMilliseconds}ms",
degradedOnError));
}
catch (Exception ex) when(ex is TimeoutException)
{
Logger.ErrorException($"HTTP Health Check '{uri}' timed out. Time taken: {sw.ElapsedMilliseconds}ms.",
ex);
return(HealthCheckResultOnError($"FAILED. '{uri}' timed out. Time taken: {sw.ElapsedMilliseconds}ms.",
degradedOnError));
}
catch (Exception ex) when(ex is HttpRequestException)
{
Logger.ErrorException($"HTTP Health Check '{uri}' failed. Time taken: {sw.ElapsedMilliseconds}ms.", ex);
return(HealthCheckResultOnError(
$"FAILED. '{uri}' request failed with an unexpected error. Time taken: {sw.ElapsedMilliseconds}ms.",
degradedOnError));
}
catch (Exception ex)
{
var message = $"HTTP Health Check failed to request '{uri}'. Time taken: {sw.ElapsedMilliseconds}ms.";
Logger.ErrorException(message, ex);
return(HealthCheckResultOnError($"FAILED. {message}", degradedOnError));
}
finally
{
sw.Stop();
}
}