private async Task ProbeCluster(ClusterState cluster)
{
var config = cluster.Model.Config.HealthCheck?.Active;
if (config == null || !config.Enabled.GetValueOrDefault())
{
return;
}
Log.StartingActiveHealthProbingOnCluster(_logger, cluster.ClusterId);
var allDestinations = cluster.DestinationsState.AllDestinations;
var probeTasks = new Task <DestinationProbingResult> [allDestinations.Count];
var probeResults = new DestinationProbingResult[probeTasks.Length];
var timeout = config.Timeout ?? _monitorOptions.DefaultTimeout;
for (var i = 0; i < probeTasks.Length; i++)
{
probeTasks[i] = ProbeDestinationAsync(cluster, allDestinations[i], timeout);
}
for (var i = 0; i < probeResults.Length; i++)
{
probeResults[i] = await probeTasks[i];
}
try
{
var policy = _policies.GetRequiredServiceById(config.Policy, HealthCheckConstants.ActivePolicy.ConsecutiveFailures);
policy.ProbingCompleted(cluster, probeResults);
}
catch (Exception ex)
{
Log.ActiveHealthProbingFailedOnCluster(_logger, cluster.ClusterId, ex);
}
finally
{
try
{
foreach (var probeResult in probeResults)
{
probeResult.Response?.Dispose();
}
}
catch (Exception ex)
{
Log.ErrorOccuredDuringActiveHealthProbingShutdownOnCluster(_logger, cluster.ClusterId, ex);
}
Log.StoppedActiveHealthProbingOnCluster(_logger, cluster.ClusterId);
}
}
private async Task ProbeCluster(ClusterInfo cluster)
{
var clusterConfig = cluster.Config;
if (!clusterConfig.HealthCheckOptions.Active.Enabled)
{
return;
}
Log.StartingActiveHealthProbingOnCluster(_logger, cluster.ClusterId);
// Policy must always be present if the active health check is enabled for a cluster.
// It's validated and ensured by a configuration validator.
var policy = _policies.GetRequiredServiceById(clusterConfig.HealthCheckOptions.Active.Policy);
var allDestinations = cluster.DynamicState.AllDestinations;
var probeTasks = new List <(Task <HttpResponseMessage> Task, CancellationTokenSource Cts)>(allDestinations.Count);
try
{
foreach (var destination in allDestinations)
{
var timeout = clusterConfig.HealthCheckOptions.Active.Timeout ?? _monitorOptions.DefaultTimeout;
var cts = new CancellationTokenSource(timeout);
try
{
var request = _probingRequestFactory.CreateRequest(clusterConfig, destination.Config);
Log.SendingHealthProbeToEndpointOfDestination(_logger, request.RequestUri, destination.DestinationId, cluster.ClusterId);
probeTasks.Add((clusterConfig.HttpClient.SendAsync(request, cts.Token), cts));
}
catch (Exception ex)
{
// Log and suppress an exception to give a chance for all destinations to be probed.
Log.ActiveHealthProbeConstructionFailedOnCluster(_logger, destination.DestinationId, cluster.ClusterId, ex);
cts.Dispose();
}
}
var probingResults = new DestinationProbingResult[probeTasks.Count];
for (var i = 0; i < probeTasks.Count; i++)
{
HttpResponseMessage response = null;
ExceptionDispatchInfo edi = null;
try
{
response = await probeTasks[i].Task;
Log.DestinationProbingCompleted(_logger, allDestinations[i].DestinationId, cluster.ClusterId, (int)response.StatusCode);
}
catch (Exception ex)
{
edi = ExceptionDispatchInfo.Capture(ex);
Log.DestinationProbingFailed(_logger, allDestinations[i].DestinationId, cluster.ClusterId, ex);
}
probingResults[i] = new DestinationProbingResult(allDestinations[i], response, edi?.SourceException);
}
policy.ProbingCompleted(cluster, probingResults);
}
catch (Exception ex)
{
Log.ActiveHealthProbingFailedOnCluster(_logger, cluster.ClusterId, ex);
}
finally
{
foreach (var probeTask in probeTasks)
{
try
{
try
{
probeTask.Cts.Cancel();
}
catch (Exception ex)
{
// Suppress exceptions to ensure the task will be awaited.
Log.ErrorOccuredDuringActiveHealthProbingShutdownOnCluster(_logger, cluster.ClusterId, ex);
}
var response = await probeTask.Task;
response.Dispose();
}
catch (Exception ex)
{
// Suppress exceptions to ensure all responses get a chance to be disposed.
Log.ErrorOccuredDuringActiveHealthProbingShutdownOnCluster(_logger, cluster.ClusterId, ex);
}
finally
{
// Dispose CancellationTokenSource even if the response task threw an exception.
// Dispose() is not expected to throw here.
probeTask.Cts.Dispose();
}
}
Log.StoppedActiveHealthProbingOnCluster(_logger, cluster.ClusterId);
}
}
