private void UpdateRuntimeBackends(DynamicConfigRoot config)
{
var desiredBackends = new HashSet <string>(StringComparer.Ordinal);
foreach (var configBackendPair in config.Backends)
{
var configBackend = configBackendPair.Value;
desiredBackends.Add(configBackendPair.Key);
_backendManager.GetOrCreateItem(
itemId: configBackendPair.Key,
setupAction: backend =>
{
UpdateRuntimeEndpoints(configBackend.Endpoints, backend.EndpointManager);
var newConfig = new BackendConfig(
new BackendConfig.BackendHealthCheckOptions(
enabled: configBackend.HealthCheckOptions?.Enabled ?? false,
interval: configBackend.HealthCheckOptions?.Interval ?? TimeSpan.FromSeconds(0),
timeout: configBackend.HealthCheckOptions?.Timeout ?? TimeSpan.FromSeconds(0),
port: configBackend.HealthCheckOptions?.Port ?? 0,
path: configBackend.HealthCheckOptions?.Path ?? string.Empty),
new BackendConfig.BackendLoadBalancingOptions(
mode: configBackend.LoadBalancing?.Mode ?? default));
var currentBackendConfig = backend.Config.Value;
if (currentBackendConfig == null ||
currentBackendConfig.HealthCheckOptions.Enabled != newConfig.HealthCheckOptions.Enabled ||
currentBackendConfig.HealthCheckOptions.Interval != newConfig.HealthCheckOptions.Interval ||
currentBackendConfig.HealthCheckOptions.Timeout != newConfig.HealthCheckOptions.Timeout ||
currentBackendConfig.HealthCheckOptions.Port != newConfig.HealthCheckOptions.Port ||
currentBackendConfig.HealthCheckOptions.Path != newConfig.HealthCheckOptions.Path)
{
if (currentBackendConfig == null)
{
_logger.LogDebug("Backend {backendId} has been added.", configBackendPair.Key);
}
else
{
_logger.LogDebug("Backend {backendId} has changed.", configBackendPair.Key);
}
// Config changed, so update runtime backend
backend.Config.Value = newConfig;
}
});
}
foreach (var existingBackend in _backendManager.GetItems())
{
if (!desiredBackends.Contains(existingBackend.BackendId))
{
// NOTE 1: This is safe to do within the `foreach` loop
// because `IBackendManager.GetItems` returns a copy of the list of backends.
//
// NOTE 2: Removing the backend from `IBackendManager` is safe and existing
// ASP .NET Core endpoints will continue to work with their existing behavior (until those endpoints are updated)
// and the Garbage Collector won't destroy this backend object while it's referenced elsewhere.
_logger.LogDebug("Backend {backendId} has been removed.", existingBackend.BackendId);
_backendManager.TryRemoveItem(existingBackend.BackendId);
}
}
}
public async Task ApplyConfigurationsAsync_OneBackendOneEndpointOneRoute_Works()
{
// Arrange
const string TestAddress = "https://localhost:123/";
var backend = new Backend
{
Endpoints =
{
{ "ep1", new BackendEndpoint {
Address = TestAddress
} }
}
};
var route = new ParsedRoute
{
RouteId = "route1",
BackendId = "backend1",
};
var dynamicConfigRoot = new DynamicConfigRoot
{
Backends = new Dictionary <string, Backend> {
{ "backend1", backend }
},
Routes = new[] { route },
};
Mock <IDynamicConfigBuilder>()
.Setup(d => d.BuildConfigAsync(It.IsAny <IConfigErrorReporter>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(Result.Success(dynamicConfigRoot));
var errorReporter = new TestConfigErrorReporter();
var proxyManager = Create <ReverseProxyConfigManager>();
// Act
var result = await proxyManager.ApplyConfigurationsAsync(errorReporter, CancellationToken.None);
// Assert
Assert.True(result);
var actualBackends = _backendManager.GetItems();
Assert.Single(actualBackends);
Assert.Equal("backend1", actualBackends[0].BackendId);
Assert.NotNull(actualBackends[0].EndpointManager);
Assert.NotNull(actualBackends[0].Config.Value);
var actualEndpoints = actualBackends[0].EndpointManager.GetItems();
Assert.Single(actualEndpoints);
Assert.Equal("ep1", actualEndpoints[0].EndpointId);
Assert.NotNull(actualEndpoints[0].Config.Value);
Assert.Equal(TestAddress, actualEndpoints[0].Config.Value.Address);
var actualRoutes = _routeManager.GetItems();
Assert.Single(actualRoutes);
Assert.Equal("route1", actualRoutes[0].RouteId);
Assert.NotNull(actualRoutes[0].Config.Value);
Assert.Same(actualBackends[0], actualRoutes[0].Config.Value.BackendOrNull);
}
public async Task UpdateTrackedBackends()
{
// TODO: (future) It's silly to go through all backends just to see which ones changed, if any.
// We should use Signals instead.
// Step 1: Get the backend table.
var proberAdded = 0;
var backends = _backendManager.GetItems();
var backendIdList = new HashSet <string>();
// Step 2: Start background tasks to probe each backend at their configured intervals. The number of concurrent probes are controlled by semaphore.
var stopTasks = new List <Task>();
foreach (var backend in backends)
{
backendIdList.Add(backend.BackendId);
var backendConfig = backend.Config.Value;
var createNewProber = false;
// Step 3A: Check whether this backend already has a prober.
if (_activeProbers.TryGetValue(backend.BackendId, out var activeProber))
{
// Step 3B: Check if the config of backend has changed.
if (backendConfig == null ||
!backendConfig.HealthCheckOptions.Enabled ||
activeProber.Config != backendConfig)
{
// Current prober is not using latest config, stop and remove the outdated prober.
_logger.LogInformation($"Health check work stopping prober for '{backend.BackendId}'.");
stopTasks.Add(activeProber.StopAsync());
_activeProbers.Remove(backend.BackendId);
// And create a new prober if needed
createNewProber = backendConfig?.HealthCheckOptions.Enabled ?? false;
}
}
else
{
// Step 3C: New backend we aren't probing yet, set up a new prober.
createNewProber = backendConfig?.HealthCheckOptions.Enabled ?? false;
}
if (!createNewProber)
{
var reason = backendConfig == null ? "no backend configuration" : $"{nameof(backendConfig.HealthCheckOptions)}.{nameof(backendConfig.HealthCheckOptions.Enabled)} = false";
_logger.LogInformation($"Health check prober for backend: '{backend.BackendId}' is disabled because {reason}.");
}
// Step 4: New prober need to been created, start the new registered prober.
if (createNewProber)
{
// Start probing health for all endpoints(replica) in this backend(service).
var newProber = _backendProberFactory.CreateBackendProber(backend.BackendId, backendConfig, backend.EndpointManager);
_activeProbers.Add(backend.BackendId, newProber);
proberAdded++;
_logger.LogInformation($"Prober for backend '{backend.BackendId}' has created.");
newProber.Start(_semaphore);
}
}
// Step 5: Stop and remove probes for backends that no longer exist.
var probersToRemove = new List <string>();
foreach (var kvp in _activeProbers)
{
if (!backendIdList.Contains(kvp.Key))
{
// Gracefully shut down the probe.
_logger.LogInformation($"Health check work stopping prober for '{kvp.Key}'.");
stopTasks.Add(kvp.Value.StopAsync());
probersToRemove.Add(kvp.Key);
}
}
// remove the probes for backends that were removed.
probersToRemove.ForEach(p => _activeProbers.Remove(p));
await Task.WhenAll(stopTasks);
_logger.LogInformation($"Health check prober are updated. Added {proberAdded} probes, removed {probersToRemove.Count} probes. There are now {_activeProbers.Count} probes.");
}