public LoadBalancer(
IDiscovery discovery,
DeploymentIdentifier deploymentIdentifier,
ReachabilityCheck reachabilityCheck,
TrafficRoutingStrategy trafficRoutingStrategy,
Func <Node, DeploymentIdentifier, ReachabilityCheck, Action, NodeMonitoringState> createNodeMonitoringState,
Func <string, AggregatingHealthStatus> getAggregatingHealthStatus,
Func <DiscoveryConfig> getConfig,
IDateTime dateTime,
ILog log,
IEnvironment environment
)
{
DeploymentIdentifier = deploymentIdentifier;
Discovery = discovery;
ReachabilityCheck = reachabilityCheck;
TrafficRoutingStrategy = trafficRoutingStrategy;
CreateNodeMonitoringState = createNodeMonitoringState;
GetConfig = getConfig;
DateTime = dateTime;
_lastUsageTime = DateTime.UtcNow;
Log = log;
var aggregatingHealthStatus = getAggregatingHealthStatus(deploymentIdentifier.ServiceName);
string healthCheckEntryName = (deploymentIdentifier.DeploymentEnvironment ?? "prod") + deploymentIdentifier.Zone == environment.Zone ? "" : $" ({deploymentIdentifier.Zone})";
_healthCheck = aggregatingHealthStatus.Register(healthCheckEntryName, CheckHealth);
}
public NewServiceDiscovery(string serviceName,
ReachabilityCheck reachabilityCheck,
IEnvironment environment,
ISourceBlock <DiscoveryConfig> configListener,
Func <DiscoveryConfig> discoveryConfigFactory,
ILog log,
IDiscovery discovery,
Func <string, AggregatingHealthStatus> getAggregatingHealthStatus)
{
Log = log;
_discovery = discovery;
_serviceName = serviceName;
_originatingEnvironmentDeployment = new DeploymentIdentifier(serviceName, environment.DeploymentEnvironment, environment);
_masterDeployment = new DeploymentIdentifier(serviceName, MASTER_ENVIRONMENT, environment);
_reachabilityCheck = reachabilityCheck;
GetConfig = discoveryConfigFactory;
_initTask = Task.Run(() => ReloadRemoteHost(discoveryConfigFactory()));
_configBlockLink = configListener.LinkTo(new ActionBlock <DiscoveryConfig>(ReloadRemoteHost));
AggregatingHealthStatus = getAggregatingHealthStatus("Discovery");
_healthCheck = AggregatingHealthStatus.RegisterCheck(_serviceName, () => new ValueTask <HealthCheckResult>(_getHealthStatus()));
_getHealthStatus = () => HealthCheckResult.Healthy("Initializing. Service was not discovered yet");
}
public void Setup()
{
_log = (LogSpy)_kernel.Get <ILog>();
_discovery = Substitute.For <IDiscovery>();
_discovery.GetNodes(Arg.Any <DeploymentIdentifier>()).Returns(_ => Task.FromResult(_getSourceNodes()));
_reachabilityCheck = (n, c) => throw new EnvironmentException("node is unreachable");
_environment = Substitute.For <IEnvironment>();
}
public NodeMonitoringState(Node node, DeploymentIdentifier deploymentIdentifier, ReachabilityCheck reachabilityCheck, Action reachabilityChanged, ILog log)
{
Node = node;
DeploymentIdentifier = deploymentIdentifier;
ReachabilityCheck = reachabilityCheck;
ReachabilityChanged = reachabilityChanged;
Log = log;
}
public async Task GetNode_NodesUnreachableButReachabilityCheckThrows_ErrorIsLogged()
{
CreateLoadBalancer();
SetupDefaultNodes();
var reachabilityException = new Exception("Simulated error while running reachability check");
_reachabilityCheck = (_, __) => throw reachabilityException;
await Run20Times(node => _loadBalancer.ReportUnreachable(node));
await Task.Delay(1500);
_log.LogEntries.ToArray().ShouldContain(e => e.Exception == reachabilityException);
}
public async Task GetNode_NodeIsReachableAgain_NodeWillBeReturned()
{
CreateLoadBalancer();
SetupDefaultNodes();
var selectedNode = await _loadBalancer.TryGetNode();
_loadBalancer.ReportUnreachable(selectedNode);
(await Get20Nodes()).ShouldNotContain(selectedNode);
_reachabilityCheck = (_, __) => Task.FromResult(true);
await Task.Delay(1000);
(await Get20Nodes()).ShouldContain(selectedNode);
}
public MultiEnvironmentServiceDiscovery(string serviceName, IEnvironment environment, ReachabilityCheck reachabilityCheck,
IDiscovery discovery, Func <DiscoveryConfig> getDiscoveryConfig, Func <string, AggregatingHealthStatus> getAggregatingHealthStatus,
IDateTime dateTime)
{
_healthStatus = new HealthMessage(Health.Info, message: null, suppressMessage: true);
ServiceName = serviceName;
Environment = environment;
ReachabilityCheck = reachabilityCheck;
Discovery = discovery;
GetDiscoveryConfig = getDiscoveryConfig;
DateTime = dateTime;
_lastUsageTime = DateTime.UtcNow;
var aggregatingHealthStatus = getAggregatingHealthStatus(serviceName);
_healthCheck = aggregatingHealthStatus.Register(Environment.DeploymentEnvironment, CheckHealth);
}
public async Task GetNode_AllNodesUnreachableThenAllNodesReachable_ReturnsAllNodes()
{
CreateLoadBalancer();
SetupSourceNodes(_node1, _node2, _node3);
await Run20Times(node => _loadBalancer.ReportUnreachable(node));
Should.Throw <EnvironmentException>(() => _loadBalancer.TryGetNode());
_reachabilityCheck = (_, __) => Task.FromResult(true);
await Task.Delay(1000);
var nodes = await Get20Nodes();
nodes.ShouldContain(_node1);
nodes.ShouldContain(_node2);
nodes.ShouldContain(_node3);
}
public LoadBalancer(
IDiscovery discovery,
DeploymentIdentifier deploymentIdentifier,
ReachabilityCheck reachabilityCheck,
TrafficRoutingStrategy trafficRoutingStrategy,
Func <Node, DeploymentIdentifier, ReachabilityCheck, Action, NodeMonitoringState> createNodeMonitoringState,
IHealthMonitor healthMonitor,
IDateTime dateTime,
ILog log)
{
DeploymentIdentifier = deploymentIdentifier;
Discovery = discovery;
ReachabilityCheck = reachabilityCheck;
TrafficRoutingStrategy = trafficRoutingStrategy;
CreateNodeMonitoringState = createNodeMonitoringState;
DateTime = dateTime;
Log = log;
_healthMonitor = healthMonitor.SetHealthFunction(DeploymentIdentifier.ToString(), () => new ValueTask <HealthCheckResult>(_healthStatus));
}
public async Task GetNode_TwoNodesUnreachable_OneBecomesReachable_ReturnOnlyReachableNode()
{
Node nodeToBeUnreachable = null;
_reachabilityCheck = async(n, c) =>
{
// ReSharper disable once AccessToModifiedClosure
if (Equals(n, nodeToBeUnreachable))
{
throw new Exception("This node is still unreachable");
}
};
CreateLoadBalancer();
var allNodes = new[] { _node1, _node2, _node3 };
SetupSourceNodes(allNodes);
nodeToBeUnreachable = await _loadBalancer.TryGetNode();
var nodeToBeReachable = await GetDifferentNode(nodeToBeUnreachable);
_loadBalancer.ReportUnreachable(nodeToBeReachable);
_loadBalancer.ReportUnreachable(nodeToBeUnreachable);
await Task.Delay(1000);
var nodes = await Get20Nodes();
foreach (var node in allNodes)
{
if (node.Equals(nodeToBeUnreachable))
{
nodes.ShouldNotContain(node);
}
else
{
nodes.ShouldContain(node);
}
}
}
public async Task GetNode_NodeUnreachableThenReturnsInBackground_NodeShouldBeReturned()
{
CreateLoadBalancer();
SetupDefaultNodes();
_reachabilityCheck = (_, __) => throw new EnvironmentException("node is unreachable");
var selectedNode = await _loadBalancer.TryGetNode();
_loadBalancer.ReportUnreachable(selectedNode);
(await Get20Nodes()).ShouldNotContain(selectedNode);
var waitForReachablitiy = new TaskCompletionSource <bool>();
_reachabilityCheck = (_, __) =>
{
waitForReachablitiy.SetResult(true);
return(Task.FromResult(true));
};
await waitForReachablitiy.Task;
await Task.Delay(50);
(await Get20Nodes()).ShouldContain(selectedNode);
}
public ILoadBalancer CreateLoadBalancer(DeploymentIdentifier deploymentIdentifier, ReachabilityCheck reachabilityCheck, TrafficRoutingStrategy trafficRoutingStrategy)
{
return(_createLoadBalancer(deploymentIdentifier, new LocalNodeSource(), reachabilityCheck, trafficRoutingStrategy));
}
