internal async Task <ConsulResponse <ConsulNode[]> > GetHealthyNodes(DeploymentIdentifier deploymentIdentifier, ulong modifyIndex, CancellationToken cancellationToken)
{
string urlCommand = $"v1/health/service/{deploymentIdentifier.GetConsulServiceName()}?dc={deploymentIdentifier.Zone}&passing&index={modifyIndex}&wait={GetConfig().HttpTimeout.TotalSeconds}s";
var response = await Call <ConsulNode[]>(urlCommand, cancellationToken).ConfigureAwait(false);
if (response.StatusCode != HttpStatusCode.OK)
{
if (response.Error == null)
{
response.Error = response.ConsulResponseCodeNotOk();
}
}
else
{
try
{
var serviceEntries = JsonConvert.DeserializeObject <ServiceEntry[]>(response.ResponseContent);
response.ResponseObject = serviceEntries.Select(ToNode).ToArray();
}
catch (Exception ex)
{
response.Error = response.UnparsableConsulResponse(ex);
}
}
return(response);
}
public void Setup()
{
_consulSimulator.Reset();
_deploymentIdentifier = new DeploymentIdentifier(ServiceName + "_" + Guid.NewGuid(), Env, _environment);
_consulConfig = new ConsulConfig();
}
private string GetConfiguredSourceType(DeploymentIdentifier deploymentIdentifier)
{
var config = GetConfig();
var serviceConfig = config.Services[deploymentIdentifier.ServiceName];
return(serviceConfig.Source);
}
public LoadBalancer(
IDiscovery discovery,
DeploymentIdentifier deploymentIdentifier,
ReachabilityCheck reachabilityCheck,
TrafficRoutingStrategy trafficRoutingStrategy,
Func <Node, DeploymentIdentifier, ReachabilityCheck, Action, NodeMonitoringState> createNodeMonitoringState,
Func <string, AggregatingHealthStatus> getAggregatingHealthStatus,
Func <DiscoveryConfig> getConfig,
IDateTime dateTime,
ILog log,
IEnvironment environment
)
{
DeploymentIdentifier = deploymentIdentifier;
Discovery = discovery;
ReachabilityCheck = reachabilityCheck;
TrafficRoutingStrategy = trafficRoutingStrategy;
CreateNodeMonitoringState = createNodeMonitoringState;
GetConfig = getConfig;
DateTime = dateTime;
_lastUsageTime = DateTime.UtcNow;
Log = log;
var aggregatingHealthStatus = getAggregatingHealthStatus(deploymentIdentifier.ServiceName);
string healthCheckEntryName = (deploymentIdentifier.DeploymentEnvironment ?? "prod") + deploymentIdentifier.Zone == environment.Zone ? "" : $" ({deploymentIdentifier.Zone})";
_healthCheck = aggregatingHealthStatus.Register(healthCheckEntryName, CheckHealth);
}
public NewServiceDiscovery(string serviceName,
ReachabilityCheck reachabilityCheck,
IEnvironment environment,
ISourceBlock <DiscoveryConfig> configListener,
Func <DiscoveryConfig> discoveryConfigFactory,
ILog log,
IDiscovery discovery,
Func <string, AggregatingHealthStatus> getAggregatingHealthStatus)
{
Log = log;
_discovery = discovery;
_serviceName = serviceName;
_originatingEnvironmentDeployment = new DeploymentIdentifier(serviceName, environment.DeploymentEnvironment, environment);
_masterDeployment = new DeploymentIdentifier(serviceName, MASTER_ENVIRONMENT, environment);
_reachabilityCheck = reachabilityCheck;
GetConfig = discoveryConfigFactory;
_initTask = Task.Run(() => ReloadRemoteHost(discoveryConfigFactory()));
_configBlockLink = configListener.LinkTo(new ActionBlock <DiscoveryConfig>(ReloadRemoteHost));
AggregatingHealthStatus = getAggregatingHealthStatus("Discovery");
_healthCheck = AggregatingHealthStatus.RegisterCheck(_serviceName, () => new ValueTask <HealthCheckResult>(_getHealthStatus()));
_getHealthStatus = () => HealthCheckResult.Healthy("Initializing. Service was not discovered yet");
}
/// <summary>
/// Creates a new RemoteHostPool using the specified system name and liveliness checker.
/// </summary>
/// <param name="reachabilityChecker">A delegate that checks if a given host is reachable or not. Used for background checks of unreachable hosts.
/// Should return true if the host is reachable, or false if it is unreachable. It should not throw an exception.</param>
/// <param name="log">An implementation of <see cref="ILog"/> used for logging.</param>
public RemoteHostPool(
DeploymentIdentifier deploymentIdentifier
, IServiceDiscoverySource discovery
, ReachabilityChecker reachabilityChecker
, Func <DiscoveryConfig> getDiscoveryConfig
, ILog log
, HealthMonitor healthMonitor
, MetricsContext metrics
)
{
DiscoverySource = discovery;
DeploymentIdentifier = deploymentIdentifier;
ReachabilityChecker = reachabilityChecker;
GetDiscoveryConfig = getDiscoveryConfig;
Log = log;
ReachabilityBroadcaster = new BroadcastBlock <ServiceReachabilityStatus>(null);
Health = healthMonitor.Get(discovery.Deployment);
Health.SetHealthData(HealthData);
ReachableHosts = new List <RemoteHost>();
UnreachableHosts = new List <RemoteHost>();
EndPointsChangedBlockLink = discovery.EndPointsChanged.LinkTo(new ActionBlock <EndPointsResult>(_ => ReloadEndpoints(_)));
ReloadEndpoints(discovery.Result);
Metrics = metrics;
var metricsContext = Metrics.Context(DiscoverySource.Deployment);
metricsContext.Gauge("ReachableHosts", () => ReachableHosts.Count, Unit.Custom("EndPoints"));
metricsContext.Gauge("UnreachableHosts", () => UnreachableHosts.Count, Unit.Custom("EndPoints"));
}
/// <inheritdoc />
private async Task <INodeSource> CreateNodeSource(string sourceType, DeploymentIdentifier deploymentIdentifier)
{
INodeSource nodeSource;
switch (sourceType)
{
case "Config":
nodeSource = CreateConfigNodeSource(deploymentIdentifier); break;
case "Local":
nodeSource = CreateLocalNodeSource(deploymentIdentifier); break;
default:
if (NodeSourceFactories.TryGetValue(sourceType, out var factory))
{
nodeSource = await factory.CreateNodeSource(deploymentIdentifier).ConfigureAwait(false);
}
else
{
throw new ConfigurationException($"Discovery Source '{sourceType}' is not supported.");
}
break;
}
return(nodeSource);
}
public void Setup()
{
var environment = Substitute.For <IEnvironment>();
var deployment = new DeploymentIdentifier(ServiceName, "prod", environment);
_configNodeSource = _kernel.Get <Func <DeploymentIdentifier, ConfigNodeSource> >()(deployment);
}
/// <inheritdoc />
public async Task <Node[]> GetNodes(DeploymentIdentifier deploymentIdentifier)
{
// We have a cached node source; query it
if (_nodeSources.TryGetValue(deploymentIdentifier, out Lazy <NodeSourceAndAccesstime> lazySource))
{
lazySource.Value.LastAccessTime = DateTime.UtcNow;
var nodeSource = await lazySource.Value.NodeSourceTask.ConfigureAwait(false);
return(nodeSource.GetNodes());
}
// No node source but the service is deployed; create one and query it
else if (await IsServiceDeployed(deploymentIdentifier).ConfigureAwait(false))
{
string sourceType = GetConfiguredSourceType(deploymentIdentifier);
lazySource = _nodeSources.GetOrAdd(deploymentIdentifier, _ => new Lazy <NodeSourceAndAccesstime>(() =>
new NodeSourceAndAccesstime {
NodeSourceType = sourceType,
LastAccessTime = DateTime.UtcNow,
NodeSourceTask = CreateNodeSource(sourceType, deploymentIdentifier)
}));
var nodeSource = await lazySource.Value.NodeSourceTask.ConfigureAwait(false);
return(nodeSource.GetNodes());
}
// No node source and the service is not deployed; return null
else
{
return(null);
}
}
private ILoadBalancer CreateLoadBalancerMock(DeploymentIdentifier di)
{
var mock = Substitute.For <ILoadBalancer>();
mock.TryGetNode().Returns(_ => _consulServiceList.Contains(di) ? _nodeResults[di]() : null);
return(mock);
}
public NodeMonitoringState(Node node, DeploymentIdentifier deploymentIdentifier, ReachabilityCheck reachabilityCheck, Action reachabilityChanged, ILog log)
{
Node = node;
DeploymentIdentifier = deploymentIdentifier;
ReachabilityCheck = reachabilityCheck;
ReachabilityChanged = reachabilityChanged;
Log = log;
}
private ILoadBalancer GetLoadBalancerMock(DeploymentIdentifier di)
{
if (_loadBalancers.ContainsKey(di))
{
return(_loadBalancers[di]);
}
return(CreateLoadBalancerMock(di));
}
private void CreateConsulMock(DeploymentIdentifier di)
{
var mock = CreateLoadBalancerMock(di);
_nodeResults[di] = () => new ConsulNode(hostName: "dummy", version: ServiceVersion);
_loadBalancers[di] = mock;
_consulServiceList.Add(di);
}
private object NodeUnencryptedTags() => new
{
deploymentIdentifier = DeploymentIdentifier.ToString(),
hostname = Node.Hostname,
port = Node.Port,
attemptCount = AttemptCount,
nextDelay = NextDelay,
nextAttemptAt = DateTime.UtcNow + NextDelay,
downtime = DateTime.UtcNow - StartTime
};
public async Task <bool> IsServiceDeployed(DeploymentIdentifier deploymentIdentifier)
{
await _initCompleted.Task.ConfigureAwait(false);
if (Services.Count == 0 && Error != null)
{
throw Error;
}
return(Services.Contains(deploymentIdentifier.GetConsulServiceName()));
}
public override void Setup()
{
base.Setup();
_dateTimeFake = new DateTimeFake();
_createdNodeSources = new List <Type>();
SetupConsulNodeSource();
SetupSlowNodeSource();
_discovery = _unitTestingKernel.Get <IDiscovery>();
_deploymentIdentifier = new DeploymentIdentifier(ServiceName, Env, Substitute.For <IEnvironment>());
}
private void SetMockToReturnHost(DeploymentIdentifier di)
{
if (!_loadBalancers.ContainsKey(di))
{
CreateConsulMock(di);
}
var newNode = new Node(HostnameFor(di));
_nodeResults[di] = () => newNode;
_consulServiceList.Add(di);
}
public async Task <INodeSource> CreateNodeSource(DeploymentIdentifier deploymentIdentifier)
{
await _initCompleted.Task.ConfigureAwait(false);
if (await IsServiceDeployed(deploymentIdentifier).ConfigureAwait(false))
{
var consulNodeSource = CreateConsulNodeSource(deploymentIdentifier);
await consulNodeSource.Init().ConfigureAwait(false);
return(consulNodeSource);
}
return(null);
}
private async Task GetFirstResult()
{
var config = new ServiceDiscoveryConfig
{
Scope = _serviceScope,
};
var sourceFactory = Kernel.Get <Func <DeploymentIdentifier, ServiceDiscoveryConfig, ConsulDiscoverySource> >();
var serviceContext = new DeploymentIdentifier(SERVICE_NAME, ENV, _environmentMock);
_consulDiscoverySource = sourceFactory(serviceContext, config);
await _consulDiscoverySource.Init();
await GetNewResult();
}
public void Setup()
{
_dateTimeFake = new DateTimeFake();
_createdNodeSources = new List <Type>();
SetupConsulNodeSource();
SetupSlowNodeSource();
_discoveryConfig = new DiscoveryConfig();
_discoveryConfig.Services = new ServiceDiscoveryCollection(new Dictionary <string, ServiceDiscoveryConfig>(), new ServiceDiscoveryConfig(), new PortAllocationConfig());
_discovery = _kernel.Get <IDiscovery>();
_deploymentIdentifier = new DeploymentIdentifier(ServiceName, Env, Substitute.For <IEnvironment>());
}
private async Task <NodeAndLoadBalancer> GetNodeAndLoadBalancer(string environment, string preferredEnvironment)
{
var loadBalancer = _loadBalancers.GetOrAdd(environment, p => {
var deploymentId = new DeploymentIdentifier(ServiceName, environment, Environment.Zone);
return(Discovery.CreateLoadBalancer(deploymentId, ReachabilityCheck, TrafficRoutingStrategy.RandomByRequestID));
});
Node node = await loadBalancer.TryGetNode().ConfigureAwait(false);
if (node == null)
{
return(null);
}
return(new NodeAndLoadBalancer {
Node = node, LoadBalancer = loadBalancer, PreferredEnvironment = preferredEnvironment
});
}
public LoadBalancer(
IDiscovery discovery,
DeploymentIdentifier deploymentIdentifier,
ReachabilityCheck reachabilityCheck,
TrafficRoutingStrategy trafficRoutingStrategy,
Func <Node, DeploymentIdentifier, ReachabilityCheck, Action, NodeMonitoringState> createNodeMonitoringState,
IHealthMonitor healthMonitor,
IDateTime dateTime,
ILog log)
{
DeploymentIdentifier = deploymentIdentifier;
Discovery = discovery;
ReachabilityCheck = reachabilityCheck;
TrafficRoutingStrategy = trafficRoutingStrategy;
CreateNodeMonitoringState = createNodeMonitoringState;
DateTime = dateTime;
Log = log;
_healthMonitor = healthMonitor.SetHealthFunction(DeploymentIdentifier.ToString(), () => new ValueTask <HealthCheckResult>(_healthStatus));
}
public async Task Setup()
{
_consulSimulator.Reset();
_testingKernel = new TestingKernel <ConsoleLog>(k =>
{
_environment = Substitute.For <IEnvironment>();
_environment.ConsulAddress.Returns($"{CurrentApplicationInfo.HostName}:{ConsulPort}");
_environment.Zone.Returns(Zone);
k.Rebind <IEnvironment>().ToMethod(_ => _environment);
k.Rebind <Func <ConsulConfig> >().ToMethod(_ => () => _consulConfig);
k.Rebind <ConsulNodeSourceFactory>().ToSelf().InTransientScope();
});
_serviceName = $"MyService_{Guid.NewGuid().ToString().Substring(5)}";
_deploymentIdentifier = new DeploymentIdentifier(_serviceName, "prod", Substitute.For <IEnvironment>());
_consulConfig = new ConsulConfig {
ErrorRetryInterval = TimeSpan.FromMilliseconds(10)
};
_consulNodeSourceFactory = _testingKernel.Get <ConsulNodeSourceFactory>();
}
private async Task <bool> IsServiceDeployed(DeploymentIdentifier deploymentIdentifier)
{
var sourceType = GetConfiguredSourceType(deploymentIdentifier);
switch (sourceType)
{
case "Config":
case "Local":
return(true);
default:
if (NodeSourceFactories.TryGetValue(sourceType, out var factory))
{
return(await factory.IsServiceDeployed(deploymentIdentifier).ConfigureAwait(false));
}
else
{
throw new ConfigurationException($"Discovery Source '{sourceType}' is not supported.");
}
}
}
public async Task <Node> TryGetNode()
{
_lastUsageTime = System.DateTime.UtcNow;
await LoadNodesFromSource().ConfigureAwait(false);
if (_isUndeployed)
{
return(null);
}
var nodes = _nodesMonitoringState;
if (!nodes.Any())
{
throw new ServiceUnreachableException("No nodes were discovered for service",
LastException,
unencrypted: new Tags
{
{ "deploymentIdentifier", DeploymentIdentifier.ToString() },
});
}
var reachableNodes = _reachableNodes; // get current state of reachable nodes
if (!reachableNodes.Any())
{
throw new ServiceUnreachableException("All nodes are unreachable",
nodes.FirstOrDefault(n => n.LastException != null)?.LastException,
unencrypted: new Tags
{
{ "deploymentIdentifier", DeploymentIdentifier.ToString() },
{ "nodes", string.Join(",", nodes.Select(n => n.Node.ToString())) }
});
}
var index = GetIndexByTrafficRoutingStrategy();
return(reachableNodes[index % reachableNodes.Length]);
}
/// <summary>
/// Creates a new RemoteHostPool using the specified system name and liveliness checker.
/// </summary>
/// <param name="reachabilityChecker">A delegate that checks if a given host is reachable or not. Used for background checks of unreachable hosts.
/// Should return true if the host is reachable, or false if it is unreachable. It should not throw an exception.</param>
/// <param name="log">An implementation of <see cref="ILog"/> used for logging.</param>
public RemoteHostPool(
DeploymentIdentifier deploymentIdentifier
, IServiceDiscoverySource discovery
, ReachabilityChecker reachabilityChecker
, Func <DiscoveryConfig> getDiscoveryConfig
, ILog log
, HealthMonitor healthMonitor
)
{
DiscoverySource = discovery;
DeploymentIdentifier = deploymentIdentifier;
ReachabilityChecker = reachabilityChecker;
GetDiscoveryConfig = getDiscoveryConfig;
Log = log;
ReachabilityBroadcaster = new BroadcastBlock <ServiceReachabilityStatus>(null);
Health = healthMonitor.Get(discovery.Deployment);
Health.SetHealthData(HealthData);
ReachableHosts = new List <RemoteHost>();
UnreachableHosts = new List <RemoteHost>();
EndPointsChangedBlockLink = discovery.EndPointsChanged.LinkTo(new ActionBlock <EndPointsResult>(_ => ReloadEndpoints(_)));
ReloadEndpoints(discovery.Result);
}
internal async Task <ConsulResponse <string> > GetDeploymentVersion(DeploymentIdentifier deploymentIdentifier, ulong modifyIndex, CancellationToken cancellationToken)
{
string version = null;
var response = await GetKey <ServiceKeyValue>(modifyIndex, "service", deploymentIdentifier.GetConsulServiceName(), deploymentIdentifier.Zone, cancellationToken);
if (response.StatusCode == HttpStatusCode.NotFound)
{
response.IsUndeployed = true;
}
else if (response.StatusCode != HttpStatusCode.OK)
{
if (response.Error == null)
{
response.Error = response.ConsulResponseCodeNotOk();
}
}
else
{
version = response.ResponseObject?.Version;
response.IsUndeployed = false;
}
return(response.SetResult(version));
}
public async Task <Node[]> GetNodes(DeploymentIdentifier deploymentIdentifier)
{
return(new LocalNodeSource().GetNodes());
}
public ILoadBalancer CreateLoadBalancer(DeploymentIdentifier deploymentIdentifier, ReachabilityCheck reachabilityCheck, TrafficRoutingStrategy trafficRoutingStrategy)
{
return(_createLoadBalancer(deploymentIdentifier, new LocalNodeSource(), reachabilityCheck, trafficRoutingStrategy));
}
private string HostnameFor(DeploymentIdentifier di) => $"{di.DeploymentEnvironment}-host";
