private HealthCheckResult CheckHealth()
{
var config = GetConfig();
if (IsHealthCheckSuppressed(config))
{
return(HealthCheckResult.Healthy($"Health check suppressed because service was not in use for more than {config.SuppressHealthCheckAfterServiceUnused.TotalSeconds} seconds."));
}
int reachableCount;
int unreachableCount;
Exception exception;
string[] unreachableHosts;
lock (_lock)
{
reachableCount = ReachableHosts.Count;
unreachableHosts = UnreachableHosts.Select(x => $"{x.HostName}:{x.Port}").ToArray();
unreachableCount = unreachableHosts.Length;
exception = UnreachableHosts.FirstOrDefault()?.LastException;
}
if (reachableCount == 0)
{
return(HealthCheckResult.Unhealthy($"All of the {unreachableCount} hosts are unreachable: " +
$"{string.Join(",", unreachableHosts)}. Last exception: {HealthMonitor.GetMessages(exception)}"));
}
else
{
if (unreachableCount > 0)
{
return(HealthCheckResult.Unhealthy($"The following {unreachableCount} hosts " +
$"(out of {unreachableCount + reachableCount}) are unreachable: {string.Join(",", unreachableHosts)}. " +
$"Last exception: {HealthMonitor.GetMessages(exception)}"));
}
else
{
return(HealthCheckResult.Healthy($"All {reachableCount} hosts are reachable."));
}
}
}
public void Reload()
{
var errors = new List <ValidationResult>();
JObject config = null;
object updatedConfig = null;
try
{
config = ConfigCache.CreateJsonConfig(ConfigPath) ?? Empty;
if (JToken.DeepEquals(LatestNode, config))
{
if (Latest != null)
{
ValidationErrors = null;
}
return;
}
}
catch (Exception ex)
{
errors.Add(new ValidationResult("Failed to acquire config JObject: " + HealthMonitor.GetMessages(ex)));
}
if (config != null && errors.Any() == false)
{
LatestNode = config;
try
{
updatedConfig = LatestNode.ToObject(ObjectType);
}
catch (JsonException ex)
{
errors.Add(new ValidationResult("Failed to deserialize config object: " + HealthMonitor.GetMessages(ex)));
}
if (updatedConfig != null)
{
Validator.TryValidateObjectRecursive(updatedConfig, errors);
}
}
if (errors.Any() == false)
{
Latest = updatedConfig;
ValidationErrors = null;
UsageTracking.AddConfigObject(Latest, ConfigPath);
Log.Info(_ => _("A config object has been updated", unencryptedTags: new
{
ConfigObjectType = ObjectType.FullName,
ConfigObjectPath = ConfigPath
}));
SendChangeNotification?.Invoke(Latest);
}
else
{
ValidationErrors = string.Join(" \n", errors.Select(a => a.ErrorMessage));
Log.Error(_ => _("A config object has been updated but failed validation", unencryptedTags: new
{
ConfigObjectType = ObjectType.FullName,
ConfigObjectPath = ConfigPath,
ValidationErrors
}));
}
}
private HealthCheckResult HealthCheck()
{
var failedQueries = _failedQueries.ToArray();
var nonExistQueries = _existingServices.Where(_ => _.Value == false).Select(_ => _.Key).ToArray();
var nonExistQueriesStr = nonExistQueries.Any() ? $"The following queries do not exist: {string.Join(", ", nonExistQueries)}" : string.Empty;
if (failedQueries.Any() == false)
{
return(HealthCheckResult.Healthy(nonExistQueriesStr));
}
else
{
var failedQueriesStr = "The following queries failed: " + string.Join(", ", failedQueries.Select(q => $"\"{q.Key}\": {HealthMonitor.GetMessages(q.Value)}"));
return(HealthCheckResult.Unhealthy(nonExistQueriesStr + "\r\n" + failedQueriesStr));
}
}
public void Reload()
{
var errors = new List <ValidationResult>();
JObject config = null;
object updatedConfig = null;
try
{
config = ConfigCache.CreateJsonConfig(ConfigPath) ?? Empty;
if (JToken.DeepEquals(LatestNode, config))
{
if (Latest != null)
{
ValidationErrors = null;
}
return;
}
}
catch (Exception ex)
{
errors.Add(new ValidationResult("Failed to acquire config JObject: " + HealthMonitor.GetMessages(ex)));
}
if (config != null && errors.Any() == false)
{
try
{
updatedConfig = config.ToObject(ObjectType);
}
catch (Exception ex)
{
// It is not only JsonException, as sometimes a custom deserializer capable to throw god knows what (including ProgrammaticException)
errors.Add(new ValidationResult("Failed to deserialize config object: " + HealthMonitor.GetMessages(ex)));
}
if (updatedConfig != null)
{
Validator.TryValidateObjectRecursive(updatedConfig, errors);
}
}
if (errors.Any() == false)
{
ValidationErrors = null;
UsageTracking.AddConfigObject(Latest, ConfigPath);
if (isCreated)
{
Log.Info(_ => _("A config object has been updated",
unencryptedTags: new {
ConfigObjectType = ObjectType.FullName,
ConfigObjectPath = ConfigPath,
OverallModifyTime = ConfigCache.LatestConfigFileModifyTime,
},
encryptedTags: new {
Changes = DiffJObjects(LatestNode, config, new StringBuilder(), new Stack <string>()).ToString(),
}));
}
else//It mean we are first time not need to send update messsage
{
isCreated = true;
}
LatestNode = config;
Latest = updatedConfig;
SendChangeNotification?.Invoke(Latest);
}
else
{
ValidationErrors = string.Join(" \n", errors.Select(a => a.ErrorMessage));
Log.Error(_ => _("A config object has been updated but failed validation", unencryptedTags: new
{
ConfigObjectType = ObjectType.FullName,
ConfigObjectPath = ConfigPath,
ValidationErrors
}));
}
}
/// <summary>
/// Loads the specified settings, overwriting existing settings.
/// </summary>
/// <param name="updatedEndPointsResult"></param>
/// this <see cref="RemoteHostPool"/>.
/// <exception cref="ArgumentNullException">Thrown when </exception>
/// <exception cref="EnvironmentException"></exception>
private void ReloadEndpoints(EndPointsResult updatedEndPointsResult)
{
lock (_lock)
{
try
{
var updatedEndPoints = updatedEndPointsResult.EndPoints;
if (updatedEndPoints.Any() == false)
{
Health.SetHealthFunction(() =>
{
var config = GetConfig();
if (IsHealthCheckSuppressed(config))
{
return(new ValueTask <HealthCheckResult>(HealthCheckResult.Healthy(
$"No endpoints were discovered from source '{config.Source}' but the remote service was not in use for more than {config.SuppressHealthCheckAfterServiceUnused.TotalSeconds} seconds.")));
}
else
{
return(new ValueTask <HealthCheckResult>(HealthCheckResult.Unhealthy(
$"No endpoints were discovered from source '{config.Source}'.")));
}
});
EndPointsResult = updatedEndPointsResult;
ReachableHosts = new List <RemoteHost>();
UnreachableHosts = new List <RemoteHost>();
}
else
{
if (EndPoints != null)
{
foreach (var removedEndPoint in EndPoints.Except(updatedEndPoints))
{
ReachableHosts.SingleOrDefault(h => h.Equals(removedEndPoint))?.StopMonitoring();
ReachableHosts.RemoveAll(h => h.Equals(removedEndPoint));
UnreachableHosts.RemoveAll(h => h.Equals(removedEndPoint));
}
}
var newHosts = updatedEndPoints
.Except(EndPoints ?? Enumerable.Empty <EndPoint>())
.Select(ep => new RemoteHost(ep.HostName, this, _lock, ep.Port));
ReachableHosts.AddRange(newHosts);
EndPointsResult = updatedEndPointsResult;
Counter = (ulong)_random.Next(0, ReachableHosts.Count);
Health.SetHealthFunction(CheckHealth);
}
_endPointsChanged.Post(EndPointsResult);
}
catch (Exception ex)
{
Log.Warn("Failed to process newly discovered endpoints", exception: ex);
Health.SetHealthFunction(() =>
new ValueTask <HealthCheckResult>(HealthCheckResult.Unhealthy("Failed to process newly discovered endpoints: " +
HealthMonitor.GetMessages(ex))));
}
}
}