private HealthCheckResult CheckHealth() { var config = GetConfig(); if (IsHealthCheckSuppressed(config)) { return(HealthCheckResult.Healthy($"Health check suppressed because service was not in use for more than {config.SuppressHealthCheckAfterServiceUnused.TotalSeconds} seconds.")); } int reachableCount; int unreachableCount; Exception exception; string[] unreachableHosts; lock (_lock) { reachableCount = ReachableHosts.Count; unreachableHosts = UnreachableHosts.Select(x => $"{x.HostName}:{x.Port}").ToArray(); unreachableCount = unreachableHosts.Length; exception = UnreachableHosts.FirstOrDefault()?.LastException; } if (reachableCount == 0) { return(HealthCheckResult.Unhealthy($"All of the {unreachableCount} hosts are unreachable: " + $"{string.Join(",", unreachableHosts)}. Last exception: {HealthMonitor.GetMessages(exception)}")); } else { if (unreachableCount > 0) { return(HealthCheckResult.Unhealthy($"The following {unreachableCount} hosts " + $"(out of {unreachableCount + reachableCount}) are unreachable: {string.Join(",", unreachableHosts)}. " + $"Last exception: {HealthMonitor.GetMessages(exception)}")); } else { return(HealthCheckResult.Healthy($"All {reachableCount} hosts are reachable.")); } } }
public void Reload() { var errors = new List <ValidationResult>(); JObject config = null; object updatedConfig = null; try { config = ConfigCache.CreateJsonConfig(ConfigPath) ?? Empty; if (JToken.DeepEquals(LatestNode, config)) { if (Latest != null) { ValidationErrors = null; } return; } } catch (Exception ex) { errors.Add(new ValidationResult("Failed to acquire config JObject: " + HealthMonitor.GetMessages(ex))); } if (config != null && errors.Any() == false) { LatestNode = config; try { updatedConfig = LatestNode.ToObject(ObjectType); } catch (JsonException ex) { errors.Add(new ValidationResult("Failed to deserialize config object: " + HealthMonitor.GetMessages(ex))); } if (updatedConfig != null) { Validator.TryValidateObjectRecursive(updatedConfig, errors); } } if (errors.Any() == false) { Latest = updatedConfig; ValidationErrors = null; UsageTracking.AddConfigObject(Latest, ConfigPath); Log.Info(_ => _("A config object has been updated", unencryptedTags: new { ConfigObjectType = ObjectType.FullName, ConfigObjectPath = ConfigPath })); SendChangeNotification?.Invoke(Latest); } else { ValidationErrors = string.Join(" \n", errors.Select(a => a.ErrorMessage)); Log.Error(_ => _("A config object has been updated but failed validation", unencryptedTags: new { ConfigObjectType = ObjectType.FullName, ConfigObjectPath = ConfigPath, ValidationErrors })); } }
private HealthCheckResult HealthCheck() { var failedQueries = _failedQueries.ToArray(); var nonExistQueries = _existingServices.Where(_ => _.Value == false).Select(_ => _.Key).ToArray(); var nonExistQueriesStr = nonExistQueries.Any() ? $"The following queries do not exist: {string.Join(", ", nonExistQueries)}" : string.Empty; if (failedQueries.Any() == false) { return(HealthCheckResult.Healthy(nonExistQueriesStr)); } else { var failedQueriesStr = "The following queries failed: " + string.Join(", ", failedQueries.Select(q => $"\"{q.Key}\": {HealthMonitor.GetMessages(q.Value)}")); return(HealthCheckResult.Unhealthy(nonExistQueriesStr + "\r\n" + failedQueriesStr)); } }
public void Reload() { var errors = new List <ValidationResult>(); JObject config = null; object updatedConfig = null; try { config = ConfigCache.CreateJsonConfig(ConfigPath) ?? Empty; if (JToken.DeepEquals(LatestNode, config)) { if (Latest != null) { ValidationErrors = null; } return; } } catch (Exception ex) { errors.Add(new ValidationResult("Failed to acquire config JObject: " + HealthMonitor.GetMessages(ex))); } if (config != null && errors.Any() == false) { try { updatedConfig = config.ToObject(ObjectType); } catch (Exception ex) { // It is not only JsonException, as sometimes a custom deserializer capable to throw god knows what (including ProgrammaticException) errors.Add(new ValidationResult("Failed to deserialize config object: " + HealthMonitor.GetMessages(ex))); } if (updatedConfig != null) { Validator.TryValidateObjectRecursive(updatedConfig, errors); } } if (errors.Any() == false) { ValidationErrors = null; UsageTracking.AddConfigObject(Latest, ConfigPath); if (isCreated) { Log.Info(_ => _("A config object has been updated", unencryptedTags: new { ConfigObjectType = ObjectType.FullName, ConfigObjectPath = ConfigPath, OverallModifyTime = ConfigCache.LatestConfigFileModifyTime, }, encryptedTags: new { Changes = DiffJObjects(LatestNode, config, new StringBuilder(), new Stack <string>()).ToString(), })); } else//It mean we are first time not need to send update messsage { isCreated = true; } LatestNode = config; Latest = updatedConfig; SendChangeNotification?.Invoke(Latest); } else { ValidationErrors = string.Join(" \n", errors.Select(a => a.ErrorMessage)); Log.Error(_ => _("A config object has been updated but failed validation", unencryptedTags: new { ConfigObjectType = ObjectType.FullName, ConfigObjectPath = ConfigPath, ValidationErrors })); } }
/// <summary> /// Loads the specified settings, overwriting existing settings. /// </summary> /// <param name="updatedEndPointsResult"></param> /// this <see cref="RemoteHostPool"/>. /// <exception cref="ArgumentNullException">Thrown when </exception> /// <exception cref="EnvironmentException"></exception> private void ReloadEndpoints(EndPointsResult updatedEndPointsResult) { lock (_lock) { try { var updatedEndPoints = updatedEndPointsResult.EndPoints; if (updatedEndPoints.Any() == false) { Health.SetHealthFunction(() => { var config = GetConfig(); if (IsHealthCheckSuppressed(config)) { return(new ValueTask <HealthCheckResult>(HealthCheckResult.Healthy( $"No endpoints were discovered from source '{config.Source}' but the remote service was not in use for more than {config.SuppressHealthCheckAfterServiceUnused.TotalSeconds} seconds."))); } else { return(new ValueTask <HealthCheckResult>(HealthCheckResult.Unhealthy( $"No endpoints were discovered from source '{config.Source}'."))); } }); EndPointsResult = updatedEndPointsResult; ReachableHosts = new List <RemoteHost>(); UnreachableHosts = new List <RemoteHost>(); } else { if (EndPoints != null) { foreach (var removedEndPoint in EndPoints.Except(updatedEndPoints)) { ReachableHosts.SingleOrDefault(h => h.Equals(removedEndPoint))?.StopMonitoring(); ReachableHosts.RemoveAll(h => h.Equals(removedEndPoint)); UnreachableHosts.RemoveAll(h => h.Equals(removedEndPoint)); } } var newHosts = updatedEndPoints .Except(EndPoints ?? Enumerable.Empty <EndPoint>()) .Select(ep => new RemoteHost(ep.HostName, this, _lock, ep.Port)); ReachableHosts.AddRange(newHosts); EndPointsResult = updatedEndPointsResult; Counter = (ulong)_random.Next(0, ReachableHosts.Count); Health.SetHealthFunction(CheckHealth); } _endPointsChanged.Post(EndPointsResult); } catch (Exception ex) { Log.Warn("Failed to process newly discovered endpoints", exception: ex); Health.SetHealthFunction(() => new ValueTask <HealthCheckResult>(HealthCheckResult.Unhealthy("Failed to process newly discovered endpoints: " + HealthMonitor.GetMessages(ex)))); } } }