/// <summary>
/// Called when [health check status changed].
/// </summary>
/// <param name="status">The status.</param>
private void OnHealthCheckStatusChanged(HealthCheckStatus status, ServerObject server)
{
if (this.HealthCheckStatusChanged != null)
{
this.HealthCheckStatusChanged(status, server);
}
}
public HealthCheckOutcome(HealthCheckStatus status, string message, IndividualHealthCheckResult[] results, DateTimeOffset timestamp)
{
_status = status;
_message = message;
_results = results;
_timestamp = timestamp;
}
public HealthCheckStatus Check()
{
HealthCheckStatus healthCheckStatus1 = new HealthCheckStatus()
{
DescriptionKey = "HealthCheckVersionHealthOkText", ImageSource = "..\\Resources\\okCheckMark.png", CanContinue = true
};
HealthCheckStatus healthCheckStatus2 = new HealthCheckStatus()
{
DescriptionKey = "HealthCheckVersionHealthErrorText", ImageSource = "..\\Resources\\errorCheckMark.png", CanContinue = false, ReadMoreKey = "HealthCheckVersionHealthFullDescriptionText"
};
try
{
if (this._isCurrentVersionProvider.IsCurrentVersion())
{
return(healthCheckStatus1);
}
healthCheckStatus2.MustUpdate = true;
return(healthCheckStatus2);
}
catch (Exception ex)
{
return(healthCheckStatus2);
}
}
/// <inheritdoc />
public async Task UpdateStatusAsync(CancellationToken ct)
{
Status = await Cache.GetOrCreateAsync(CacheKey, async k =>
{
Logger.LogDebug(
"Expired cache status. Updating to the most recent status [CacheExpiration:'{expiration}']...",
Properties.Expiration);
k.AbsoluteExpirationRelativeToNow = Properties.Expiration;
if (Properties.CacheExceptions)
{
try
{
await HealthCheck.UpdateStatusAsync(ct);
}
catch (Exception e)
{
Logger.LogWarning(e, "Inner health check failed but status 'Red' will be cached");
return(HealthCheckStatus.Red);
}
}
else
{
await HealthCheck.UpdateStatusAsync(ct);
}
return(HealthCheck.Status);
});
}
public async Task Should_Check_Status_If_Outdated()
{
var oldHealthCheckStatus =
new HealthCheckStatus(HealthStatus.Unhealthy, "Some error message")
{
DateTime = DateTime.UtcNow.AddMinutes(-2)
};
_storageMock.Setup(s => s.GetCurrentStatusAsync(It.IsAny <string>()))
.ReturnsAsync(oldHealthCheckStatus);
var newHealthCheckStatus = new HealthCheckStatus(HealthStatus.Healthy);
_serviceMock.Setup(s => s.CheckHealthAsync(It.IsAny <CancellationToken>()))
.ReturnsAsync(newHealthCheckStatus);
var check = new TestPeriodicHealthCheck(_storageMock.Object, _serviceMock.Object, TimeSpan.FromMinutes(1));
var status = await check.CheckHealthAsync(new HealthCheckContext());
Assert.Equal(Microsoft.Extensions.Diagnostics.HealthChecks.HealthStatus.Healthy, status.Status);
Assert.Null(status.Exception);
Assert.Null(status.Description);
_serviceMock.Verify(s => s.CheckHealthAsync(It.IsAny <CancellationToken>()), Times.Once);
_storageMock.Verify(s => s.CheckedAsync(TestPeriodicHealthCheck.CheckServiceName, newHealthCheckStatus), Times.Once);
}
private HealthCheckStatus GetCdfFolder()
{
var path = ClientDependency.Core.Config.ClientDependencySettings.Instance.DefaultCompositeFileProcessingProvider.CompositeFilePath;
var files = path.GetFiles();
var msg = string.Format("CDF Path: {0}", path);
var status = new HealthCheckStatus(msg)
{
ResultType = StatusResultType.Info,
Description = string.Format("There are currently {1} files in the cdf folder. Clearing these files will refresh both front end and back end scripts and stylesheets", path, files.Count()),
};
if (files.Count() > 0)
{
status.Actions = new List <HealthCheckAction>()
{
new HealthCheckAction("Clear", this.Id)
{
Name = "Clear"
}
};
}
return(status);
}
public HealthCheckResult(string checkName)
{
// Assume Unknown until we hear otherwise
HealthCheckStatus = HealthCheckStatus.Unknown;
CheckName = checkName;
_resultMessages = new List <string>();
}
protected string GetStatusAsColorStyle(HealthCheckStatus status)
{
string color;
switch (status)
{
case HealthCheckStatus.OK:
color = "green";
break;
case HealthCheckStatus.Warning:
color = "orange";
break;
case HealthCheckStatus.Critical:
color = "red";
break;
case HealthCheckStatus.Unknown:
color = "orange";
break;
default:
throw new ArgumentOutOfRangeException(nameof(status), status, null);
}
return($"color:{color}");
}
public OverallHealthCheckStatus Check()
{
OverallHealthCheckStatus healthCheckStatus1 = new OverallHealthCheckStatus();
List <HealthCheckStatus> source = new List <HealthCheckStatus>();
bool flag1 = true;
bool flag2 = false;
foreach (IHealthCheckService healthCheckService in (IEnumerable <IHealthCheckService>) this.healthChecks.OrderBy <IHealthCheckService, int>((Func <IHealthCheckService, int>)(hc => hc.RunOrder)))
{
HealthCheckStatus healthCheckStatus2 = healthCheckService.Check();
source.Add(healthCheckStatus2);
flag1 = healthCheckStatus2.CanContinue & flag1;
flag2 = healthCheckStatus2.MustUpdate | flag2;
}
List <HealthCheckStatus> list = source.ToList <HealthCheckStatus>();
healthCheckStatus1.HealthCheckStatuses = (IEnumerable <HealthCheckStatus>)list;
healthCheckStatus1.CanContinue = flag1;
healthCheckStatus1.MustUpdate = flag2;
string str1 = "HealthCheckOverallStatusErrorText";
string str2 = "HealthCheckOverallStatusOkText";
healthCheckStatus1.OverallStatusTextKey = flag1 ? str2 : str1;
return(healthCheckStatus1);
}
public static async Task <StatusReport> WaitForHealthCheckAsync(this Route53Helper r53h, string name,
HealthCheckStatus status, //Unhealthy, Healthy
int timeout_s)
{
var sw = Stopwatch.StartNew();
var hc = await r53h.GetHealthCheckAsync(name, throwIfNotFound : true);
StatusReport report;
var healthyStatus = hc.HealthCheckConfig.Inverted ? HealthCheckStatus.Unhealthy : HealthCheckStatus.Healthy;
var unHealthyStatus = hc.HealthCheckConfig.Inverted ? HealthCheckStatus.Healthy : HealthCheckStatus.Unhealthy;
do
{
var hcs = await r53h.GetHealthCheckStatusAsync(hc.Id);
report = hcs.HealthCheckObservations.OrderByDescending(x => x.StatusReport.CheckedTime).First().StatusReport;
if ((report.Status.ContainsAny("Success", HealthCheckStatus.Healthy.ToString()) && status == healthyStatus) ||
(report.Status.ContainsAny("Failure", HealthCheckStatus.Unhealthy.ToString()) && status == unHealthyStatus))
{
return(report);
}
await _locker.Lock(() => Task.Delay(1000));
}while (sw.ElapsedMilliseconds < (timeout_s * 1000));
throw new Exception($"Health Check '{name}' coudn't reach '{status}' status within {timeout_s} [s], last state was: '{report.JsonSerialize(Newtonsoft.Json.Formatting.Indented)}'.");
}
public HealthCheckStatus Check()
{
HealthCheckStatus healthCheckStatus1 = new HealthCheckStatus()
{
DescriptionKey = "HealthCheckConnectionHealthOkText", ImageSource = "..\\Resources\\okCheckMark.png", CanContinue = true
};
HealthCheckStatus healthCheckStatus2 = new HealthCheckStatus()
{
DescriptionKey = "HealthCheckConnectionHealthErrorText", ImageSource = "..\\Resources\\errorCheckMark.png", CanContinue = false, ReadMoreKey = "HealthCheckConnectionHealthFullDescriptionText"
};
try
{
if (this._configurationService.GlobalResponse != null)
{
return(healthCheckStatus1);
}
GlobalResponse globalSettings = this._flexClient.GetGlobalSettings();
this._configurationService.GlobalResponse = globalSettings;
return(globalSettings != null ? healthCheckStatus1 : healthCheckStatus2);
}
catch (Exception ex)
{
return(healthCheckStatus2);
}
}
public HealthCheckSettings(Dictionary <string, string> config)
{
Status = config == null || !config.ContainsKey(STATUS)
? HealthCheckStatus.Undefined
: (BoolKey(config, STATUS) ? HealthCheckStatus.Enabled : HealthCheckStatus.Disabled);
IntervalInDays = IntKey(config, INTERVAL_IN_DAYS, DEFAULT_INTERVAL_IN_DAYS);
if (!Enum.TryParse(Get(config, DAY_OF_WEEK), out DayOfWeek))
{
DayOfWeek = (DayOfWeek)GetDefaultDay();
}
TimeOfDay = IntKey(config, TIME_OF_DAY, GetDefaultTime());
RetryInterval = IntKey(config, RETRY_INTERVAL, DEFAULT_RETRY_INTERVAL);
UploadTokenSecretUuid = Get(config, UPLOAD_TOKEN_SECRET);
DiagnosticTokenSecretUuid = Get(config, DIAGNOSTIC_TOKEN_SECRET);
NewUploadRequest = Get(config, NEW_UPLOAD_REQUEST);
UserNameSecretUuid = Get(config, UPLOAD_CREDENTIAL_USER_SECRET);
PasswordSecretUuid = Get(config, UPLOAD_CREDENTIAL_PASSWORD_SECRET);
LastSuccessfulUpload = Get(config, LAST_SUCCESSFUL_UPLOAD);
LastFailedUpload = Get(config, LAST_FAILED_UPLOAD);
UploadUuid = Get(config, UPLOAD_UUID);
ReportAnalysisSeverity = StringToDiagnosticAlertSeverity(Get(config, REPORT_ANALYSIS_SEVERITY));
ReportAnalysisIssuesDetected = IntKey(config, REPORT_ANALYSIS_ISSUES_DETECTED, 0);
ReportAnalysisUploadUuid = Get(config, REPORT_ANALYSIS_UPLOAD_UUID);
ReportAnalysisUploadTime = Get(config, REPORT_ANALYSIS_UPLOAD_TIME);
}
public HealthCheckOutcome(HealthCheckStatus status, string message, IndividualHealthCheckResult[] results, DateTimeOffset timestamp)
{
_status = status;
_message = message;
_results = results;
_timestamp = timestamp;
}
public HealthCheckStatus Check()
{
HealthCheckStatus healthCheckStatus1 = new HealthCheckStatus()
{
DescriptionKey = "HealthCheckFileAccessHealthOkText", ImageSource = "..\\Resources\\okCheckMark.png", CanContinue = true
};
HealthCheckStatus healthCheckStatus2 = new HealthCheckStatus()
{
DescriptionKey = "HealthCheckFileAccessHealthErrorText", ImageSource = "..\\Resources\\errorCheckMark.png", CanContinue = false, ReadMoreKey = "HealthCheckFileAccessHealthFullDescriptionText"
};
string temporaryFolderPath = this.configurationService.TemporaryFolderPath;
try
{
if (!this._driveService.HasRequiredAvailableDiskSpace(Path.GetPathRoot(temporaryFolderPath), this.configurationService.GlobalResponse.Configuration.MinimumDiskSpaceInMegabytes))
{
return(healthCheckStatus2);
}
this.fileService.WriteToFile(temporaryFolderPath + "\\file.temp", new byte[8]);
return(healthCheckStatus1);
}
catch (Exception ex)
{
return(healthCheckStatus2);
}
}
private void AddHealthAttachments(
HealthCheckStatus checkStatus,
IReadOnlyCollection <HealthCheck.Result> checks,
SlackPayload slackMessage,
bool reFailure = false)
{
var checkOrChecks = checks.Count > 1 ? "checks" : "check";
var textEnd = string.Empty;
var color = string.Empty;
if (checkStatus == HealthCheckStatus.Unhealthy)
{
textEnd = reFailure ? "still failing" : "failing";
color = FailingColor;
}
if (checkStatus == HealthCheckStatus.Degraded)
{
textEnd = reFailure ? "still degrading" : "degrading";
color = DegradingColor;
}
if (checkStatus == HealthCheckStatus.Healthy)
{
textEnd = "recovered";
color = SuccessColor;
}
var attachment = new SlackAttachment
{
Text = $"*_{checks.Count} {checkOrChecks} {textEnd}_ on server {Environment.MachineName}*\n\n",
Color = color,
Ts = (DateTime.UtcNow - Epoch).TotalSeconds
};
foreach (var result in checks)
{
if (result.Check.Status == HealthCheckStatus.Unhealthy)
{
LastDegradedCheckCache.Remove(result.Name);
LastUnhealthyCheckCache.Add(result.Name);
}
else if (result.Check.Status == HealthCheckStatus.Degraded)
{
LastUnhealthyCheckCache.Remove(result.Name);
LastDegradedCheckCache.Add(result.Name);
}
attachment.Fields.Add(
new SlackAttachmentFields
{
Title = $"{result.Name}",
Value = $"```{result.Check.Message}```",
Short = true
});
}
slackMessage.Attachments.Add(attachment);
}
private void ShouldReturnNullWhenHealthCheckStatusIsEmpty()
{
HealthCheckStatus healthCheckStatus = new HealthCheckStatus();
Dictionary <string, string> result = healthCheckStatus.GetStatus("health");
result.Should().BeNull();
}
public void TestDelegation()
{
var healthCheck = new DelegateHealthCheck("Test", () => Task.FromResult(HealthCheckStatus.Passed()));
Assert.Equal("Test", healthCheck.Name);
var result = healthCheck.Check().Result;
Assert.False(result.HasFailed);
}
/// <summary>
/// Initializes a new instance of the <see cref="OveralHealthCheckResult" /> class.
/// </summary>
/// <param name="status">The status.</param>
/// <param name="results">The individual health check results.</param>
internal OveralHealthCheckResult(
HealthCheckStatus status,
IList <BaseIndividualHealthCheckResult> results
)
{
Status = status;
Results = results;
Version = "1.0";
Timestamp = DateTime.UtcNow;
}
/// <summary>
/// Comports with Nagios performance format of 'label'=value[UOM];[warn];[crit];[min];[max]
/// See: https://nagios-plugins.org/doc/guidelines.html#THRESHOLDFORMAT
/// </summary>
/// <summary>
/// Gets more detailed secondary counters
/// </summary>
private string GetHealthCheckResultsAsPerformanceDataLongPerStatus(HealthCheckStatus healthCheckStatus)
{
var max = Results.Count;
var countInThisStatus = Results.Count(r => r.HealthCheckStatus == healthCheckStatus);
var warn = healthCheckStatus == HealthCheckStatus.OK ? Results.Count : 1;
var crit = warn;
const int min = 0;
return(FormatNagiosPerfCounterLine($"{healthCheckStatus}Count", countInThisStatus, warn, crit, min, max));
}
/// <summary>
/// Creates a new <see cref="HealthCheckResult"/> with the specified <paramref name="status"/>, <paramref name="exception"/>,
/// <paramref name="description"/>, and <paramref name="data"/>.
/// </summary>
/// <param name="status">A <see cref="HealthCheckStatus"/> value indicating the status of the component that was checked.</param>
/// <param name="exception">An <see cref="Exception"/> representing the exception that was thrown when checking for status (if any).</param>
/// <param name="description">A human-readable description of the status of the component that was checked.</param>
/// <param name="data">Additional key-value pairs describing the health of the component.</param>
public HealthCheckResult(HealthCheckStatus status, Exception exception, string description, IReadOnlyDictionary <string, object> data)
{
if (status == HealthCheckStatus.Unknown)
{
throw new ArgumentException($"'{nameof(HealthCheckStatus.Unknown)}' is not a valid value for the 'status' parameter.", nameof(status));
}
Status = status;
Exception = exception;
_description = description;
_data = data;
}
private HealthCheckStatus VerifyElasticsearchVersion()
{
var versionNumber = Task.Run(UmbracoSearchFactory.GetElasticsearchVersionAsync).Result;
var status = new HealthCheckStatus($"Elasticsearch version: <strong>{versionNumber}</strong>")
{
Description = "Elasticsearch cluster must be running at least version 5.0",
ResultType = versionNumber.Major < 5 ? StatusResultType.Error : StatusResultType.Info
};
return(status);
}
public override async Task <HealthCheckStatus> Check()
{
using (var connection = new SqlConnection(_connectionString))
{
await connection.OpenAsync().ConfigureAwait(false);
var cmd = new SqlCommand("select 1", connection);
await cmd.ExecuteScalarAsync().ConfigureAwait(false);
return(HealthCheckStatus.Passed());
}
}
public async Task Should_Store_Checks_For_Separate_Services(IHealthCheckStorage storage)
{
var unhealthy = new HealthCheckStatus(HealthStatus.Unhealthy);
var healthy = new HealthCheckStatus(HealthStatus.Healthy);
await storage.CheckedAsync("s1", unhealthy);
await storage.CheckedAsync("s2", healthy);
Assert.Equal(unhealthy, await storage.GetCurrentStatusAsync("s1"));
Assert.Equal(healthy, await storage.GetCurrentStatusAsync("s2"));
}
private void ShouldAddHealthCheckStatusWhenAddStatusIsInvoked()
{
Dictionary <string, string> status = new Dictionary <string, string> {
{ "Service1", "Unhealthy" }, { "Service2", "Healthy" }
};
HealthCheckStatus healthCheckStatus = new HealthCheckStatus();
healthCheckStatus.AddStatus("health", status);
Dictionary <string, string> result = healthCheckStatus.GetStatus("health");
result.Should().Equal(status);
}
public async Task can_execute_process_private_memory_check(HealthCheckStatus expectedResult, long thresholdBytes, bool degradedOnError = false)
{
var healthChecks = Enumerable.Empty <HealthCheck>();
var name = "private memory";
var factory = new HealthCheckFactory(_logger, new Lazy <IMetrics>(), healthChecks);
factory.RegisterProcessPrivateMemorySizeHealthCheck(name, thresholdBytes, degradedOnError);
var check = factory.Checks.FirstOrDefault();
var result = await check.Value.ExecuteAsync().ConfigureAwait(false);
result.Check.Status.Should().Be(expectedResult);
}
public async Task can_execute_ping_check(HealthCheckStatus expectedResult, string host, bool degradedOnError = false)
{
var healthChecks = Enumerable.Empty <HealthCheck>();
var name = "github ping";
var factory = new HealthCheckFactory(_logger, new Lazy <IMetrics>(), healthChecks);
factory.RegisterPingHealthCheck(name, host, TimeSpan.FromSeconds(5), degradedOnError: degradedOnError);
var check = factory.Checks.FirstOrDefault();
var result = await check.Value.ExecuteAsync().ConfigureAwait(false);
result.Check.Status.Should().Be(expectedResult);
}
public void Status_MatchesWorstStatusInResults(HealthCheckStatus statusValue)
{
var result = new CompositeHealthCheckResult(new Dictionary <string, HealthCheckResult>()
{
{ "Foo", HealthCheckResult.Healthy() },
{ "Bar", HealthCheckResult.Healthy() },
{ "Baz", new HealthCheckResult(statusValue, exception: null, description: null, data: null) },
{ "Quick", HealthCheckResult.Healthy() },
{ "Quack", HealthCheckResult.Healthy() },
{ "Quock", HealthCheckResult.Healthy() },
});
Assert.Equal(statusValue, result.Status);
}
public async Task Can_execute_process_virtual_memory_check(HealthCheckStatus expectedResult, long thresholdBytes, bool degradedOnError = false)
{
// Arrange
const string name = "virtual memory";
var builder = new HealthBuilder()
.HealthChecks.AddProcessVirtualMemorySizeCheck(name, thresholdBytes, degradedOnError: degradedOnError);
var health = builder.Build();
var check = health.Checks.First();
// Act
var result = await check.ExecuteAsync();
// Assert
result.Check.Status.Should().Be(expectedResult);
}
public async Task Should_Replace_Older_Value(IHealthCheckStorage storage)
{
var olderCheck = new HealthCheckStatus(HealthStatus.Unhealthy)
{
DateTime = DateTime.UtcNow.AddSeconds(-1)
};
var newerCheck = new HealthCheckStatus(HealthStatus.Healthy);
await storage.CheckedAsync("test", olderCheck);
Assert.Equal(olderCheck, await storage.GetCurrentStatusAsync("test"));
await storage.CheckedAsync("test", newerCheck);
Assert.Equal(newerCheck, await storage.GetCurrentStatusAsync("test"));
}
private void Notify(HealthCheckResult <THealthCheckParams, TResult> result)
{
HealthCheckStatus newStatus = result.Status;
if (newStatus == HealthCheckStatus.Healthy)
{
NotifyOnHealthyStatus(result);
}
else if (newStatus == HealthCheckStatus.Unhealthy)
{
NotifyOnUnHealthyStatus(result);
}
else
{
NotifyOnWarningStatus(result);
}
}
private void ShouldUpdateHealthCheckStatusWhenStatusAlreadyExists()
{
Dictionary <string, string> status1 = new Dictionary <string, string> {
{ "Service1", "Unhealthy" }, { "Service2", "Healthy" }
};
HealthCheckStatus healthCheckStatus = new HealthCheckStatus();
healthCheckStatus.AddStatus("health", status1);
Dictionary <string, string> status2 = new Dictionary <string, string> {
{ "Service1", "Healthy" }, { "Service2", "Healthy" }
};
healthCheckStatus.AddStatus("health", status2);
Dictionary <string, string> result = healthCheckStatus.GetStatus("health");
result.Should().Equal(status2);
}
