private async Task <LivenessExecution> GetLivenessExecution(LivenessConfiguration liveness)
{
return(await _context.LivenessExecutions
.Include(le => le.History)
.Where(le => le.LivenessName.Equals(liveness.LivenessName, StringComparison.InvariantCultureIgnoreCase))
.SingleOrDefaultAsync());
}
private async Task NotifyFailureIfIsConfigured(LivenessConfiguration item, string content)
{
_logger.LogWarning($"LivenessRuner notify liveness failure for {item.LivenessUri}.");
var lastNotification = _context.LivenessFailuresNotifications
.Where(lf => lf.LivenessName.Equals(item.LivenessName, StringComparison.InvariantCultureIgnoreCase))
.OrderByDescending(lf => lf.LastNotified)
.Take(1)
.SingleOrDefault();
if (lastNotification != null
&&
(DateTime.UtcNow - lastNotification.LastNotified).Seconds < _settings.MinimunSecondsBetweenFailureNotifications)
{
_logger.LogInformation("Notification is not performed becaused is already notified and the elapsed time is less than configured.");
}
else
{
await _failureNotifier.NotifyFailure(item.LivenessName, content);
var notification = new LivenessFailureNotification()
{
LivenessName = item.LivenessName,
LastNotified = DateTime.UtcNow
};
await SaveNotification(notification);
_logger.LogWarning("A new notification failure is created and sent.");
}
}
private async Task SaveExecutionHistory(LivenessConfiguration liveness, string content, bool isHealthy)
{
_logger.LogDebug("LivenessRuner save a new liveness execution history.");
var livenessExecution = await _context.LivenessExecutions
.Include(le => le.History)
.Where(le => le.LivenessName.Equals(liveness.LivenessName, StringComparison.InvariantCultureIgnoreCase))
.SingleOrDefaultAsync();
var currentStatus = GetDetailedStatusFromContent(isHealthy, content);
var currentStatusName = Enum.GetName(typeof(LivenessStatus), currentStatus);
var lastExecutionTime = DateTime.UtcNow;
if (livenessExecution != null)
{
if (livenessExecution.Status.Equals(currentStatusName, StringComparison.InvariantCultureIgnoreCase))
{
_logger.LogDebug("LivenessExecutionHistory already exist and is in the same state, update the values.");
livenessExecution.LastExecuted = lastExecutionTime;
livenessExecution.LivenessResult = content;
}
else
{
_logger.LogDebug("LivenessExecutionHistory already exist but on different state, update the values.");
livenessExecution.History.Add(new LivenessExecutionHistory()
{
On = lastExecutionTime,
Status = livenessExecution.Status
});
livenessExecution.IsHealthy = isHealthy;
livenessExecution.OnStateFrom = lastExecutionTime;
livenessExecution.LastExecuted = lastExecutionTime;
livenessExecution.LivenessResult = content;
livenessExecution.Status = currentStatusName;
}
}
else
{
_logger.LogDebug("LivenessExecutionHistory is a new liveness execution history.");
livenessExecution = new LivenessExecution()
{
IsHealthy = isHealthy,
LastExecuted = lastExecutionTime,
OnStateFrom = lastExecutionTime,
LivenessResult = content,
Status = currentStatusName,
LivenessName = liveness.LivenessName,
LivenessUri = liveness.LivenessUri,
};
await _context.LivenessExecutions
.AddAsync(livenessExecution);
}
await _context.SaveChangesAsync();
}
private async Task <(string response, bool ishealthy, LivenessConfiguration livenessConfiguration)> EvaluateLiveness(
LivenessConfiguration livenessConfiguration,
CancellationToken cancellationToken)
{
var(uri, name) = livenessConfiguration;
try
{
using (var response = await PerformRequest(uri, cancellationToken))
{
var success = response.IsSuccessStatusCode;
var content = await response.Content
.ReadAsStringAsync();
return(content, success, livenessConfiguration);
}
}
catch (Exception exception)
{
_logger.LogError(exception, "LivenessRunner EvaluateLiveness throw the exception.");
return(exception.Message, false, livenessConfiguration);
}
}
public LivenessContextBuilder WithLiveness(LivenessConfiguration liveness)
{
if (liveness != null)
{
_configurations.Add(liveness);
}
return(this);
}
public LivenessHostedService(
IContentProvider contentProvider,
IOptions <ClusterExporterConfiguration> clusterExporterConfiguration,
IOptions <LivenessConfiguration> configuration,
ILogger <LivenessHostedService> logger)
{
_contentProvider = contentProvider;
_clusterExporterConfiguration = clusterExporterConfiguration.Value;
_configuration = configuration.Value;
_logger = logger;
}
private async Task <bool> HasLivenessRecoveredFromFailure(LivenessConfiguration liveness)
{
var previousLivenessExecution = await GetLivenessExecution(liveness);
if (previousLivenessExecution != null)
{
var previousStatus = (LivenessStatus)Enum.Parse(typeof(LivenessStatus), previousLivenessExecution.Status);
return(previousStatus != LivenessStatus.Up);
}
return(false);
}
public void Validate_Attributes_Should_Pass(LivenessConfiguration conf, bool expectedResult)
{
try
{
conf.Validate();
}
catch (Exception e)
{
if (expectedResult)
{
throw;
}
e.Should().BeOfType <AggregateException>();
}
}