public async Task IsHealthOkChecksCosmosRepository()
{
bool expectedIsOkFlag = new RandomBoolean();
string expectedMessage = new RandomString();
GivenTheRepositoryServiceHealth(expectedIsOkFlag, expectedMessage);
ServiceHealth isHealthOk = await _repository.IsHealthOk();
isHealthOk
.Should()
.NotBeNull();
isHealthOk
.Name
.Should()
.Be(nameof(PublishedFundingRepository));
DependencyHealth dependencyHealth = isHealthOk
.Dependencies
.FirstOrDefault();
dependencyHealth
.Should()
.Match <DependencyHealth>(_ => _.HealthOk == expectedIsOkFlag &&
_.Message == expectedMessage);
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth datasetsRepoHealth = await((IHealthChecker)_resultsRepository).IsHealthOk();
(bool Ok, string Message)searchRepoHealth = await _searchRepository.IsHealthOk();
ServiceHealth providerSourceDatasetRepoHealth = await((IHealthChecker)_providerSourceDatasetRepository).IsHealthOk();
(bool Ok, string Message)calcSearchRepoHealth = await _calculationProviderResultsSearchRepository.IsHealthOk();
ServiceHealth health = new ServiceHealth()
{
Name = nameof(ResultsService)
};
health.Dependencies.AddRange(datasetsRepoHealth.Dependencies);
health.Dependencies.Add(new DependencyHealth {
HealthOk = searchRepoHealth.Ok, DependencyName = _searchRepository.GetType().GetFriendlyName(), Message = searchRepoHealth.Message
});
health.Dependencies.AddRange(providerSourceDatasetRepoHealth.Dependencies);
health.Dependencies.Add(new DependencyHealth {
HealthOk = calcSearchRepoHealth.Ok, DependencyName = _calculationProviderResultsSearchRepository.GetType().GetFriendlyName(), Message = calcSearchRepoHealth.Message
});
return(health);
}
/// <summary>
/// Serializes the object to JSON.
/// </summary>
/// <param name="writer">The <see cref="T: Newtonsoft.Json.JsonWriter" /> to write to.</param>
/// <param name="obj">The object to serialize to JSON.</param>
internal static void Serialize(JsonWriter writer, ServiceHealth obj)
{
// Required properties are always serialized, optional properties are serialized when not null.
writer.WriteStartObject();
writer.WriteProperty(obj.AggregatedHealthState, "AggregatedHealthState", HealthStateConverter.Serialize);
if (obj.HealthEvents != null)
{
writer.WriteEnumerableProperty(obj.HealthEvents, "HealthEvents", HealthEventConverter.Serialize);
}
if (obj.UnhealthyEvaluations != null)
{
writer.WriteEnumerableProperty(obj.UnhealthyEvaluations, "UnhealthyEvaluations", HealthEvaluationWrapperConverter.Serialize);
}
if (obj.HealthStatistics != null)
{
writer.WriteProperty(obj.HealthStatistics, "HealthStatistics", HealthStatisticsConverter.Serialize);
}
if (obj.Name != null)
{
writer.WriteProperty(obj.Name, "Name", ServiceNameConverter.Serialize);
}
if (obj.PartitionHealthStates != null)
{
writer.WriteEnumerableProperty(obj.PartitionHealthStates, "PartitionHealthStates", PartitionHealthStateConverter.Serialize);
}
writer.WriteEndObject();
}
public async Task HealthCheckCollectsStatusFromRepository()
{
DependencyHealth firstExpectedDependency = new DependencyHealth();
DependencyHealth secondExpectedDependency = new DependencyHealth();
DependencyHealth thirdExpectedDependency = new DependencyHealth();
GivenTheRepositoryServiceHealth(firstExpectedDependency,
secondExpectedDependency,
thirdExpectedDependency);
ServiceHealth isHealthOk = await _service.IsHealthOk();
isHealthOk
.Should()
.NotBeNull();
isHealthOk
.Name
.Should()
.Be(nameof(ProviderFundingPublishingService));
isHealthOk
.Dependencies
.Should()
.BeEquivalentTo(firstExpectedDependency, secondExpectedDependency, thirdExpectedDependency);
}
private ServiceHealth TryResolveService(Type serviceType)
{
ServiceHealth health = new ServiceHealth {
Name = serviceType.GetFriendlyName(), HealthOk = false
};
try
{
object service = this.HttpContext.RequestServices.GetService(serviceType);
health.HealthOk = true;
}
catch (DependencyResolutionException drEx)
{
health.HealthOk = false;
if (drEx.InnerException != null)
{
health.Message = drEx.InnerException.Message;
}
else
{
health.Message = drEx.Message;
}
}
return(health);
}
public async Task IsHealthOkDelegatesToCosmos()
{
bool expectedOk = NewRandomFlag();
string expectedMessage = NewRandomString();
GivenTheCosmosHealth(expectedOk, expectedMessage);
ServiceHealth serviceHealth = await WhenTheServiceHealthIsQueried();
serviceHealth.Dependencies.Count
.Should()
.Be(1);
DependencyHealth health = serviceHealth.Dependencies.Single();
health
.Should()
.BeEquivalentTo(new DependencyHealth
{
HealthOk = expectedOk,
Message = expectedMessage
},
opt
=> opt.Excluding(_ => _.DependencyName));
}
public async Task CatchesResolveExceptionsForControllerTypes()
{
Exception exception = new Exception(new RandomString());
GivenResolvingTheTypeThrowsTheException(typeof(ControllerOne), exception);
ServiceHealth serviceHealth = await _healthCheck.IsHealthOk();
serviceHealth
.Should()
.NotBeNull();
serviceHealth
.Dependencies
.Should()
.ContainEquivalentOf(new DependencyHealth
{
Message = $"{exception.Message}{Environment.NewLine}{exception.StackTrace}",
HealthOk = false,
DependencyName = nameof(ControllerOne)
});
serviceHealth
.Dependencies
.Should()
.ContainEquivalentOf(new DependencyHealth
{
HealthOk = true,
DependencyName = nameof(ControllerTwo)
});
}
public async Task IsHealthOk_ReturnsAsExpected(bool blobOk, string blobMessage)
{
//Arrange
IBlobClient blobClient = Substitute.For <IBlobClient>();
blobClient
.IsHealthOk()
.Returns((blobOk, blobMessage));
ProviderFundingVersionService providerFundingVersionService = CreateProviderFundingVersionService(blobClient: blobClient);
ServiceHealth health = await providerFundingVersionService.IsHealthOk();
health.Name
.Should()
.Be(nameof(ProviderFundingVersionService));
health.Dependencies.Count.Should().Be(1);
health
.Dependencies
.Count(x => x.HealthOk == blobOk && x.Message == blobMessage)
.Should()
.Be(1);
}
public async Task <ServiceHealth> IsHealthOk()
{
(bool Ok, string Message)blobHealth = await _blobClient.IsHealthOk();
ServiceHealth datasetsRepoHealth = await((IHealthChecker)_datasetRepository).IsHealthOk();
(bool Ok, string Message)cacheHealth = await _cacheProvider.IsHealthOk();
ServiceHealth providersResultsRepoHealth = await((IHealthChecker)_providersResultsRepository).IsHealthOk();
ServiceHealth datasetsAggregationsRepoHealth = await((IHealthChecker)_datasetsAggregationsRepository).IsHealthOk();
ServiceHealth health = new ServiceHealth()
{
Name = nameof(DatasetService)
};
health.Dependencies.Add(new DependencyHealth {
HealthOk = blobHealth.Ok, DependencyName = _blobClient.GetType().GetFriendlyName(), Message = blobHealth.Message
});
health.Dependencies.AddRange(datasetsRepoHealth.Dependencies);
health.Dependencies.Add(new DependencyHealth {
HealthOk = cacheHealth.Ok, DependencyName = _cacheProvider.GetType().GetFriendlyName(), Message = cacheHealth.Message
});
health.Dependencies.AddRange(providersResultsRepoHealth.Dependencies);
health.Dependencies.AddRange(datasetsAggregationsRepoHealth.Dependencies);
return(health);
}
private void GivenTheRepositoryServiceHealth(params DependencyHealth[] dependencies)
{
ServiceHealth serviceHealth = new ServiceHealth();
serviceHealth.Dependencies.AddRange(dependencies);
_publishedFundingRepository.IsHealthOk().Returns(serviceHealth);
}
private void AssertServiceHealthsEqual(ServiceHealth expected, ServiceHealth found)
{
if (expected == null)
{
Assert.Null(found);
}
else
{
Assert.True(InfrastructureInsightsCommon.ResourceAreSame(expected, found));
}
}
private async Task CatchHealthError(Service service, ServiceHealth serviceHealth, Exception exception)
{
serviceHealth.ResponseMessage = exception.Message;
serviceHealth.HealthState = HealthState.Critical;
serviceHealth.StatusCode = 0;
serviceHealth.CreatedDate = DateTime.UtcNow;
await AddServiceHealthToStore(serviceHealth);
_logger.LogError($"Health state of service '{service.Id}': {serviceHealth.HealthState}. Status code: {serviceHealth.StatusCode}. Message: '{serviceHealth.ResponseMessage}'.");
}
/// <summary>
/// Used to clear the events on Coordinator Service for nodes which are deleted from cluster.
/// </summary>
public async Task ClearOrphanEvents(CancellationToken cancellationToken)
{
try
{
Uri nodeAgentServiceUri = new Uri(NodeAgentServiceName);
ServiceHealth health = await this.fabricClient.HealthManager.GetServiceHealthAsync(nodeAgentServiceUri);
List <HealthEvent> healthEventsToCheck = new List <HealthEvent>();
foreach (var e in health.HealthEvents)
{
if (e.HealthInformation.Property.Contains(WUOperationStatus) || e.HealthInformation.Property.Contains(WUOperationSetting))
{
healthEventsToCheck.Add(e);
}
}
cancellationToken.ThrowIfCancellationRequested();
NodeList nodeList = await this.fabricClient.QueryManager.GetNodeListAsync(null, null, this.DefaultTimeoutForOperation, cancellationToken);
List <string> orphanProperties = new List <string>();
Dictionary <string, bool> propertyDict = new Dictionary <string, bool>();
if (healthEventsToCheck.Count == 2 * nodeList.Count)
{
return;
}
else
{
foreach (var node in nodeList)
{
propertyDict.Add(WUOperationStatus + "-" + node.NodeName, true);
propertyDict.Add(WUOperationSetting + "-" + node.NodeName, true);
}
foreach (var e in healthEventsToCheck)
{
if (!propertyDict.ContainsKey(e.HealthInformation.Property))
{
orphanProperties.Add(e.HealthInformation.Property);
}
}
foreach (var property in orphanProperties)
{
ServiceEventSource.Current.VerboseMessage("Property {0}'s event is removed from CoordinatorService", property);
string description = "This node is no longer part of the cluster.";
HealthManagerHelper.PostNodeHealthReport(fabricClient, nodeAgentServiceUri, property, description, HealthState.Ok, 1);
}
}
}
catch (Exception ex)
{
ServiceEventSource.Current.ErrorMessage("ClearOrphanEvents failed with exception {0}", ex.ToString());
}
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth health = new ServiceHealth
{
Name = nameof(ProviderFundingPublishingService)
};
health.Dependencies.AddRange((await _publishedFundingRepository.IsHealthOk()).Dependencies);
return(health);
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth jobsRepoHealth = await((IHealthChecker)_jobRepository).IsHealthOk();
ServiceHealth health = new ServiceHealth
{
Name = nameof(JobManagementService)
};
health.Dependencies.AddRange(jobsRepoHealth.Dependencies);
return(health);
}
public async Task<ServiceHealth> IsHealthOk()
{
(bool Ok, string Message) cosmosRepoHealth = await _cosmosRepo.IsHealthOk();
ServiceHealth health = new ServiceHealth()
{
Name = nameof(ProviderSourceDatasetRepository)
};
health.Dependencies.Add(new DependencyHealth { HealthOk = cosmosRepoHealth.Ok, DependencyName = _cosmosRepo.GetType().GetFriendlyName(), Message = cosmosRepoHealth.Message });
return health;
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth gherkinHealth = await((IHealthChecker)_gherkinExecutor).IsHealthOk();
ServiceHealth health = new ServiceHealth()
{
Name = nameof(TestEngine)
};
health.Dependencies.AddRange(gherkinHealth.Dependencies);
return(health);
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth health = new ServiceHealth();
var cacheHealth = await _cacheProvider.IsHealthOk();
health.Name = nameof(ScenariosRepository);
health.Dependencies.Add(new DependencyHealth {
HealthOk = cacheHealth.Ok, DependencyName = this.GetType().GetFriendlyName(), Message = cacheHealth.Message
});
return(health);
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth testResultsRepoHealth = await((IHealthChecker)_testResultsRepository).IsHealthOk();
ServiceHealth health = new ServiceHealth()
{
Name = nameof(TestResultsCountsService)
};
health.Dependencies.AddRange(testResultsRepoHealth.Dependencies);
return(health);
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth calcsRepoHealth = await((IHealthChecker)_calculationsRepository).IsHealthOk();
ServiceHealth health = new ServiceHealth
{
Name = nameof(CalculationService)
};
health.Dependencies.AddRange(calcsRepoHealth.Dependencies);
return(health);
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth health = new ServiceHealth
{
Name = nameof(CalculateProfileService)
};
ServiceHealth profilePatternRepositoryHealthStatus = await((IHealthChecker)_profilePatternRepository).IsHealthOk();
health.Dependencies.AddRange(profilePatternRepositoryHealthStatus.Dependencies);
return(health);
}
public Task <ServiceHealth> IsHealthOk()
{
ServiceHealth health = new ServiceHealth();
var(Ok, Message) = _cosmosRepository.IsHealthOk();
health.Name = nameof(DatasetsAggregationsRepository);
health.Dependencies.Add(new DependencyHealth {
HealthOk = Ok, DependencyName = this.GetType().Name, Message = Message
});
return(Task.FromResult(health));
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth health = new ServiceHealth();
var cosmosHealth = await _cosmosRepository.IsHealthOk();
health.Name = this.GetType().Name;
health.Dependencies.Add(new DependencyHealth {
HealthOk = cosmosHealth.Ok, DependencyName = typeof(CosmosRepository).Name, Message = cosmosHealth.Message
});
return(health);
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth policyRepoHealth = await((IHealthChecker)_policyRepository).IsHealthOk();
ServiceHealth health = new ServiceHealth()
{
Name = nameof(FundingDateService)
};
health.Dependencies.AddRange(policyRepoHealth.Dependencies);
return(health);
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth versioningService = await((IHealthChecker)_publishedProviderVersioningService).IsHealthOk();
ServiceHealth health = new ServiceHealth()
{
Name = nameof(PublishedProviderStatusUpdateService)
};
health.Dependencies.AddRange(versioningService.Dependencies);
return(health);
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth health = new ServiceHealth();
(bool Ok, string Message)cosmosHealth = await _cosmosRepository.IsHealthOk();
health.Name = nameof(DataSetsRepository);
health.Dependencies.Add(new DependencyHealth {
HealthOk = cosmosHealth.Ok, DependencyName = this.GetType().Name, Message = cosmosHealth.Message
});
return(health);
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth health = new ServiceHealth();
(bool Ok, string Message) = await _cacheProvider.IsHealthOk();
health.Name = nameof(GherkinExecutor);
health.Dependencies.Add(new DependencyHealth {
HealthOk = Ok, DependencyName = GetType().Name, Message = Message
});
return(health);
}
public Task <ServiceHealth> IsHealthOk()
{
ServiceHealth health = new ServiceHealth();
(bool Ok, string Message) = _cosmosRepository.IsHealthOk();
health.Name = GetType().Name;
health.Dependencies.Add(new DependencyHealth {
HealthOk = Ok, DependencyName = typeof(CosmosRepository).Name, Message = Message
});
return(Task.FromResult(health));
}
public async Task <ServiceHealth> IsHealthOk()
{
ServiceHealth providerVersionServiceHealth = await _providerVersionService.IsHealthOk();
ServiceHealth health = new ServiceHealth
{
Name = nameof(ProviderSnapshotDataLoadService)
};
health.Dependencies.AddRange(providerVersionServiceHealth.Dependencies);
return(health);
}
public ServiceHealth GetServiceHealth(string serviceId)
{
var key = GetKey(serviceId);
ServiceHealth serviceHealth = null;
if (_memoryCache.TryGetValue(key, out serviceHealth))
{
return(serviceHealth);
}
return(new ServiceHealth {
HealthState = HealthState.Unknown
});
}
public HealthCheck(ServiceHealth status = ServiceHealth.Ok, string message = null)
{
HealthResult = status;
Message = message ?? status.ToString();
}