public async void ExecuteAsync_InvalidLabelsForCluster_NoClustersAndBadHealthReported(string keyToOverride, string value)
{
// Setup
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
var labels = SFTestHelpers.DummyLabels("SomeClusterId");
labels[keyToOverride] = value;
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MyService", out var service, out var replica));
Mock_ServiceLabels(application, service, labels);
// Act
var(routes, clusters) = await RunScenarioAsync();
// Assert
clusters.Should().BeEmpty();
routes.Should().BeEmpty();
AssertServiceHealthReported(service, HealthState.Warning, (description) =>
description.Contains(keyToOverride)); // Check that the invalid key is mentioned in the description
_healthReports.Should().HaveCount(1);
}
public async void ExecuteAsync_StatefulService_SkipReplicaWork(string selectionMode, ReplicaRole?replicaRole)
{
// Setup
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
labels["YARP.Backend.ServiceFabric.StatefulReplicaSelectionMode"] = selectionMode;
ApplicationWrapper application;
Mock_AppsResponse(application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MYService", out var service, out var replica, serviceKind: ServiceKind.Stateful));
Mock_ServiceLabels(application, service, labels);
replica.ServiceKind = ServiceKind.Stateful;
replica.Role = replicaRole;
// Act
var(routes, clusters) = await RunScenarioAsync();
// Assert
var expectedClusters = new[]
{
LabelsParser.BuildCluster(_testServiceName, labels),
};
clusters.Should().BeEquivalentTo(expectedClusters);
_healthReports.Should().HaveCount(1);
}
public async void ExecuteAsync_NotHttpsSchemeForStatelessService_NoEndpointsAndBadHealthReported()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterId = "MyService123";
const string ServiceName = "fabric:/MyApp/MyService";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
labels["YARP.Backend.ServiceFabric.ListenerName"] = "ExampleTeamEndpoint";
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MyService", out var service, out var replica, serviceKind: ServiceKind.Stateless));
var nonHttpAddress = $"http://127.0.0.1/{ServiceName}/0";
replica.ReplicaAddress = $"{{'Endpoints': {{'ExampleTeamEndpoint': '{nonHttpAddress}' }} }}".Replace("'", "\"");
Mock_ServiceLabels(application, service, labels);
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
LabelsParser.BuildCluster(_testServiceName, labels, new Dictionary <string, Destination>()),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica, HealthState.Warning, (description) =>
description.StartsWith("Could not build service endpoint") &&
description.Contains("ExampleTeamEndpoint"));
_healthReports.Should().HaveCount(2);
}
public async void ExecuteAsync_ReplicaHealthReportDisabled_ReplicasHealthIsNotReported()
{
// Setup
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = false
};
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MYService", out var service, out var replica));
Mock_ServiceLabels(application, service, labels);
// Act
var(routes, clusters) = await RunScenarioAsync();
// Assert
var expectedClusters = new[]
{
ClusterWithDestinations(
LabelsParser.BuildCluster(_testServiceName, labels),
SFTestHelpers.BuildDestinationFromReplica(replica)),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service, HealthState.Ok);
_healthReports.Should().HaveCount(1);
}
public async void ExecuteAsync_InvalidListenerNameForStatelessService_NoEndpointsAndBadHealthReported()
{
// Setup
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
labels["YARP.Backend.ServiceFabric.ListenerName"] = "UnexistingListener";
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MyService", out var service, out var replica, serviceKind: ServiceKind.Stateless));
Mock_ServiceLabels(application, service, labels);
// Act
var(routes, clusters) = await RunScenarioAsync();
// Assert
var expectedClusters = new[]
{
LabelsParser.BuildCluster(_testServiceName, labels),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica, HealthState.Warning, (description) =>
description.StartsWith("Could not build service endpoint") &&
description.Contains("UnexistingListener"));
_healthReports.Should().HaveCount(2);
}
public async void ExecuteAsync_InvalidRouteOrder_NoRoutesAndBadHealthReported()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
var labels = new Dictionary <string, string>()
{
{ "YARP.Enable", "true" },
{ "YARP.Backend.BackendId", "SomeClusterId" },
{ "YARP.Routes.MyRoute.Hosts", "example.com" },
{ "YARP.Routes.MyRoute.Order", "not a number" },
};
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MyService", out var service, out var replica));
Mock_ServiceLabels(application, service, labels);
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
ClusterWithDestinations(_testServiceName, labels,
SFTestHelpers.BuildDestinationFromReplica(replica)),
};
var expectedRoutes = new List <ProxyRoute>();
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEmpty();
AssertServiceHealthReported(service, HealthState.Warning, (description) =>
description.Contains("Order")); // Check that the invalid key is mentioned in the description
_healthReports.Should().HaveCount(2);
}
public async void ExecuteAsync_OneServiceWithGatewayEnabledAndOneNotEnabled_OnlyTheOneEnabledFound()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterIdApp1Sv1 = "MyService123";
const string TestClusterIdApp2Sv2 = "MyService234";
var gatewayEnabledLabels = SFTestHelpers.DummyLabels(TestClusterIdApp1Sv1);
var gatewayNotEnabledLabels = SFTestHelpers.DummyLabels(TestClusterIdApp2Sv2, false);
ApplicationWrapper application1, application2;
Mock_AppsResponse(
application1 = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MyService1", out var service1, out var replica1),
application2 = CreateApp_1Service_SingletonPartition_1Replica("MyApp2", "MyService2", out var service2, out var replica2));
Mock_ServiceLabels(application1, service1, gatewayEnabledLabels);
Mock_ServiceLabels(application2, service2, gatewayNotEnabledLabels);
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
ClusterWithDestinations(_testServiceName, gatewayEnabledLabels,
SFTestHelpers.BuildDestinationFromReplica(replica1)),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, gatewayEnabledLabels);
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service1, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica1, HealthState.Ok);
_healthReports.Should().HaveCount(2);
}
private void ReportServiceHealth(
ServiceFabricDiscoveryOptions options,
Uri serviceName,
HealthState state,
string description = null)
{
// TODO: this method and the one below have repeated code. Refactor out.
var healthReport = new ServiceHealthReport(
serviceName: serviceName,
healthInformation: new HealthInformation(
sourceId: HealthReportSourceId,
property: HealthReportProperty,
healthState: state)
{
Description = description,
TimeToLive = HealthReportTimeToLive(options),
RemoveWhenExpired = true,
});
var sendOptions = new HealthReportSendOptions
{
// Report immediately if unhealthy or if explicitly requested
Immediate = options.AlwaysSendImmediateHealthReports ? true : state != HealthState.Ok
};
try
{
_serviceFabricCaller.ReportHealth(healthReport, sendOptions);
}
catch (Exception ex) // TODO: davidni: not fatal?
{
Log.ServiceHealthReportFailed(_logger, state, serviceName, ex);
}
}
public async void ExecuteAsync_SingleServiceWithGatewayEnabledAndActiveHealthCheck()
{
// Setup
_scenarioOptions = new ServiceFabricDiscoveryOptions();
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId, activeHealthChecks: true);
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1StatelessService_2Partition_2ReplicasEach("MyApp", "MYService", out var service, out var replicas));
Mock_ServiceLabels(application, service, labels);
// Act
var(routes, clusters) = await RunScenarioAsync();
// Assert
var expectedClusters = new[]
{
ClusterWithDestinations(
LabelsParser.BuildCluster(_testServiceName, labels),
SFTestHelpers.BuildDestinationFromReplica(replicas[0]),
SFTestHelpers.BuildDestinationFromReplica(replicas[1]),
SFTestHelpers.BuildDestinationFromReplica(replicas[2]),
SFTestHelpers.BuildDestinationFromReplica(replicas[3])),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
routes.Should().BeEquivalentTo(expectedRoutes);
clusters.Should().BeEquivalentTo(expectedClusters);
AssertServiceHealthReported(service, HealthState.Ok);
_healthReports.Should().HaveCount(1);
}
public async void ExecuteAsync_ValidListenerNameForStatelessService_Work()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
labels["YARP.Backend.ServiceFabric.ListenerName"] = "ExampleTeamEndpoint";
labels["YARP.Backend.HealthCheck.Active.ServiceFabric.ListenerName"] = "ExampleTeamHealthEndpoint";
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MyService", out var service, out var replica, serviceKind: ServiceKind.Stateless));
replica.ReplicaAddress = MockReplicaAdressWithListenerName("MyApp", "MyService", new string[] { "ExampleTeamEndpoint", "ExampleTeamHealthEndpoint" });
Mock_ServiceLabels(application, service, labels);
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
ClusterWithDestinations(_testServiceName, labels,
SFTestHelpers.BuildDestinationFromReplica(replica, "ExampleTeamHealthEndpoint")),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica, HealthState.Ok, (description) =>
description.StartsWith("Successfully built"));
_healthReports.Should().HaveCount(2);
}
public async void ExecuteAsync_MultipleServicesWithGatewayEnabled_MultipleClustersFound()
{
// Setup
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterIdApp1Sv1 = "MyService123";
const string TestClusterIdApp1Sv2 = "MyService234";
const string TestClusterIdApp2Sv3 = "MyService345";
var labels1 = SFTestHelpers.DummyLabels(TestClusterIdApp1Sv1);
var labels2 = SFTestHelpers.DummyLabels(TestClusterIdApp1Sv2);
var labels3 = SFTestHelpers.DummyLabels(TestClusterIdApp2Sv3);
ApplicationWrapper application1, application2;
Mock_AppsResponse(
application1 = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MYService", out var service1, out var replica1),
application2 = CreateApp_2StatelessService_SingletonPartition_1Replica("MyApp2", "MyService2", "MyService3", out var service2, out var service3, out var replica2, out var replica3));
Mock_ServiceLabels(application1, service1, labels1);
Mock_ServiceLabels(application2, service2, labels2);
Mock_ServiceLabels(application2, service3, labels3);
// Act
var(routes, clusters) = await RunScenarioAsync();
// Assert
var expectedClusters = new[]
{
ClusterWithDestinations(
LabelsParser.BuildCluster(_testServiceName, labels1),
SFTestHelpers.BuildDestinationFromReplica(replica1)),
ClusterWithDestinations(
LabelsParser.BuildCluster(_testServiceName, labels2),
SFTestHelpers.BuildDestinationFromReplica(replica2)),
ClusterWithDestinations(
LabelsParser.BuildCluster(_testServiceName, labels3),
SFTestHelpers.BuildDestinationFromReplica(replica3)),
};
var expectedRoutes = new List <ProxyRoute>();
expectedRoutes.AddRange(LabelsParser.BuildRoutes(_testServiceName, labels1));
expectedRoutes.AddRange(LabelsParser.BuildRoutes(_testServiceName, labels2));
expectedRoutes.AddRange(LabelsParser.BuildRoutes(_testServiceName, labels3));
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service1, HealthState.Ok);
AssertServiceHealthReported(service2, HealthState.Ok);
AssertServiceHealthReported(service3, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica1, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica2, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica3, HealthState.Ok);
_healthReports.Should().HaveCount(6);
}
private void ReportReplicaHealth(
ServiceFabricDiscoveryOptions options,
ServiceWrapper service,
Guid partitionId,
ReplicaWrapper replica,
HealthState state,
string description = null)
{
if (!options.ReportReplicasHealth)
{
return;
}
var healthInformation = new HealthInformation(
sourceId: HealthReportSourceId,
property: HealthReportProperty,
healthState: state)
{
Description = description,
TimeToLive = HealthReportTimeToLive(options),
RemoveWhenExpired = true,
};
HealthReport healthReport;
switch (service.ServiceKind)
{
case ServiceKind.Stateful:
healthReport = new StatefulServiceReplicaHealthReport(
partitionId: partitionId,
replicaId: replica.Id,
healthInformation: healthInformation);
break;
case ServiceKind.Stateless:
healthReport = new StatelessServiceInstanceHealthReport(
partitionId: partitionId,
instanceId: replica.Id,
healthInformation: healthInformation);
break;
default:
Log.ReplicaHealthReportFailedInvalidServiceKind(_logger, state, replica.Id, service.ServiceKind);
return;
}
var sendOptions = new HealthReportSendOptions { Immediate = state != HealthState.Ok }; // Report immediately if unhealthy
try
{
_serviceFabricCaller.ReportHealth(healthReport, sendOptions);
}
catch (Exception ex) // TODO: davidni: not fatal?
{
Log.ReplicaHealthReportFailed(_logger, state, replica.Id, ex);
}
}
public async void ExecuteAsync_SomeUnhealthyReplicas_OnlyHealthyReplicasAreUsed()
{
// Setup
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1StatelessService_2Partition_2ReplicasEach(
"MyApp",
"MYService",
out var service,
out var replicas));
Mock_ServiceLabels(application, service, labels);
replicas[0].ReplicaStatus = ServiceReplicaStatus.Ready; // Should be used despite Warning health state
replicas[0].HealthState = HealthState.Warning;
replicas[1].ReplicaStatus = ServiceReplicaStatus.Ready; // Should be used
replicas[1].HealthState = HealthState.Ok;
replicas[2].ReplicaStatus = ServiceReplicaStatus.Ready; // Should be used despite Error health state
replicas[2].HealthState = HealthState.Error;
replicas[3].ReplicaStatus = ServiceReplicaStatus.Down; // Should be skipped because of status
replicas[3].HealthState = HealthState.Ok;
// Act
var(routes, clusters) = await RunScenarioAsync();
// Assert
var expectedClusters = new[]
{
ClusterWithDestinations(
LabelsParser.BuildCluster(_testServiceName, labels),
SFTestHelpers.BuildDestinationFromReplica(replicas[0]),
SFTestHelpers.BuildDestinationFromReplica(replicas[1]),
SFTestHelpers.BuildDestinationFromReplica(replicas[2])),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replicas[0], HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replicas[1], HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replicas[2], HealthState.Ok);
_healthReports.Should().HaveCount(4); // 1 service + 3 replicas = 4 health reports
}
public async void ExecuteAsync_NoAppsDiscovered_NoClusters()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
Mock_AppsResponse();
var(routes, clusters) = await RunScenarioAsync();
routes.Should().BeEmpty();
clusters.Should().BeEmpty();
_healthReports.Should().BeEmpty();
}
public async void ExecuteAsync_NoExtensions_NoClusters()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
ApplicationWrapper application;
Mock_AppsResponse(application = CreateApp_1Service_SingletonPartition_1Replica("MyCoolApp", "MyAwesomeService", out var service, out _));
Mock_ServiceLabels(application, service, new Dictionary <string, string>());
var(routes, clusters) = await RunScenarioAsync();
routes.Should().BeEmpty();
clusters.Should().BeEmpty();
_healthReports.Should().BeEmpty();
}
public async void ExecuteAsync_GetLabelsFails_NoClustersAndBadHealthReported()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MYService", out var service, out var replica1));
Mock_ServiceLabelsException(application, service, new ConfigException("foo"));
var(routes, clusters) = await RunScenarioAsync();
clusters.Should().BeEmpty();
routes.Should().BeEmpty();
AssertServiceHealthReported(service, HealthState.Warning, (description) => description.Contains("foo"));
_healthReports.Should().HaveCount(1);
}
public async void ExecuteAsync_SingleServiceWithGatewayEnabled_OneClusterFound()
{
// Setup
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
ApplicationWrapper application, anotherApplication;
Mock_AppsResponse(
application = CreateApp_1StatelessService_2Partition_2ReplicasEach("MyApp", "MYService", out var service, out var replicas),
anotherApplication = CreateApp_1StatelessService_2Partition_2ReplicasEach("AnotherApp", "AnotherService", out var anotherService, out var otherReplicas));
Mock_ServiceLabels(application, service, labels);
Mock_ServiceLabels(anotherApplication, anotherService, new Dictionary <string, string>());
// Act
var(routes, clusters) = await RunScenarioAsync();
// Assert
var expectedClusters = new[]
{
ClusterWithDestinations(
LabelsParser.BuildCluster(_testServiceName, labels),
SFTestHelpers.BuildDestinationFromReplica(replicas[0]),
SFTestHelpers.BuildDestinationFromReplica(replicas[1]),
SFTestHelpers.BuildDestinationFromReplica(replicas[2]),
SFTestHelpers.BuildDestinationFromReplica(replicas[3])),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
routes.Should().BeEquivalentTo(expectedRoutes);
clusters.Should().BeEquivalentTo(expectedClusters);
AssertServiceHealthReported(service, HealthState.Ok);
foreach (var replica in replicas)
{
AssertStatelessServiceInstanceHealthReported(replica, HealthState.Ok);
}
_healthReports.Should().HaveCount(5);
}
public async void ExecuteAsync_GatewayNotEnabled_NoClusters()
{
// Setup
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
ApplicationWrapper application;
Mock_AppsResponse(application = CreateApp_1Service_SingletonPartition_1Replica("MyCoolApp", "MyAwesomeService", out var service, out _));
Mock_ServiceLabels(application, service, new Dictionary <string, string>()
{
{ "YARP.Enable", "false" }
});
// Act
var(routes, clusters) = await RunScenarioAsync();
// Assert
routes.Should().BeEmpty();
clusters.Should().BeEmpty();
_healthReports.Should().BeEmpty();
}
private static TimeSpan HealthReportTimeToLive(ServiceFabricDiscoveryOptions options) => options.DiscoveryPeriod.Multiply(3);
/// <summary>
/// Finds all eligible destinations (replica endpoints) for the <paramref name="service"/> specified,
/// and returns the <see cref="ClusterConfig.Destinations"/> accordingly.
/// </summary>
/// <remarks>All non-fatal exceptions are caught and logged.</remarks>
private async Task<IReadOnlyDictionary<string, DestinationConfig>> DiscoverDestinationsAsync(
ServiceFabricDiscoveryOptions options,
ServiceWrapper service,
Dictionary<string, string> serviceExtensionLabels,
CancellationToken cancellation)
{
IEnumerable<PartitionWrapper> partitions;
try
{
partitions = await _serviceFabricCaller.GetPartitionListAsync(service.ServiceName, cancellation);
}
catch (Exception ex) // TODO: davidni: not fatal?
{
Log.GettingPartitionFailed(_logger, service.ServiceName, ex);
return null;
}
var destinations = new Dictionary<string, DestinationConfig>(StringComparer.OrdinalIgnoreCase);
var listenerName = serviceExtensionLabels.GetValueOrDefault("YARP.Backend.ServiceFabric.ListenerName", string.Empty);
var healthListenerName = serviceExtensionLabels.GetValueOrDefault("YARP.Backend.HealthCheck.Active.ServiceFabric.ListenerName", string.Empty);
var statefulReplicaSelectionMode = ParseStatefulReplicaSelectionMode(serviceExtensionLabels, service.ServiceName);
foreach (var partition in partitions)
{
IEnumerable<ReplicaWrapper> replicas;
try
{
replicas = await _serviceFabricCaller.GetReplicaListAsync(partition.Id, cancellation);
}
catch (Exception ex) // TODO: davidni: not fatal?
{
Log.GettingReplicaFailed(_logger, partition.Id, service.ServiceName, ex);
continue;
}
foreach (var replica in replicas)
{
if (!IsHealthyReplica(replica))
{
Log.UnhealthyReplicaSkipped(_logger, replica.Id, partition.Id, service.ServiceName, replica.ReplicaStatus, replica.HealthState);
continue;
}
// If service is stateful, we need to determine which replica should we route to (e.g Primary, Secondary, All).
if (!IsReplicaEligible(replica, statefulReplicaSelectionMode))
{
// Skip this endpoint.
Log.IneligibleEndpointSkipped(_logger, replica.Id, service.ServiceName, statefulReplicaSelectionMode);
continue;
}
try
{
var destination = BuildDestination(replica, listenerName, healthListenerName, partition);
ReportReplicaHealth(options, service, partition.Id, replica, HealthState.Ok, $"Successfully built the endpoint from listener '{listenerName}'.");
// DestinationId is the concatenation of partitionId and replicaId.
var destinationId = $"{partition.Id}/{replica.Id}";
if (!destinations.TryAdd(destinationId, destination))
{
throw new ConfigException($"Duplicated endpoint id '{replica.Id}'. Skipping repeated definition for service '{service.ServiceName}'.");
}
}
catch (ConfigException ex)
{
// The user's problem
Log.InvalidReplicaConfig(_logger, replica.Id, service.ServiceName, ex);
// TODO: emit Error health report once we are able to detect config issues *during* (as opposed to *after*) a target service upgrade.
// Proactive Error health report would trigger a rollback of the target service as desired. However, an Error report after rhe fact
// will NOT cause a rollback and will prevent the target service from performing subsequent monitored upgrades to mitigate, making things worse.
ReportReplicaHealth(options, service, partition.Id, replica, HealthState.Warning, $"Could not build service endpoint: {ex.Message}");
}
catch (Exception ex) // TODO: davidni: not fatal?
{
// Not the user's problem
Log.ErrorLoadingReplicaConfig(_logger, replica.Id, service.ServiceName, ex);
}
}
}
return destinations;
}
