private void AssertStatefulServiceReplicaHealthReported(ReplicaWrapper replica, HealthState expectedHealthState, Func <string, bool> descriptionCheck = null)
{
// TODO: test helpers don't return the fake partition ID so we can't verify replica.PartitioinId is the correct one. Pending to refactor the fixture helpers.
AssertHealthReported(
expectedHealthState: expectedHealthState,
descriptionCheck: descriptionCheck,
extraChecks: report => (report as StatefulServiceReplicaHealthReport) != null && (report as StatefulServiceReplicaHealthReport).ReplicaId == replica.Id,
because: $"health '{expectedHealthState}' for stateful replica {replica.Id} should be reported");
}
private static bool IsHealthyReplica(ReplicaWrapper replica)
{
// TODO: Should we only consider replicas that Service Fabric reports as healthy (`replica.HealthState != HealthState.Error`)?
// That is precisely what Traefik does, see: https://github.com/containous/traefik-extra-service-fabric/blob/a5c54b8d5409be7aa21b06d55cf186ee4cc25a13/servicefabric.go#L219
// It seems misguided in our case, however, since we have an active health probing model
// that can determine endpoint health more reliably. In particular because Service Fabric "Error" states does not necessarily mean
// that the replica is unavailable, rather only that something in the cluster issued an "Error" report against it.
// Skipping the replica here because we *suspect* it might be unavailable could lead to snowball cascading failures.
return replica.ReplicaStatus == ServiceReplicaStatus.Ready;
}
// Mocking helpers
private ApplicationWrapper CreateApp_1Service_SingletonPartition_1Replica(
string appTypeName,
string serviceTypeName,
out ServiceWrapper service,
out ReplicaWrapper replica,
ServiceKind serviceKind = ServiceKind.Stateless)
{
service = CreateService(appTypeName, serviceTypeName, 1, 1, out var replicas, serviceKind);
replica = replicas[0];
Mock_ServicesResponse(new Uri($"fabric:/{appTypeName}"), service);
return(SFTestHelpers.FakeApp(appTypeName, appTypeName));
}
private void ReportReplicaHealth(
ServiceFabricDiscoveryOptions options,
ServiceWrapper service,
Guid partitionId,
ReplicaWrapper replica,
HealthState state,
string description = null)
{
if (!options.ReportReplicasHealth)
{
return;
}
var healthInformation = new HealthInformation(
sourceId: HealthReportSourceId,
property: HealthReportProperty,
healthState: state)
{
Description = description,
TimeToLive = HealthReportTimeToLive(options),
RemoveWhenExpired = true,
};
HealthReport healthReport;
switch (service.ServiceKind)
{
case ServiceKind.Stateful:
healthReport = new StatefulServiceReplicaHealthReport(
partitionId: partitionId,
replicaId: replica.Id,
healthInformation: healthInformation);
break;
case ServiceKind.Stateless:
healthReport = new StatelessServiceInstanceHealthReport(
partitionId: partitionId,
instanceId: replica.Id,
healthInformation: healthInformation);
break;
default:
Log.ReplicaHealthReportFailedInvalidServiceKind(_logger, state, replica.Id, service.ServiceKind);
return;
}
var sendOptions = new HealthReportSendOptions { Immediate = state != HealthState.Ok }; // Report immediately if unhealthy
try
{
_serviceFabricCaller.ReportHealth(healthReport, sendOptions);
}
catch (Exception ex) // TODO: davidni: not fatal?
{
Log.ReplicaHealthReportFailed(_logger, state, replica.Id, ex);
}
}
private static bool IsReplicaEligible(ReplicaWrapper replica, StatefulReplicaSelectionMode statefulReplicaSelectionMode)
{
if (replica.ServiceKind != ServiceKind.Stateful)
{
// Stateless service replicas are always eligible
return true;
}
return statefulReplicaSelectionMode switch
{
StatefulReplicaSelectionMode.Primary => replica.Role == ReplicaRole.Primary,
StatefulReplicaSelectionMode.ActiveSecondary => replica.Role == ReplicaRole.ActiveSecondary,
_ => true,
};
}
private ApplicationWrapper CreateApp_2StatelessService_SingletonPartition_1Replica(
string appTypeName,
string serviceTypeName1,
string serviceTypeName2,
out ServiceWrapper service1,
out ServiceWrapper service2,
out ReplicaWrapper service1replica,
out ReplicaWrapper service2replica)
{
service1 = CreateService(appTypeName, serviceTypeName1, 1, 1, out var replicas1);
service2 = CreateService(appTypeName, serviceTypeName2, 1, 1, out var replicas2);
service1replica = replicas1[0];
service2replica = replicas2[0];
Mock_ServicesResponse(new Uri($"fabric:/{appTypeName}"), service1, service2);
return(SFTestHelpers.FakeApp(appTypeName, appTypeName));
}
private DestinationConfig BuildDestination(ReplicaWrapper replica, string listenerName, string healthListenerName, PartitionWrapper partition)
{
if (!ServiceEndpointCollection.TryParseEndpointsString(replica.ReplicaAddress, out var serviceEndpointCollection))
{
throw new ConfigException($"Could not parse endpoints for replica {replica.Id}.");
}
// TODO: FabricServiceEndpoint has some other fields we are ignoring here. Decide which ones are relevant and fix this call.
var serviceEndpoint = new FabricServiceEndpoint(
listenerNames: new[] { listenerName },
allowedSchemePredicate: HttpsSchemeSelector,
emptyStringMatchesAnyListener: true);
if (!FabricServiceEndpointSelector.TryGetEndpoint(serviceEndpoint, serviceEndpointCollection, out var endpointUri))
{
throw new ConfigException($"No acceptable endpoints found for replica '{replica.Id}'. Search criteria: listenerName='{listenerName}', emptyStringMatchesAnyListener=true.");
}
// Get service endpoint from the health listener, health listener is optional.
Uri healthEndpointUri = null;
if (!string.IsNullOrEmpty(healthListenerName))
{
var healthEndpoint = new FabricServiceEndpoint(
listenerNames: new[] { healthListenerName },
allowedSchemePredicate: HttpsSchemeSelector,
emptyStringMatchesAnyListener: true);
if (!FabricServiceEndpointSelector.TryGetEndpoint(healthEndpoint, serviceEndpointCollection, out healthEndpointUri))
{
throw new ConfigException($"No acceptable health endpoints found for replica '{replica.Id}'. Search criteria: listenerName='{healthListenerName}', emptyStringMatchesAnyListener=true.");
}
}
return new DestinationConfig
{
Address = endpointUri.AbsoluteUri,
Health = healthEndpointUri?.AbsoluteUri,
Metadata = new Dictionary<string, string>
{
{ "PartitionId", partition.Id.ToString() ?? string.Empty },
{ "NamedPartitionName", partition.Name ?? string.Empty },
{ "ReplicaId", replica.Id.ToString() ?? string.Empty }
}
};
}
/// <summary>
/// Build a <see cref="DestinationConfig" /> from a Service Fabric <see cref="ReplicaWrapper" />.
/// </summary>
/// <remarks>
/// The address JSON of the replica is expected to have exactly one endpoint, and that one will be used.
/// </remarks>
internal static KeyValuePair <string, DestinationConfig> BuildDestinationFromReplicaAndPartition(ReplicaWrapper replica, PartitionWrapper partition, string healthListenerName = null)
{
ServiceEndpointCollection.TryParseEndpointsString(replica.ReplicaAddress, out var endpoints);
endpoints.TryGetFirstEndpointAddress(out var address);
string healthAddressUri = null;
if (healthListenerName != null)
{
endpoints.TryGetEndpointAddress(healthListenerName, out healthAddressUri);
}
var destinationId = $"{partition.Id}/{replica.Id}";
return(KeyValuePair.Create(
destinationId,
new DestinationConfig
{
Address = address,
Health = healthAddressUri,
Metadata = new Dictionary <string, string>
{
{ "PartitionId", partition.Id.ToString() ?? string.Empty },
{ "NamedPartitionName", partition.Name ?? string.Empty },
{ "ReplicaId", replica.Id.ToString() ?? string.Empty }
}
}));
}
/// <summary>
/// Build a <see cref="Destination" /> from a Service Fabric <see cref="ReplicaWrapper" />.
/// </summary>
/// <remarks>
/// The address JSON of the replica is expected to have exactly one endpoint, and that one will be used.
/// </remarks>
internal static KeyValuePair <string, Destination> BuildDestinationFromReplica(ReplicaWrapper replica, string healthListenerName = null)
{
ServiceEndpointCollection.TryParseEndpointsString(replica.ReplicaAddress, out var endpoints);
endpoints.TryGetFirstEndpointAddress(out var address);
string healthAddressUri = null;
if (healthListenerName != null)
{
endpoints.TryGetEndpointAddress(healthListenerName, out healthAddressUri);
}
return(KeyValuePair.Create(
replica.Id.ToString(),
new Destination
{
Address = address,
Health = healthAddressUri,
Metadata = null,
}));
}
