public async void ExecuteAsync_ReplicaHealthReportDisabled_ReplicasHealthIsNotReported()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = false
};
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MYService", out var service, out var replica));
Mock_ServiceLabels(application, service, labels);
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
ClusterWithDestinations(_testServiceName, labels,
SFTestHelpers.BuildDestinationFromReplica(replica)),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service, HealthState.Ok);
_healthReports.Should().HaveCount(1);
}
public async void ExecuteAsync_InvalidRouteOrder_NoRoutesAndBadHealthReported()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
var labels = new Dictionary <string, string>()
{
{ "YARP.Enable", "true" },
{ "YARP.Backend.BackendId", "SomeClusterId" },
{ "YARP.Routes.MyRoute.Hosts", "example.com" },
{ "YARP.Routes.MyRoute.Order", "not a number" },
};
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MyService", out var service, out var replica));
Mock_ServiceLabels(application, service, labels);
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
ClusterWithDestinations(_testServiceName, labels,
SFTestHelpers.BuildDestinationFromReplica(replica)),
};
var expectedRoutes = new List <ProxyRoute>();
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEmpty();
AssertServiceHealthReported(service, HealthState.Warning, (description) =>
description.Contains("Order")); // Check that the invalid key is mentioned in the description
_healthReports.Should().HaveCount(2);
}
public async void ExecuteAsync_InvalidListenerNameForStatefulService_NoEndpointsAndBadHealthReported()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
labels["YARP.Backend.ServiceFabric.ListenerName"] = "UnexistingListener";
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MyService", out var service, out var replica, serviceKind: ServiceKind.Stateful));
Mock_ServiceLabels(application, service, labels);
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
LabelsParser.BuildCluster(_testServiceName, labels, new Dictionary <string, Destination>()),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service, HealthState.Ok);
AssertStatefulServiceReplicaHealthReported(replica, HealthState.Warning, (description) =>
description.StartsWith("Could not build service endpoint") &&
description.Contains("UnexistingListener"));
_healthReports.Should().HaveCount(2);
}
public async void ExecuteAsync_OneServiceWithGatewayEnabledAndOneNotEnabled_OnlyTheOneEnabledFound()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterIdApp1Sv1 = "MyService123";
const string TestClusterIdApp2Sv2 = "MyService234";
var gatewayEnabledLabels = SFTestHelpers.DummyLabels(TestClusterIdApp1Sv1);
var gatewayNotEnabledLabels = SFTestHelpers.DummyLabels(TestClusterIdApp2Sv2, false);
ApplicationWrapper application1, application2;
Mock_AppsResponse(
application1 = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MyService1", out var service1, out var replica1),
application2 = CreateApp_1Service_SingletonPartition_1Replica("MyApp2", "MyService2", out var service2, out var replica2));
Mock_ServiceLabels(application1, service1, gatewayEnabledLabels);
Mock_ServiceLabels(application2, service2, gatewayNotEnabledLabels);
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
ClusterWithDestinations(_testServiceName, gatewayEnabledLabels,
SFTestHelpers.BuildDestinationFromReplica(replica1)),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, gatewayEnabledLabels);
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service1, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica1, HealthState.Ok);
_healthReports.Should().HaveCount(2);
}
public async void ExecuteAsync_StatefulService_SkipReplicaWork(string selectionMode, ReplicaRole?replicaRole)
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
labels["YARP.Backend.ServiceFabric.StatefulReplicaSelectionMode"] = selectionMode;
ApplicationWrapper application;
Mock_AppsResponse(application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MYService", out var service, out var replica, serviceKind: ServiceKind.Stateful));
Mock_ServiceLabels(application, service, labels);
replica.ServiceKind = ServiceKind.Stateful;
replica.Role = replicaRole;
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
LabelsParser.BuildCluster(_testServiceName, labels, new Dictionary <string, Destination>()),
};
clusters.Should().BeEquivalentTo(expectedClusters);
_healthReports.Should().HaveCount(1);
}
public async void ExecuteAsync_SingleServiceWithGatewayEnabledAndActiveHealthCheck()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions();
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId, activeHealthChecks: true);
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1StatelessService_2Partition_2ReplicasEach("MyApp", "MYService", out var service, out var replicas));
Mock_ServiceLabels(application, service, labels);
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
ClusterWithDestinations(_testServiceName, labels,
SFTestHelpers.BuildDestinationFromReplica(replicas[0]),
SFTestHelpers.BuildDestinationFromReplica(replicas[1]),
SFTestHelpers.BuildDestinationFromReplica(replicas[2]),
SFTestHelpers.BuildDestinationFromReplica(replicas[3])),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
routes.Should().BeEquivalentTo(expectedRoutes);
clusters.Should().BeEquivalentTo(expectedClusters);
AssertServiceHealthReported(service, HealthState.Ok);
_healthReports.Should().HaveCount(1);
}
public async void ExecuteAsync_ValidListenerNameForStatelessService_Work()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
labels["YARP.Backend.ServiceFabric.ListenerName"] = "ExampleTeamEndpoint";
labels["YARP.Backend.HealthCheck.Active.ServiceFabric.ListenerName"] = "ExampleTeamHealthEndpoint";
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MyService", out var service, out var replica, serviceKind: ServiceKind.Stateless));
replica.ReplicaAddress = MockReplicaAdressWithListenerName("MyApp", "MyService", new string[] { "ExampleTeamEndpoint", "ExampleTeamHealthEndpoint" });
Mock_ServiceLabels(application, service, labels);
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
ClusterWithDestinations(_testServiceName, labels,
SFTestHelpers.BuildDestinationFromReplica(replica, "ExampleTeamHealthEndpoint")),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica, HealthState.Ok, (description) =>
description.StartsWith("Successfully built"));
_healthReports.Should().HaveCount(2);
}
public async void ExecuteAsync_NotHttpsSchemeForStatelessService_NoEndpointsAndBadHealthReported()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterId = "MyService123";
const string ServiceName = "fabric:/MyApp/MyService";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
labels["YARP.Backend.ServiceFabric.ListenerName"] = "ExampleTeamEndpoint";
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MyService", out var service, out var replica, serviceKind: ServiceKind.Stateless));
var nonHttpAddress = $"http://127.0.0.1/{ServiceName}/0";
replica.ReplicaAddress = $"{{'Endpoints': {{'ExampleTeamEndpoint': '{nonHttpAddress}' }} }}".Replace("'", "\"");
Mock_ServiceLabels(application, service, labels);
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
LabelsParser.BuildCluster(_testServiceName, labels, new Dictionary <string, Destination>()),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica, HealthState.Warning, (description) =>
description.StartsWith("Could not build service endpoint") &&
description.Contains("ExampleTeamEndpoint"));
_healthReports.Should().HaveCount(2);
}
private ApplicationWrapper CreateApp_1StatelessService_2Partition_2ReplicasEach(
string appTypeName,
string serviceTypeName,
out ServiceWrapper service,
out List <ReplicaWrapper> replicas)
{
service = CreateService(appTypeName, serviceTypeName, 2, 2, out replicas);
Mock_ServicesResponse(new Uri($"fabric:/{appTypeName}"), service);
return(SFTestHelpers.FakeApp(appTypeName, appTypeName));
}
// Mocking helpers
private ApplicationWrapper CreateApp_1Service_SingletonPartition_1Replica(
string appTypeName,
string serviceTypeName,
out ServiceWrapper service,
out ReplicaWrapper replica,
ServiceKind serviceKind = ServiceKind.Stateless)
{
service = CreateService(appTypeName, serviceTypeName, 1, 1, out var replicas, serviceKind);
replica = replicas[0];
Mock_ServicesResponse(new Uri($"fabric:/{appTypeName}"), service);
return(SFTestHelpers.FakeApp(appTypeName, appTypeName));
}
public async void ExecuteAsync_SomeUnhealthyReplicas_OnlyHealthyReplicasAreUsed()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1StatelessService_2Partition_2ReplicasEach(
"MyApp",
"MYService",
out var service,
out var replicas,
out var partitions));
Mock_ServiceLabels(application, service, labels);
replicas[0].ReplicaStatus = ServiceReplicaStatus.Ready; // Should be used despite Warning health state
replicas[0].HealthState = HealthState.Warning;
replicas[1].ReplicaStatus = ServiceReplicaStatus.Ready; // Should be used
replicas[1].HealthState = HealthState.Ok;
replicas[2].ReplicaStatus = ServiceReplicaStatus.Ready; // Should be used despite Error health state
replicas[2].HealthState = HealthState.Error;
replicas[3].ReplicaStatus = ServiceReplicaStatus.Down; // Should be skipped because of status
replicas[3].HealthState = HealthState.Ok;
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
ClusterWithDestinations(_testServiceName, labels,
SFTestHelpers.BuildDestinationFromReplicaAndPartition(replicas[0], partitions[0]),
SFTestHelpers.BuildDestinationFromReplicaAndPartition(replicas[1], partitions[0]),
SFTestHelpers.BuildDestinationFromReplicaAndPartition(replicas[2], partitions[1])),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replicas[0], HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replicas[1], HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replicas[2], HealthState.Ok);
_healthReports.Should().HaveCount(4); // 1 service + 3 replicas = 4 health reports
}
public async void ExecuteAsync_MultipleServicesWithGatewayEnabled_MultipleClustersFound()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterIdApp1Sv1 = "MyService123";
const string TestClusterIdApp1Sv2 = "MyService234";
const string TestClusterIdApp2Sv3 = "MyService345";
var labels1 = SFTestHelpers.DummyLabels(TestClusterIdApp1Sv1);
var labels2 = SFTestHelpers.DummyLabels(TestClusterIdApp1Sv2);
var labels3 = SFTestHelpers.DummyLabels(TestClusterIdApp2Sv3);
ApplicationWrapper application1, application2;
Mock_AppsResponse(
application1 = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MYService", out var service1, out var replica1),
application2 = CreateApp_2StatelessService_SingletonPartition_1Replica("MyApp2", "MyService2", "MyService3", out var service2, out var service3, out var replica2, out var replica3));
Mock_ServiceLabels(application1, service1, labels1);
Mock_ServiceLabels(application2, service2, labels2);
Mock_ServiceLabels(application2, service3, labels3);
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
ClusterWithDestinations(_testServiceName, labels1,
SFTestHelpers.BuildDestinationFromReplica(replica1)),
ClusterWithDestinations(_testServiceName, labels2,
SFTestHelpers.BuildDestinationFromReplica(replica2)),
ClusterWithDestinations(_testServiceName, labels3,
SFTestHelpers.BuildDestinationFromReplica(replica3)),
};
var expectedRoutes = new List <ProxyRoute>();
expectedRoutes.AddRange(LabelsParser.BuildRoutes(_testServiceName, labels1));
expectedRoutes.AddRange(LabelsParser.BuildRoutes(_testServiceName, labels2));
expectedRoutes.AddRange(LabelsParser.BuildRoutes(_testServiceName, labels3));
clusters.Should().BeEquivalentTo(expectedClusters);
routes.Should().BeEquivalentTo(expectedRoutes);
AssertServiceHealthReported(service1, HealthState.Ok);
AssertServiceHealthReported(service2, HealthState.Ok);
AssertServiceHealthReported(service3, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica1, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica2, HealthState.Ok);
AssertStatelessServiceInstanceHealthReported(replica3, HealthState.Ok);
_healthReports.Should().HaveCount(6);
}
private ApplicationWrapper CreateApp_2StatelessService_SingletonPartition_1Replica(
string appTypeName,
string serviceTypeName1,
string serviceTypeName2,
out ServiceWrapper service1,
out ServiceWrapper service2,
out ReplicaWrapper service1replica,
out ReplicaWrapper service2replica)
{
service1 = CreateService(appTypeName, serviceTypeName1, 1, 1, out var replicas1);
service2 = CreateService(appTypeName, serviceTypeName2, 1, 1, out var replicas2);
service1replica = replicas1[0];
service2replica = replicas2[0];
Mock_ServicesResponse(new Uri($"fabric:/{appTypeName}"), service1, service2);
return(SFTestHelpers.FakeApp(appTypeName, appTypeName));
}
public async void GetApplicationListAsync_ServiceFabricFails_ResultIsCached()
{
var caller = Create <CachedServiceFabricCaller>();
var originalApps = new List <ApplicationWrapper> {
SFTestHelpers.FakeApp("MyApp")
};
Mock <IQueryClientWrapper>()
.SetupSequence(m => m.GetApplicationListAsync(
It.IsAny <TimeSpan>(),
It.IsAny <CancellationToken>()))
.ReturnsAsync(originalApps)
.ThrowsAsync(new Exception("the cake is a lie"))
.ThrowsAsync(new Exception("the cake is still a lie"));
await CallThreeTimesAndAssertAsync(() => caller.GetApplicationListAsync(CancellationToken.None));
}
public async void GetPartitionListAsync_ServiceFabricFails_ResultIsCached()
{
var caller = Create <CachedServiceFabricCaller>();
var originalPartitionInfoList = new List <PartitionWrapper> {
SFTestHelpers.FakePartition()
};
Mock <IQueryClientWrapper>()
.SetupSequence(m => m.GetPartitionListAsync(
new Uri("http://localhost/app1/sv1"),
It.IsAny <TimeSpan>(),
It.IsAny <CancellationToken>()))
.ReturnsAsync(originalPartitionInfoList)
.ThrowsAsync(new Exception("the cake is a lie"))
.ThrowsAsync(new Exception("the cake is still a lie"));
await CallThreeTimesAndAssertAsync(() => caller.GetPartitionListAsync(new Uri("http://localhost/app1/sv1"), CancellationToken.None));
}
private ServiceWrapper CreateService(string appName, string serviceName, int numPartitions, int numReplicasPerPartition, out List <ReplicaWrapper> replicas, out List <PartitionWrapper> partitions, ServiceKind serviceKind = ServiceKind.Stateless)
{
var svcName = new Uri($"fabric:/{appName}/{serviceName}");
var service = SFTestHelpers.FakeService(svcName, $"{appName}_{serviceName}_Type", serviceKind: serviceKind);
replicas = new List <ReplicaWrapper>();
partitions = new List <PartitionWrapper>();
for (var i = 0; i < numPartitions; i++)
{
var partitionReplicas = Enumerable.Range(i * numReplicasPerPartition, numReplicasPerPartition).Select(replicaId => SFTestHelpers.FakeReplica(svcName, replicaId)).ToList();
replicas.AddRange(partitionReplicas);
var partition = SFTestHelpers.FakePartition();
partitions.Add(partition);
Mock_ReplicasResponse(partition.Id, partitionReplicas.ToArray());
}
Mock_PartitionsResponse(svcName, partitions.ToArray());
return(service);
}
public async void ExecuteAsync_InvalidLabelsForCluster_NoClustersAndBadHealthReported(string keyToOverride, string value)
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
var labels = SFTestHelpers.DummyLabels("SomeClusterId");
labels[keyToOverride] = value;
ApplicationWrapper application;
Mock_AppsResponse(
application = CreateApp_1Service_SingletonPartition_1Replica("MyApp", "MyService", out var service, out var replica));
Mock_ServiceLabels(application, service, labels);
var(routes, clusters) = await RunScenarioAsync();
clusters.Should().BeEmpty();
routes.Should().BeEmpty();
AssertServiceHealthReported(service, HealthState.Warning, (description) =>
description.Contains(keyToOverride)); // Check that the invalid key is mentioned in the description
_healthReports.Where(report => report.HealthInformation.HealthState == HealthState.Warning).Should().HaveCount(1);
}
public async void ExecuteAsync_SingleServiceWithGatewayEnabled_OneClusterFound()
{
_scenarioOptions = new ServiceFabricDiscoveryOptions {
ReportReplicasHealth = true
};
const string TestClusterId = "MyService123";
var labels = SFTestHelpers.DummyLabels(TestClusterId);
ApplicationWrapper application, anotherApplication;
Mock_AppsResponse(
application = CreateApp_1StatelessService_2Partition_2ReplicasEach("MyApp", "MYService", out var service, out var replicas),
anotherApplication = CreateApp_1StatelessService_2Partition_2ReplicasEach("AnotherApp", "AnotherService", out var anotherService, out var otherReplicas));
Mock_ServiceLabels(application, service, labels);
Mock_ServiceLabels(anotherApplication, anotherService, new Dictionary <string, string>());
var(routes, clusters) = await RunScenarioAsync();
var expectedClusters = new[]
{
ClusterWithDestinations(_testServiceName, labels,
SFTestHelpers.BuildDestinationFromReplica(replicas[0]),
SFTestHelpers.BuildDestinationFromReplica(replicas[1]),
SFTestHelpers.BuildDestinationFromReplica(replicas[2]),
SFTestHelpers.BuildDestinationFromReplica(replicas[3])),
};
var expectedRoutes = LabelsParser.BuildRoutes(_testServiceName, labels);
routes.Should().BeEquivalentTo(expectedRoutes);
clusters.Should().BeEquivalentTo(expectedClusters);
AssertServiceHealthReported(service, HealthState.Ok);
foreach (var replica in replicas)
{
AssertStatelessServiceInstanceHealthReported(replica, HealthState.Ok);
}
_healthReports.Should().HaveCount(5);
}