public void RequestProxied_FailureRateLimitExceeded_MarkDestinationUnhealthy()
{
var options = Options.Create(
new TransportFailureRateHealthPolicyOptions {
DefaultFailureRateLimit = 0.5, DetectionWindowSize = TimeSpan.FromSeconds(30), MinimalTotalCountThreshold = 1
});
var clock = new ManualClock(TimeSpan.FromMilliseconds(10000));
var healthUpdater = new Mock <IDestinationHealthUpdater>();
var policy = new TransportFailureRateHealthPolicy(options, clock, healthUpdater.Object);
Assert.Equal(HealthCheckConstants.PassivePolicy.TransportFailureRate, policy.Name);
var reactivationPeriod0 = TimeSpan.FromSeconds(60);
var reactivationPeriod1 = TimeSpan.FromSeconds(100);
var cluster0 = GetClusterInfo("cluster0", destinationCount: 2);
var cluster1 = GetClusterInfo("cluster1", destinationCount: 2, failureRateLimit: 0.61, reactivationPeriod1);
// Initial state
Assert.All(cluster0.Destinations.Values, d => Assert.Equal(DestinationHealth.Unknown, d.Health.Passive));
Assert.All(cluster1.Destinations.Values, d => Assert.Equal(DestinationHealth.Unknown, d.Health.Passive));
// Successful requests
for (var i = 0; i < 3; i++)
{
policy.RequestProxied(new DefaultHttpContext(), cluster0, cluster0.Destinations.Values.First());
policy.RequestProxied(new DefaultHttpContext(), cluster0, cluster0.Destinations.Values.Skip(1).First());
policy.RequestProxied(new DefaultHttpContext(), cluster1, cluster1.Destinations.Values.First());
policy.RequestProxied(new DefaultHttpContext(), cluster1, cluster1.Destinations.Values.Skip(1).First());
clock.AdvanceClockBy(TimeSpan.FromMilliseconds(4000));
}
healthUpdater.Verify(u => u.SetPassive(cluster0, cluster0.Destinations.Values.First(), DestinationHealth.Healthy, reactivationPeriod0), Times.Exactly(3));
healthUpdater.Verify(u => u.SetPassive(cluster0, cluster0.Destinations.Values.Skip(1).First(), DestinationHealth.Healthy, reactivationPeriod0), Times.Exactly(3));
healthUpdater.Verify(u => u.SetPassive(cluster1, cluster1.Destinations.Values.First(), DestinationHealth.Healthy, reactivationPeriod1), Times.Exactly(3));
healthUpdater.Verify(u => u.SetPassive(cluster1, cluster1.Destinations.Values.Skip(1).First(), DestinationHealth.Healthy, reactivationPeriod1), Times.Exactly(3));
healthUpdater.VerifyNoOtherCalls();
// Failed requests
for (var i = 0; i < 3; i++)
{
policy.RequestProxied(GetFailedRequestContext(ForwarderError.RequestTimedOut), cluster0, cluster0.Destinations.Values.Skip(1).First());
policy.RequestProxied(GetFailedRequestContext(ForwarderError.Request), cluster1, cluster1.Destinations.Values.First());
clock.AdvanceClockBy(TimeSpan.FromMilliseconds(4000));
}
healthUpdater.Verify(u => u.SetPassive(cluster0, cluster0.Destinations.Values.Skip(1).First(), DestinationHealth.Healthy, reactivationPeriod0), Times.Exactly(5));
healthUpdater.Verify(u => u.SetPassive(cluster0, cluster0.Destinations.Values.Skip(1).First(), DestinationHealth.Unhealthy, reactivationPeriod0), Times.Once);
healthUpdater.Verify(u => u.SetPassive(cluster1, cluster1.Destinations.Values.First(), DestinationHealth.Healthy, reactivationPeriod1), Times.Exactly(6));
healthUpdater.VerifyNoOtherCalls();
// Two more failed requests
policy.RequestProxied(GetFailedRequestContext(ForwarderError.Request), cluster1, cluster1.Destinations.Values.First());
// End of the detection window
clock.AdvanceClockBy(TimeSpan.FromMilliseconds(6000));
policy.RequestProxied(GetFailedRequestContext(ForwarderError.Request), cluster1, cluster1.Destinations.Values.First());
healthUpdater.Verify(u => u.SetPassive(cluster1, cluster1.Destinations.Values.First(), DestinationHealth.Healthy, reactivationPeriod1), Times.Exactly(7));
healthUpdater.Verify(u => u.SetPassive(cluster1, cluster1.Destinations.Values.First(), DestinationHealth.Unhealthy, reactivationPeriod1), Times.Once);
healthUpdater.VerifyNoOtherCalls();
}
public void RequestProxied_FailedAndReactivationLessDetection_UseDetectionPeriodForReactivation()
{
var options = Options.Create(
new TransportFailureRateHealthPolicyOptions {
DefaultFailureRateLimit = 0.5, DetectionWindowSize = TimeSpan.FromSeconds(30), MinimalTotalCountThreshold = 1
});
var clock = new ManualClock(TimeSpan.FromMilliseconds(10000));
var healthUpdater = new Mock <IDestinationHealthUpdater>();
var policy = new TransportFailureRateHealthPolicy(options, clock, healthUpdater.Object);
var cluster = GetClusterInfo("cluster0", destinationCount: 2, reactivationPeriod: TimeSpan.FromSeconds(10));
// Initial failed requests
for (var i = 0; i < 2; i++)
{
policy.RequestProxied(GetFailedRequestContext(ForwarderError.RequestTimedOut), cluster, cluster.Destinations.Values.Skip(1).First());
clock.AdvanceClockBy(TimeSpan.FromMilliseconds(1000));
}
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Unhealthy, TimeSpan.FromSeconds(30)), Times.Exactly(2));
healthUpdater.VerifyNoOtherCalls();
// Simulate a reactivation
clock.AdvanceClockBy(TimeSpan.FromMilliseconds(31000));
cluster.Destinations.Values.Skip(1).First().Health.Passive = DestinationHealth.Unknown;
// One successful request to the reactivated destination
policy.RequestProxied(new DefaultHttpContext(), cluster, cluster.Destinations.Values.Skip(1).First());
clock.AdvanceClockBy(TimeSpan.FromMilliseconds(100));
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Healthy, TimeSpan.FromSeconds(30)), Times.Exactly(1));
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Unhealthy, TimeSpan.FromSeconds(30)), Times.Exactly(2));
healthUpdater.VerifyNoOtherCalls();
}
public override ValueTask <int> ReadAsync(Memory <byte> buffer, CancellationToken cancellationToken = default)
{
if (buffer.Length == 0)
{
return(new ValueTask <int>(0));
}
_clock.AdvanceClockBy(_waitTime);
return(base.ReadAsync(buffer.Slice(0, Math.Min(buffer.Length, MaxBytesPerRead)), cancellationToken));
}
public void RequestProxied_FailureMovedOutOfDetectionWindow_MarkDestinationHealthy()
{
var options = Options.Create(
new TransportFailureRateHealthPolicyOptions {
DefaultFailureRateLimit = 0.5, DetectionWindowSize = TimeSpan.FromSeconds(30), MinimalTotalCountThreshold = 1
});
var clock = new ManualClock(TimeSpan.FromMilliseconds(10000));
var healthUpdater = new Mock <IDestinationHealthUpdater>();
var policy = new TransportFailureRateHealthPolicy(options, clock, healthUpdater.Object);
var cluster = GetClusterInfo("cluster0", destinationCount: 2);
// Initial failed requests
for (var i = 0; i < 2; i++)
{
policy.RequestProxied(cluster, cluster.Destinations.Values.Skip(1).First(), GetFailedRequestContext(ProxyError.RequestTimedOut));
clock.AdvanceClockBy(TimeSpan.FromMilliseconds(1000));
}
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Unhealthy, TimeSpan.FromSeconds(60)), Times.Exactly(2));
healthUpdater.VerifyNoOtherCalls();
// Successful requests
for (var i = 0; i < 4; i++)
{
policy.RequestProxied(cluster, cluster.Destinations.Values.First(), new DefaultHttpContext());
policy.RequestProxied(cluster, cluster.Destinations.Values.Skip(1).First(), new DefaultHttpContext());
clock.AdvanceClockBy(TimeSpan.FromMilliseconds(5000));
}
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.First(), DestinationHealth.Healthy, TimeSpan.FromSeconds(60)), Times.Exactly(4));
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Healthy, TimeSpan.FromSeconds(60)), Times.Exactly(2));
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Unhealthy, TimeSpan.FromSeconds(60)), Times.Exactly(4));
healthUpdater.VerifyNoOtherCalls();
// Failed requests
for (var i = 0; i < 2; i++)
{
policy.RequestProxied(cluster, cluster.Destinations.Values.Skip(1).First(), GetFailedRequestContext(ProxyError.RequestTimedOut));
clock.AdvanceClockBy(TimeSpan.FromMilliseconds(1));
}
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Healthy, TimeSpan.FromSeconds(60)), Times.Exactly(3));
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Unhealthy, TimeSpan.FromSeconds(60)), Times.Exactly(5));
healthUpdater.VerifyNoOtherCalls();
// Shift the detection window to the future
clock.AdvanceClockBy(TimeSpan.FromMilliseconds(10998));
// New failed request, but 2 oldest failures have moved out of the detection window
policy.RequestProxied(cluster, cluster.Destinations.Values.Skip(1).First(), GetFailedRequestContext(ProxyError.RequestTimedOut));
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Healthy, TimeSpan.FromSeconds(60)), Times.Exactly(4));
healthUpdater.VerifyNoOtherCalls();
}
private async Task CallThreeTimesAndAssertAsync <T>(Func <Task <T> > call)
{
var almostExpirationOffset = CachedServiceFabricCaller.CacheExpirationOffset.Subtract(TimeSpan.FromTicks(1));
var firstResult = await call(); // First call is successful
_clock.AdvanceClockBy(almostExpirationOffset);
var secondResult = await call(); // Second call should use last result from cache
_clock.AdvanceClockBy(almostExpirationOffset);
secondResult.Should().BeEquivalentTo(firstResult);
await call.Should().ThrowAsync <Exception>(); // Third call fails
}
public void RequestProxied_MultipleConcurrentRequests_MarkDestinationUnhealthyAndHealthyAgain()
{
var options = Options.Create(
new TransportFailureRateHealthPolicyOptions {
DefaultFailureRateLimit = 0.5, DetectionWindowSize = TimeSpan.FromSeconds(30), MinimalTotalCountThreshold = 1
});
var clock = new ManualClock(TimeSpan.FromMilliseconds(10000));
var healthUpdater = new Mock <IDestinationHealthUpdater>();
var reactivationPeriod = TimeSpan.FromSeconds(40);
var policy = new TransportFailureRateHealthPolicy(options, clock, healthUpdater.Object);
var cluster = GetClusterInfo("cluster0", destinationCount: 2, reactivationPeriod: reactivationPeriod);
// Initial state
Assert.All(cluster.Destinations.Values, d => Assert.Equal(DestinationHealth.Unknown, d.Health.Passive));
// Initial sucessful requests
for (var i = 0; i < 2; i++)
{
policy.RequestProxied(new DefaultHttpContext(), cluster, cluster.Destinations.Values.Skip(1).First());
}
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Healthy, reactivationPeriod), Times.Exactly(2));
healthUpdater.VerifyNoOtherCalls();
// Concurrent failed requests.
// They are 'concurrent' because the clock is not updated.
for (var i = 0; i < 2; i++)
{
policy.RequestProxied(GetFailedRequestContext(ForwarderError.RequestTimedOut), cluster, cluster.Destinations.Values.Skip(1).First());
}
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Healthy, reactivationPeriod), Times.Exactly(3));
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Unhealthy, reactivationPeriod), Times.Once);
healthUpdater.VerifyNoOtherCalls();
// More successful requests
for (var i = 0; i < 2; i++)
{
policy.RequestProxied(new DefaultHttpContext(), cluster, cluster.Destinations.Values.Skip(1).First());
clock.AdvanceClockBy(TimeSpan.FromMilliseconds(100));
}
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Healthy, reactivationPeriod), Times.Exactly(5));
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Unhealthy, reactivationPeriod), Times.Once);
healthUpdater.VerifyNoOtherCalls();
// More failed requests
for (var i = 0; i < 2; i++)
{
policy.RequestProxied(GetFailedRequestContext(ForwarderError.RequestTimedOut), cluster, cluster.Destinations.Values.Skip(1).First());
clock.AdvanceClockBy(TimeSpan.FromMilliseconds(100));
}
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Healthy, reactivationPeriod), Times.Exactly(6));
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.Skip(1).First(), DestinationHealth.Unhealthy, reactivationPeriod), Times.Exactly(2));
healthUpdater.VerifyNoOtherCalls();
policy.RequestProxied(new DefaultHttpContext(), cluster, cluster.Destinations.Values.First());
healthUpdater.Verify(u => u.SetPassive(cluster, cluster.Destinations.Values.First(), DestinationHealth.Healthy, reactivationPeriod), Times.Once);
healthUpdater.VerifyNoOtherCalls();
}