public virtual void testSingleTestOnOpenCircuitAfterTimeWindow()
{
int sleepWindow = 200;
var properties = GetCommandConfig();
properties.CircuitBreakerSleepWindowInMilliseconds = sleepWindow;
ICommandMetrics metrics = GetMetrics(properties);
var cb = new TestCircuitBreaker(properties, metrics);
// fail
metrics.MarkFailure();
metrics.MarkFailure();
metrics.MarkFailure();
metrics.MarkFailure();
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
}
public virtual void TestTripCircuit()
{
var properties = GetCommandConfig();
ICommandMetrics metrics = GetMetrics(properties);
var cb = new TestCircuitBreaker(properties, metrics);
metrics.MarkSuccess();
metrics.MarkSuccess();
metrics.MarkSuccess();
// this should still allow requests as everything has been successful
Thread.Sleep(properties.MetricsHealthSnapshotIntervalInMilliseconds);
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
// fail
metrics.MarkFailure();
metrics.MarkFailure();
metrics.MarkFailure();;
Thread.Sleep(properties.MetricsHealthSnapshotIntervalInMilliseconds);
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());;
}
public void TestTripCircuitOnTimeouts()
{
var properties = GetCommandConfig();
ICommandMetrics metrics = GetMetrics(properties);
var cb = new TestCircuitBreaker(properties, metrics);
// this should start as allowing requests
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
metrics.MarkTimeout();
metrics.MarkTimeout();
metrics.MarkTimeout();
metrics.MarkTimeout();
Thread.Sleep(properties.MetricsHealthSnapshotIntervalInMilliseconds);
// this should remain open as the failure threshold is below the percentage limit
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
}
public void TestCircuitDoesNotTripOnFailuresBelowThreshold()
{
var properties = GetCommandConfig();
ICommandMetrics metrics = GetMetrics(properties);
var cb = new TestCircuitBreaker(properties, metrics);
// this should start as allowing requests
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
metrics.MarkSuccess();
metrics.MarkSuccess();
metrics.MarkSuccess();
metrics.MarkSuccess();
metrics.MarkFailure();
metrics.MarkFailure();
// this should remain open as the failure threshold is below the percentage limit
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
}
public void TestTripCircuitOnFailuresAboveThreshold()
{
var properties = GetCommandConfig();
ICommandMetrics metrics = GetMetrics(properties);
var cb = new TestCircuitBreaker(properties, metrics);
// this should start as allowing requests
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
// success with high latency
metrics.MarkSuccess();
metrics.MarkFailure();
metrics.MarkSuccess();
metrics.MarkFailure();
metrics.MarkFailure();
metrics.MarkFailure();
Thread.Sleep(properties.MetricsHealthSnapshotIntervalInMilliseconds);
// this should trip the circuit as the error percentage is above the threshold
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
}
public void testLowVolumeDoesNotTripCircuit()
{
int sleepWindow = 200;
int lowVolume = 5;
var properties = GetCommandConfig();
properties.CircuitBreakerSleepWindowInMilliseconds = sleepWindow;
properties.CircuitBreakerRequestCountThreshold = lowVolume;
ICommandMetrics metrics = GetMetrics(properties);
var cb = new TestCircuitBreaker(properties, metrics);
// fail
metrics.MarkFailure();
metrics.MarkFailure();
metrics.MarkFailure();
metrics.MarkFailure();
// even though it has all failed we won't trip the circuit because the volume is low
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
}
public virtual void testCircuitClosedAfterSuccessAndClearsStatisticalWindow()
{
int statisticalWindow = 200;
int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow
var properties = GetCommandConfig();
properties.CircuitBreakerSleepWindowInMilliseconds = sleepWindow;
properties.MetricsRollingStatisticalWindowInMilliseconds = statisticalWindow;
ICommandMetrics metrics = GetMetrics(properties);
var cb = new TestCircuitBreaker(properties, metrics);
// fail
metrics.MarkFailure();
metrics.MarkFailure();
metrics.MarkFailure();
metrics.MarkFailure();
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// the 'singleTest' succeeds so should cause the circuit to be closed
metrics.MarkSuccess();
cb.MarkSuccess();
// all requests should be open again
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
// and the circuit should be closed again
Assert.IsFalse(cb.IsOpen());
}
public void TestMultipleTimeWindowRetriesBeforeClosingCircuit()
{
int sleepWindow = 200;
var properties = GetCommandConfig();
properties.CircuitBreakerSleepWindowInMilliseconds = sleepWindow;
ICommandMetrics metrics = GetMetrics(properties);
var cb = new TestCircuitBreaker(properties, metrics);
// fail
metrics.MarkFailure();
metrics.MarkFailure();
metrics.MarkFailure();
metrics.MarkFailure();
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// the 'singleTest' fails so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep
metrics.MarkFailure();
Assert.IsFalse(cb.AllowRequest());
Assert.IsFalse(cb.AllowRequest());
Assert.IsFalse(cb.AllowRequest());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// the 'singleTest' fails again so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep
metrics.MarkFailure();
Assert.IsFalse(cb.AllowRequest());
Assert.IsFalse(cb.AllowRequest());
Assert.IsFalse(cb.AllowRequest());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// now it finally succeeds
metrics.MarkSuccess();
cb.MarkSuccess();
// all requests should be open again
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
// and the circuit should be closed again
Assert.IsFalse(cb.IsOpen());
}