public void testSingleTestOnOpenCircuitAfterTimeWindow()
{
int sleepWindow = 200;
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
}
public void testTripCircuit()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
// this should still allow requests as everything has been successful
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
}
public void testTripCircuitOnTimeoutsAboveThreshold()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// success with high latency
metrics.MarkSuccess(400);
metrics.MarkSuccess(400);
metrics.MarkTimeout(10);
metrics.MarkSuccess(400);
metrics.MarkTimeout(10);
metrics.MarkTimeout(10);
metrics.MarkSuccess(400);
metrics.MarkTimeout(10);
metrics.MarkTimeout(10);
// this should trip the circuit as the error percentage is above the threshold
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
}
public void testCircuitDoesNotTripOnFailuresBelowThreshold()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// success with high latency
metrics.MarkSuccess(400);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkSuccess(400);
metrics.MarkSuccess(40);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkFailure(10);
// this should remain open as the failure threshold is below the percentage limit
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
}
public void testCircuitClosedAfterSuccessAndClearsStatisticalWindow()
{
int statisticalWindow = 200;
int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter()
.WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow)
.WithMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' succeeds so should cause the circuit to be closed
metrics.MarkSuccess(500);
cb.MarkSuccess();
// all requests should be open again
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen());
}
public void testLowVolumeDoesNotTripCircuit()
{
int sleepWindow = 200;
int lowVolume = 5;
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithCircuitBreakerRequestVolumeThreshold(lowVolume);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// even though it has all failed we won't trip the circuit because the volume is low
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
}
public void testBadRequestsDoNotAffectErrorPercentage()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);;
metrics.MarkSuccess(100);
Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage);
metrics.MarkFailure(1000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
metrics.MarkBadRequest(1);
metrics.MarkBadRequest(2);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
metrics.MarkFailure(45);
metrics.MarkFailure(55);
Assert.Equal(75, metrics.GetHealthCounts().ErrorPercentage);
}
public void testTripCircuitOnTimeouts()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// timeouts
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
// everything has been a timeout so we should not allow any requests
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
}
public void testErrorPercentage()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
metrics.MarkSuccess(100);
Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkFailure(1000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
Assert.Equal(25, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkTimeout(5000);
metrics.MarkTimeout(5000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
// latent
clock.Increment(1);
metrics.MarkSuccess(5000);
// 6 success + 1 latent success + 1 failure + 2 timeout = 10 total
// latent success not considered error
// error percentage = 1 failure + 2 timeout / 10
Assert.Equal(30, metrics.GetHealthCounts().ErrorPercentage);
}
public static TestServiceCommand Get(IJellyfishContext ctx, ITestOutputHelper output, string commandName, ExecutionIsolationStrategy strategy,
ExecutionResult executionResult = ExecutionResult.NONE, FallbackResult fallbackResult = FallbackResult.UNIMPLEMENTED,
Action <CommandPropertiesBuilder> setter = null, TestCircuitBreaker circuitBreaker = null, IClock clock = null)
{
var builder = CommandPropertiesTest.GetUnitTestPropertiesSetter();
builder.WithExecutionIsolationStrategy(strategy);
if (setter != null)
{
setter(builder);
}
if (clock == null)
{
clock = new MockedClock();
}
var cmd = new TestServiceCommand(ctx, commandName, builder, clock, circuitBreaker ?? new TestCircuitBreaker(clock), new TestExecutionHook(output));
cmd.Log = msg => output.WriteLine("({0}) - {1}", cmd.Id, msg);
if (executionResult == ExecutionResult.SUCCESS)
{
cmd.Action = () =>
{
cmd.Log("Execution success"); return(Task.FromResult(TestCommandFactory.EXECUTE_VALUE));
};
}
else if (executionResult == ExecutionResult.FAILURE)
{
cmd.Action = () => { cmd.Log("Execution failure");; throw new Exception("Execution Failure for TestCommand"); };
}
else if (executionResult == ExecutionResult.HYSTRIX_FAILURE)
{
cmd.Action = () =>
{
cmd.Log("Execution failure");
throw new Exception("Execution Failure for TestCommand", new Exception("Fallback Failure for TestCommand"));
};
}
else if (executionResult == ExecutionResult.RECOVERABLE_ERROR)
{
cmd.Action = () =>
{
cmd.Log("Execution recoverable error");
throw new Exception("Execution ERROR for TestCommand");
};
}
else if (executionResult == ExecutionResult.UNRECOVERABLE_ERROR)
{
cmd.Action = () =>
{
cmd.Log("Execution unrecoverable error");
throw new StackOverflowException("Unrecoverable Error for TestCommand");
};
}
else if (executionResult == ExecutionResult.BAD_REQUEST)
{
cmd.Action = () =>
{
cmd.Log("Execution bad request");
throw new BadRequestException("Execution BadRequestException for TestCommand");
};
}
if (fallbackResult == FallbackResult.SUCCESS)
{
cmd.Fallback = () =>
{
cmd.Log("Execution fallback success");
return(Task.FromResult(TestCommandFactory.FALLBACK_VALUE));
};
}
else if (fallbackResult == FallbackResult.FAILURE)
{
cmd.Fallback = () => { cmd.Log("Execution fallback error"); throw new Exception("Fallback Failure for TestCommand"); };
}
else if (fallbackResult == FallbackResult.UNIMPLEMENTED)
{
cmd.Fallback = () =>
{
cmd.Log("Execution fallback unimplemented");
throw new NotImplementedException();
};
}
return(cmd);
}
public SuccessfulCacheableCommand(IJellyfishContext ctx, TestCircuitBreaker circuitBreaker, bool cacheEnabled, T value)
: base(ctx, circuitBreaker.Clock, "SuccessfulCacheableCommand", null, metrics: circuitBreaker.Metrics, circuitBreaker: circuitBreaker, properties: CommandPropertiesTest.GetUnitTestPropertiesSetter().WithRequestCacheEnabled(true))
{
this.value = value;
this.cacheEnabled = cacheEnabled;
}
public void testMultipleTimeWindowRetriesBeforeClosingCircuit()
{
int sleepWindow = 200;
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' fails so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep
metrics.MarkFailure(1000);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' fails again so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep
metrics.MarkFailure(1000);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// now it finally succeeds
metrics.MarkSuccess(200);
cb.MarkSuccess();
// all requests should be open again
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen());
}
public TestCircuitBreaker(IClock clock = null)
{
Clock = clock ?? Jellyfish.Commands.Utils.Clock.GetInstance();
this.Metrics = CircuitBreakerTests.getMetrics(CommandPropertiesTest.GetUnitTestPropertiesSetter(), Clock);
forceShortCircuit = false;
}