public void testSingleTestOnOpenCircuitAfterTimeWindow() { int sleepWindow = 200; var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.False(cb.AllowRequest); Assert.True(cb.IsOpen()); // wait for sleepWindow to pass clock.Increment(sleepWindow + 50); // we should now allow 1 request Assert.True(cb.AllowRequest); // but the circuit should still be open Assert.True(cb.IsOpen()); // and further requests are still blocked Assert.False(cb.AllowRequest); }
public void testTripCircuit() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); metrics.MarkSuccess(1000); metrics.MarkSuccess(1000); metrics.MarkSuccess(1000); metrics.MarkSuccess(1000); // this should still allow requests as everything has been successful Assert.True(cb.AllowRequest); Assert.False(cb.IsOpen()); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.False(cb.AllowRequest); Assert.True(cb.IsOpen()); }
public void testTripCircuitOnTimeoutsAboveThreshold() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); // this should start as allowing requests Assert.True(cb.AllowRequest); Assert.False(cb.IsOpen()); // success with high latency metrics.MarkSuccess(400); metrics.MarkSuccess(400); metrics.MarkTimeout(10); metrics.MarkSuccess(400); metrics.MarkTimeout(10); metrics.MarkTimeout(10); metrics.MarkSuccess(400); metrics.MarkTimeout(10); metrics.MarkTimeout(10); // this should trip the circuit as the error percentage is above the threshold Assert.False(cb.AllowRequest); Assert.True(cb.IsOpen()); }
public void testCircuitDoesNotTripOnFailuresBelowThreshold() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); // this should start as allowing requests Assert.True(cb.AllowRequest); Assert.False(cb.IsOpen()); // success with high latency metrics.MarkSuccess(400); metrics.MarkSuccess(400); metrics.MarkFailure(10); metrics.MarkSuccess(400); metrics.MarkSuccess(40); metrics.MarkSuccess(400); metrics.MarkFailure(10); metrics.MarkFailure(10); // this should remain open as the failure threshold is below the percentage limit Assert.True(cb.AllowRequest); Assert.False(cb.IsOpen()); }
public void testCircuitClosedAfterSuccessAndClearsStatisticalWindow() { int statisticalWindow = 200; int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter() .WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow) .WithMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.False(cb.AllowRequest); Assert.True(cb.IsOpen()); // wait for sleepWindow to pass clock.Increment(sleepWindow + 50); // we should now allow 1 request Assert.True(cb.AllowRequest); // but the circuit should still be open Assert.True(cb.IsOpen()); // and further requests are still blocked Assert.False(cb.AllowRequest); // the 'singleTest' succeeds so should cause the circuit to be closed metrics.MarkSuccess(500); cb.MarkSuccess(); // all requests should be open again Assert.True(cb.AllowRequest); Assert.True(cb.AllowRequest); Assert.True(cb.AllowRequest); // and the circuit should be closed again Assert.False(cb.IsOpen()); }
public void testLowVolumeDoesNotTripCircuit() { int sleepWindow = 200; int lowVolume = 5; var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithCircuitBreakerRequestVolumeThreshold(lowVolume); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // even though it has all failed we won't trip the circuit because the volume is low Assert.True(cb.AllowRequest); Assert.False(cb.IsOpen()); }
public void testBadRequestsDoNotAffectErrorPercentage() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock);; metrics.MarkSuccess(100); Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage); metrics.MarkFailure(1000); Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage); metrics.MarkBadRequest(1); metrics.MarkBadRequest(2); Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage); metrics.MarkFailure(45); metrics.MarkFailure(55); Assert.Equal(75, metrics.GetHealthCounts().ErrorPercentage); }
public void testTripCircuitOnTimeouts() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); // this should start as allowing requests Assert.True(cb.AllowRequest); Assert.False(cb.IsOpen()); // timeouts metrics.MarkTimeout(2000); metrics.MarkTimeout(2000); metrics.MarkTimeout(2000); metrics.MarkTimeout(2000); // everything has been a timeout so we should not allow any requests Assert.False(cb.AllowRequest); Assert.True(cb.IsOpen()); }
public void testErrorPercentage() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); metrics.MarkSuccess(100); Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage); clock.Increment(1); metrics.MarkFailure(1000); Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage); clock.Increment(1); metrics.MarkSuccess(100); metrics.MarkSuccess(100); Assert.Equal(25, metrics.GetHealthCounts().ErrorPercentage); clock.Increment(1); metrics.MarkTimeout(5000); metrics.MarkTimeout(5000); Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage); clock.Increment(1); metrics.MarkSuccess(100); metrics.MarkSuccess(100); metrics.MarkSuccess(100); // latent clock.Increment(1); metrics.MarkSuccess(5000); // 6 success + 1 latent success + 1 failure + 2 timeout = 10 total // latent success not considered error // error percentage = 1 failure + 2 timeout / 10 Assert.Equal(30, metrics.GetHealthCounts().ErrorPercentage); }
public static TestServiceCommand Get(IJellyfishContext ctx, ITestOutputHelper output, string commandName, ExecutionIsolationStrategy strategy, ExecutionResult executionResult = ExecutionResult.NONE, FallbackResult fallbackResult = FallbackResult.UNIMPLEMENTED, Action <CommandPropertiesBuilder> setter = null, TestCircuitBreaker circuitBreaker = null, IClock clock = null) { var builder = CommandPropertiesTest.GetUnitTestPropertiesSetter(); builder.WithExecutionIsolationStrategy(strategy); if (setter != null) { setter(builder); } if (clock == null) { clock = new MockedClock(); } var cmd = new TestServiceCommand(ctx, commandName, builder, clock, circuitBreaker ?? new TestCircuitBreaker(clock), new TestExecutionHook(output)); cmd.Log = msg => output.WriteLine("({0}) - {1}", cmd.Id, msg); if (executionResult == ExecutionResult.SUCCESS) { cmd.Action = () => { cmd.Log("Execution success"); return(Task.FromResult(TestCommandFactory.EXECUTE_VALUE)); }; } else if (executionResult == ExecutionResult.FAILURE) { cmd.Action = () => { cmd.Log("Execution failure");; throw new Exception("Execution Failure for TestCommand"); }; } else if (executionResult == ExecutionResult.HYSTRIX_FAILURE) { cmd.Action = () => { cmd.Log("Execution failure"); throw new Exception("Execution Failure for TestCommand", new Exception("Fallback Failure for TestCommand")); }; } else if (executionResult == ExecutionResult.RECOVERABLE_ERROR) { cmd.Action = () => { cmd.Log("Execution recoverable error"); throw new Exception("Execution ERROR for TestCommand"); }; } else if (executionResult == ExecutionResult.UNRECOVERABLE_ERROR) { cmd.Action = () => { cmd.Log("Execution unrecoverable error"); throw new StackOverflowException("Unrecoverable Error for TestCommand"); }; } else if (executionResult == ExecutionResult.BAD_REQUEST) { cmd.Action = () => { cmd.Log("Execution bad request"); throw new BadRequestException("Execution BadRequestException for TestCommand"); }; } if (fallbackResult == FallbackResult.SUCCESS) { cmd.Fallback = () => { cmd.Log("Execution fallback success"); return(Task.FromResult(TestCommandFactory.FALLBACK_VALUE)); }; } else if (fallbackResult == FallbackResult.FAILURE) { cmd.Fallback = () => { cmd.Log("Execution fallback error"); throw new Exception("Fallback Failure for TestCommand"); }; } else if (fallbackResult == FallbackResult.UNIMPLEMENTED) { cmd.Fallback = () => { cmd.Log("Execution fallback unimplemented"); throw new NotImplementedException(); }; } return(cmd); }
public SuccessfulCacheableCommand(IJellyfishContext ctx, TestCircuitBreaker circuitBreaker, bool cacheEnabled, T value) : base(ctx, circuitBreaker.Clock, "SuccessfulCacheableCommand", null, metrics: circuitBreaker.Metrics, circuitBreaker: circuitBreaker, properties: CommandPropertiesTest.GetUnitTestPropertiesSetter().WithRequestCacheEnabled(true)) { this.value = value; this.cacheEnabled = cacheEnabled; }
public void testMultipleTimeWindowRetriesBeforeClosingCircuit() { int sleepWindow = 200; var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.False(cb.AllowRequest); Assert.True(cb.IsOpen()); // wait for sleepWindow to pass clock.Increment(sleepWindow + 50); // we should now allow 1 request Assert.True(cb.AllowRequest); // but the circuit should still be open Assert.True(cb.IsOpen()); // and further requests are still blocked Assert.False(cb.AllowRequest); // the 'singleTest' fails so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep metrics.MarkFailure(1000); Assert.False(cb.AllowRequest); Assert.False(cb.AllowRequest); Assert.False(cb.AllowRequest); // wait for sleepWindow to pass clock.Increment(sleepWindow + 50); // we should now allow 1 request Assert.True(cb.AllowRequest); // but the circuit should still be open Assert.True(cb.IsOpen()); // and further requests are still blocked Assert.False(cb.AllowRequest); // the 'singleTest' fails again so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep metrics.MarkFailure(1000); Assert.False(cb.AllowRequest); Assert.False(cb.AllowRequest); Assert.False(cb.AllowRequest); // wait for sleepWindow to pass clock.Increment(sleepWindow + 50); // we should now allow 1 request Assert.True(cb.AllowRequest); // but the circuit should still be open Assert.True(cb.IsOpen()); // and further requests are still blocked Assert.False(cb.AllowRequest); // now it finally succeeds metrics.MarkSuccess(200); cb.MarkSuccess(); // all requests should be open again Assert.True(cb.AllowRequest); Assert.True(cb.AllowRequest); Assert.True(cb.AllowRequest); // and the circuit should be closed again Assert.False(cb.IsOpen()); }
public TestCircuitBreaker(IClock clock = null) { Clock = clock ?? Jellyfish.Commands.Utils.Clock.GetInstance(); this.Metrics = CircuitBreakerTests.getMetrics(CommandPropertiesTest.GetUnitTestPropertiesSetter(), Clock); forceShortCircuit = false; }