public void testTripCircuitOnTimeoutsAboveThreshold() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); // this should start as allowing requests Assert.True(cb.AllowRequest); Assert.False(cb.IsOpen()); // success with high latency metrics.MarkSuccess(400); metrics.MarkSuccess(400); metrics.MarkTimeout(10); metrics.MarkSuccess(400); metrics.MarkTimeout(10); metrics.MarkTimeout(10); metrics.MarkSuccess(400); metrics.MarkTimeout(10); metrics.MarkTimeout(10); // this should trip the circuit as the error percentage is above the threshold Assert.False(cb.AllowRequest); Assert.True(cb.IsOpen()); }
public void testTripCircuit() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); metrics.MarkSuccess(1000); metrics.MarkSuccess(1000); metrics.MarkSuccess(1000); metrics.MarkSuccess(1000); // this should still allow requests as everything has been successful Assert.True(cb.AllowRequest); Assert.False(cb.IsOpen()); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.False(cb.AllowRequest); Assert.True(cb.IsOpen()); }
public void testCircuitDoesNotTripOnFailuresBelowThreshold() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); // this should start as allowing requests Assert.True(cb.AllowRequest); Assert.False(cb.IsOpen()); // success with high latency metrics.MarkSuccess(400); metrics.MarkSuccess(400); metrics.MarkFailure(10); metrics.MarkSuccess(400); metrics.MarkSuccess(40); metrics.MarkSuccess(400); metrics.MarkFailure(10); metrics.MarkFailure(10); // this should remain open as the failure threshold is below the percentage limit Assert.True(cb.AllowRequest); Assert.False(cb.IsOpen()); }
public void testCircuitClosedAfterSuccess() { int sleepWindow = 1000; var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter() .WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkTimeout(1000); // everything has failed in the test window so we should return false now Assert.False(cb.AllowRequest); Assert.True(cb.IsOpen()); // wait for sleepWindow to pass clock.Increment(sleepWindow + 50); // we should now allow 1 request Assert.True(cb.AllowRequest); // but the circuit should still be open Assert.True(cb.IsOpen()); clock.Increment(sleepWindow); // and further requests are still blocked Assert.False(cb.AllowRequest); // the 'singleTest' succeeds so should cause the circuit to be closed metrics.MarkSuccess(500); clock.Increment(sleepWindow); cb.MarkSuccess(); // all requests should be open again Assert.True(cb.AllowRequest); clock.Increment(sleepWindow); Assert.True(cb.AllowRequest); clock.Increment(sleepWindow); Assert.True(cb.AllowRequest); // and the circuit should be closed again Assert.False(cb.IsOpen()); }
public void testBadRequestsDoNotAffectErrorPercentage() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock);; metrics.MarkSuccess(100); Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage); metrics.MarkFailure(1000); Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage); metrics.MarkBadRequest(1); metrics.MarkBadRequest(2); Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage); metrics.MarkFailure(45); metrics.MarkFailure(55); Assert.Equal(75, metrics.GetHealthCounts().ErrorPercentage); }
public void testErrorPercentage() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); metrics.MarkSuccess(100); Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage); clock.Increment(1); metrics.MarkFailure(1000); Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage); clock.Increment(1); metrics.MarkSuccess(100); metrics.MarkSuccess(100); Assert.Equal(25, metrics.GetHealthCounts().ErrorPercentage); clock.Increment(1); metrics.MarkTimeout(5000); metrics.MarkTimeout(5000); Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage); clock.Increment(1); metrics.MarkSuccess(100); metrics.MarkSuccess(100); metrics.MarkSuccess(100); // latent clock.Increment(1); metrics.MarkSuccess(5000); // 6 success + 1 latent success + 1 failure + 2 timeout = 10 total // latent success not considered error // error percentage = 1 failure + 2 timeout / 10 Assert.Equal(30, metrics.GetHealthCounts().ErrorPercentage); }
private T InnerExecute <T>(Func <T> primaryFunction, Func <IEnumerable <Exception>, T> fallbackFunction, CancellationTokenSource cancellationTokenSource) { if (!ConfigurationService.GetHystrixCommandEnabled()) { return(primaryFunction.Invoke()); } var innerExceptions = new List <Exception>(); Stopwatch userThreadStopWatch = Stopwatch.StartNew(); if (CircuitBreaker.AllowRequest()) { CommandMetrics.IncrementConcurrentExecutionCount(); ThreadPoolMetrics.MarkThreadExecution(); Stopwatch commandStopWatch = Stopwatch.StartNew(); try { var result = timeoutWrapper.Execute(primaryFunction, cancellationTokenSource); commandStopWatch.Stop(); CircuitBreaker.CloseCircuit(); CommandMetrics.MarkSuccess(); return(result); } catch (HystrixTimeoutException hte) { commandStopWatch.Stop(); CommandMetrics.MarkTimeout(); innerExceptions.Add(hte); } catch (Exception ex) { commandStopWatch.Stop(); CommandMetrics.MarkFailure(); CommandMetrics.MarkExceptionThrown(); innerExceptions.Add(ex); } finally { // track command execution time commandStopWatch.Stop(); CommandMetrics.AddCommandExecutionTime(commandStopWatch.Elapsed.TotalMilliseconds); CommandMetrics.DecrementConcurrentExecutionCount(); ThreadPoolMetrics.MarkThreadCompletion(); // track execution time including threading overhead userThreadStopWatch.Stop(); CommandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds); } } else { CommandMetrics.MarkShortCircuited(); // track execution time including threading overhead userThreadStopWatch.Stop(); CommandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds); } T fallbackResult = fallbackFunction.Invoke(innerExceptions); CommandMetrics.MarkFallbackSuccess(); return(fallbackResult); }
public void testMultipleTimeWindowRetriesBeforeClosingCircuit() { int sleepWindow = 200; var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.False(cb.AllowRequest); Assert.True(cb.IsOpen()); // wait for sleepWindow to pass clock.Increment(sleepWindow + 50); // we should now allow 1 request Assert.True(cb.AllowRequest); // but the circuit should still be open Assert.True(cb.IsOpen()); // and further requests are still blocked Assert.False(cb.AllowRequest); // the 'singleTest' fails so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep metrics.MarkFailure(1000); Assert.False(cb.AllowRequest); Assert.False(cb.AllowRequest); Assert.False(cb.AllowRequest); // wait for sleepWindow to pass clock.Increment(sleepWindow + 50); // we should now allow 1 request Assert.True(cb.AllowRequest); // but the circuit should still be open Assert.True(cb.IsOpen()); // and further requests are still blocked Assert.False(cb.AllowRequest); // the 'singleTest' fails again so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep metrics.MarkFailure(1000); Assert.False(cb.AllowRequest); Assert.False(cb.AllowRequest); Assert.False(cb.AllowRequest); // wait for sleepWindow to pass clock.Increment(sleepWindow + 50); // we should now allow 1 request Assert.True(cb.AllowRequest); // but the circuit should still be open Assert.True(cb.IsOpen()); // and further requests are still blocked Assert.False(cb.AllowRequest); // now it finally succeeds metrics.MarkSuccess(200); cb.MarkSuccess(); // all requests should be open again Assert.True(cb.AllowRequest); Assert.True(cb.AllowRequest); Assert.True(cb.AllowRequest); // and the circuit should be closed again Assert.False(cb.IsOpen()); }