public void CircuitBreaker_TripCircuit() { try { HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter(); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); metrics.MarkSuccess(1000); metrics.MarkSuccess(1000); metrics.MarkSuccess(1000); metrics.MarkSuccess(1000); // this should still allow requests as everything has been successful Assert.IsTrue(cb.AllowRequest()); Assert.IsFalse(cb.IsOpen()); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
public void CircuitBreaker_TripCircuitOnFailuresAboveThreshold() { try { HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter(); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // this should start as allowing requests Assert.IsTrue(cb.AllowRequest()); Assert.IsFalse(cb.IsOpen()); // success with high latency metrics.MarkSuccess(400); metrics.MarkSuccess(400); metrics.MarkFailure(10); metrics.MarkSuccess(400); metrics.MarkFailure(10); metrics.MarkFailure(10); metrics.MarkSuccess(400); metrics.MarkFailure(10); metrics.MarkFailure(10); // this should trip the circuit as the error percentage is above the threshold Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
public void CircuitBreaker_SingleTestOnOpenCircuitAfterTimeWindow() { try { int sleepWindow = 200; HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
public void CircuitBreaker_TripCircuitOnTimeouts() { try { HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter(); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // this should start as allowing requests Assert.IsTrue(cb.AllowRequest()); Assert.IsFalse(cb.IsOpen()); // timeouts metrics.MarkTimeout(2000); metrics.MarkTimeout(2000); metrics.MarkTimeout(2000); metrics.MarkTimeout(2000); // everything has been a timeout so we should not allow any requests Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
public void CircuitBreaker_LowVolumeDoesNotTripCircuit() { try { int sleepWindow = 200; int lowVolume = 5; HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithCircuitBreakerRequestVolumeThreshold(lowVolume); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // even though it has all failed we won't trip the circuit because the volume is low Assert.IsTrue(cb.AllowRequest()); Assert.IsFalse(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
public void CircuitBreaker_CircuitClosedAfterSuccessAndClearsStatisticalWindow() { try { int statisticalWindow = 200; int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); // the 'singleTest' succeeds so should cause the circuit to be closed metrics.MarkSuccess(500); cb.MarkSuccess(); // all requests should be open again Assert.IsTrue(cb.AllowRequest()); Assert.IsTrue(cb.AllowRequest()); Assert.IsTrue(cb.AllowRequest()); // and the circuit should be closed again Assert.IsFalse(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
private T InnerExecute <T>(Func <T> primaryFunction, Func <IEnumerable <Exception>, T> fallbackFunction, CancellationTokenSource cancellationTokenSource) { if (!configurationService.GetHystrixCommandEnabled()) { return(primaryFunction.Invoke()); } var innerExceptions = new List <Exception>(); Stopwatch userThreadStopWatch = Stopwatch.StartNew(); if (circuitBreaker.AllowRequest()) { commandMetrics.IncrementConcurrentExecutionCount(); threadPoolMetrics.MarkThreadExecution(); Stopwatch commandStopWatch = Stopwatch.StartNew(); try { var result = timeoutWrapper.Execute(primaryFunction, cancellationTokenSource); commandStopWatch.Stop(); circuitBreaker.CloseCircuit(); commandMetrics.MarkSuccess(); return(result); } catch (HystrixTimeoutException hte) { commandStopWatch.Stop(); commandMetrics.MarkTimeout(); innerExceptions.Add(hte); } catch (Exception ex) { commandStopWatch.Stop(); commandMetrics.MarkFailure(); commandMetrics.MarkExceptionThrown(); innerExceptions.Add(ex); } finally { // track command execution time commandStopWatch.Stop(); commandMetrics.AddCommandExecutionTime(commandStopWatch.Elapsed.TotalMilliseconds); commandMetrics.DecrementConcurrentExecutionCount(); threadPoolMetrics.MarkThreadCompletion(); // track execution time including threading overhead userThreadStopWatch.Stop(); commandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds); } } else { commandMetrics.MarkShortCircuited(); // track execution time including threading overhead userThreadStopWatch.Stop(); commandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds); } T fallbackResult = fallbackFunction.Invoke(innerExceptions); commandMetrics.MarkFallbackSuccess(); return(fallbackResult); }
public void CircuitBreaker_MultipleTimeWindowRetriesBeforeClosingCircuit() { try { int sleepWindow = 200; HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); // the 'singleTest' fails so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep metrics.MarkFailure(1000); Assert.IsFalse(cb.AllowRequest()); Assert.IsFalse(cb.AllowRequest()); Assert.IsFalse(cb.AllowRequest()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); // the 'singleTest' fails again so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep metrics.MarkFailure(1000); Assert.IsFalse(cb.AllowRequest()); Assert.IsFalse(cb.AllowRequest()); Assert.IsFalse(cb.AllowRequest()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); // now it finally succeeds metrics.MarkSuccess(200); cb.MarkSuccess(); // all requests should be open again Assert.IsTrue(cb.AllowRequest()); Assert.IsTrue(cb.AllowRequest()); Assert.IsTrue(cb.AllowRequest()); // and the circuit should be closed again Assert.IsFalse(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }