public void CircuitBreaker_TripCircuit()
{
try
{
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter();
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
// this should still allow requests as everything has been successful
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void CircuitBreaker_TripCircuitOnFailuresAboveThreshold()
{
try
{
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter();
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// this should start as allowing requests
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
// success with high latency
metrics.MarkSuccess(400);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkFailure(10);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkFailure(10);
// this should trip the circuit as the error percentage is above the threshold
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void CircuitBreaker_SingleTestOnOpenCircuitAfterTimeWindow()
{
try
{
int sleepWindow = 200;
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void CircuitBreaker_TripCircuitOnTimeouts()
{
try
{
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter();
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// this should start as allowing requests
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
// timeouts
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
// everything has been a timeout so we should not allow any requests
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void CircuitBreaker_LowVolumeDoesNotTripCircuit()
{
try
{
int sleepWindow = 200;
int lowVolume = 5;
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithCircuitBreakerRequestVolumeThreshold(lowVolume);
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// even though it has all failed we won't trip the circuit because the volume is low
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void CircuitBreaker_CircuitClosedAfterSuccessAndClearsStatisticalWindow()
{
try
{
int statisticalWindow = 200;
int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow);
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// the 'singleTest' succeeds so should cause the circuit to be closed
metrics.MarkSuccess(500);
cb.MarkSuccess();
// all requests should be open again
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
// and the circuit should be closed again
Assert.IsFalse(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
private T InnerExecute <T>(Func <T> primaryFunction, Func <IEnumerable <Exception>, T> fallbackFunction, CancellationTokenSource cancellationTokenSource)
{
if (!configurationService.GetHystrixCommandEnabled())
{
return(primaryFunction.Invoke());
}
var innerExceptions = new List <Exception>();
Stopwatch userThreadStopWatch = Stopwatch.StartNew();
if (circuitBreaker.AllowRequest())
{
commandMetrics.IncrementConcurrentExecutionCount();
threadPoolMetrics.MarkThreadExecution();
Stopwatch commandStopWatch = Stopwatch.StartNew();
try
{
var result = timeoutWrapper.Execute(primaryFunction, cancellationTokenSource);
commandStopWatch.Stop();
circuitBreaker.CloseCircuit();
commandMetrics.MarkSuccess();
return(result);
}
catch (HystrixTimeoutException hte)
{
commandStopWatch.Stop();
commandMetrics.MarkTimeout();
innerExceptions.Add(hte);
}
catch (Exception ex)
{
commandStopWatch.Stop();
commandMetrics.MarkFailure();
commandMetrics.MarkExceptionThrown();
innerExceptions.Add(ex);
}
finally
{
// track command execution time
commandStopWatch.Stop();
commandMetrics.AddCommandExecutionTime(commandStopWatch.Elapsed.TotalMilliseconds);
commandMetrics.DecrementConcurrentExecutionCount();
threadPoolMetrics.MarkThreadCompletion();
// track execution time including threading overhead
userThreadStopWatch.Stop();
commandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds);
}
}
else
{
commandMetrics.MarkShortCircuited();
// track execution time including threading overhead
userThreadStopWatch.Stop();
commandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds);
}
T fallbackResult = fallbackFunction.Invoke(innerExceptions);
commandMetrics.MarkFallbackSuccess();
return(fallbackResult);
}
public void CircuitBreaker_MultipleTimeWindowRetriesBeforeClosingCircuit()
{
try
{
int sleepWindow = 200;
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// the 'singleTest' fails so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep
metrics.MarkFailure(1000);
Assert.IsFalse(cb.AllowRequest());
Assert.IsFalse(cb.AllowRequest());
Assert.IsFalse(cb.AllowRequest());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// the 'singleTest' fails again so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep
metrics.MarkFailure(1000);
Assert.IsFalse(cb.AllowRequest());
Assert.IsFalse(cb.AllowRequest());
Assert.IsFalse(cb.AllowRequest());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// now it finally succeeds
metrics.MarkSuccess(200);
cb.MarkSuccess();
// all requests should be open again
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
// and the circuit should be closed again
Assert.IsFalse(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}