public void testTripCircuitOnTimeoutsAboveThreshold()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// success with high latency
metrics.MarkSuccess(400);
metrics.MarkSuccess(400);
metrics.MarkTimeout(10);
metrics.MarkSuccess(400);
metrics.MarkTimeout(10);
metrics.MarkTimeout(10);
metrics.MarkSuccess(400);
metrics.MarkTimeout(10);
metrics.MarkTimeout(10);
// this should trip the circuit as the error percentage is above the threshold
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
}
public void testTripCircuitOnTimeouts()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// timeouts
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
// everything has been a timeout so we should not allow any requests
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
}
public void testErrorPercentage()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
metrics.MarkSuccess(100);
Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkFailure(1000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
Assert.Equal(25, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkTimeout(5000);
metrics.MarkTimeout(5000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
// latent
clock.Increment(1);
metrics.MarkSuccess(5000);
// 6 success + 1 latent success + 1 failure + 2 timeout = 10 total
// latent success not considered error
// error percentage = 1 failure + 2 timeout / 10
Assert.Equal(30, metrics.GetHealthCounts().ErrorPercentage);
}
public void testCircuitClosedAfterSuccess()
{
int sleepWindow = 1000;
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter()
.WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkTimeout(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
clock.Increment(sleepWindow);
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' succeeds so should cause the circuit to be closed
metrics.MarkSuccess(500);
clock.Increment(sleepWindow);
cb.MarkSuccess();
// all requests should be open again
Assert.True(cb.AllowRequest);
clock.Increment(sleepWindow);
Assert.True(cb.AllowRequest);
clock.Increment(sleepWindow);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen());
}
private T InnerExecute <T>(Func <T> primaryFunction, Func <IEnumerable <Exception>, T> fallbackFunction, CancellationTokenSource cancellationTokenSource)
{
if (!ConfigurationService.GetHystrixCommandEnabled())
{
return(primaryFunction.Invoke());
}
var innerExceptions = new List <Exception>();
Stopwatch userThreadStopWatch = Stopwatch.StartNew();
if (CircuitBreaker.AllowRequest())
{
CommandMetrics.IncrementConcurrentExecutionCount();
ThreadPoolMetrics.MarkThreadExecution();
Stopwatch commandStopWatch = Stopwatch.StartNew();
try
{
var result = timeoutWrapper.Execute(primaryFunction, cancellationTokenSource);
commandStopWatch.Stop();
CircuitBreaker.CloseCircuit();
CommandMetrics.MarkSuccess();
return(result);
}
catch (HystrixTimeoutException hte)
{
commandStopWatch.Stop();
CommandMetrics.MarkTimeout();
innerExceptions.Add(hte);
}
catch (Exception ex)
{
commandStopWatch.Stop();
CommandMetrics.MarkFailure();
CommandMetrics.MarkExceptionThrown();
innerExceptions.Add(ex);
}
finally
{
// track command execution time
commandStopWatch.Stop();
CommandMetrics.AddCommandExecutionTime(commandStopWatch.Elapsed.TotalMilliseconds);
CommandMetrics.DecrementConcurrentExecutionCount();
ThreadPoolMetrics.MarkThreadCompletion();
// track execution time including threading overhead
userThreadStopWatch.Stop();
CommandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds);
}
}
else
{
CommandMetrics.MarkShortCircuited();
// track execution time including threading overhead
userThreadStopWatch.Stop();
CommandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds);
}
T fallbackResult = fallbackFunction.Invoke(innerExceptions);
CommandMetrics.MarkFallbackSuccess();
return(fallbackResult);
}
