public void Command_CircuitBreakerAcrossMultipleCommandsButSameCircuitBreaker()
{
TestCircuitBreaker circuitBreaker = new TestCircuitBreaker();
/* fail 3 times and then it should trip the circuit and stop executing */
// failure 1
KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker);
attempt1.Execute();
Assert.IsTrue(attempt1.IsResponseFromFallback);
Assert.IsFalse(attempt1.IsCircuitBreakerOpen);
Assert.IsFalse(attempt1.IsResponseShortCircuited);
// failure 2 with a different command, same circuit breaker
KnownFailureTestCommandWithoutFallback attempt2 = new KnownFailureTestCommandWithoutFallback(circuitBreaker);
try
{
attempt2.Execute();
}
catch (Exception)
{
// ignore ... this doesn't have a fallback so will throw an exception
}
Assert.IsTrue(attempt2.IsFailedExecution);
Assert.IsFalse(attempt2.IsResponseFromFallback); // false because no fallback
Assert.IsFalse(attempt2.IsCircuitBreakerOpen);
Assert.IsFalse(attempt2.IsResponseShortCircuited);
// failure 3 of the Hystrix, 2nd for this particular HystrixCommand
KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker);
attempt3.Execute();
Assert.IsTrue(attempt2.IsFailedExecution);
Assert.IsTrue(attempt3.IsResponseFromFallback);
Assert.IsFalse(attempt3.IsResponseShortCircuited);
// it should now be 'open' and prevent further executions
// after having 3 failures on the Hystrix that these 2 different HystrixCommand objects are for
Assert.IsTrue(attempt3.IsCircuitBreakerOpen);
// attempt 4
KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker);
attempt4.Execute();
Assert.IsTrue(attempt4.IsResponseFromFallback);
// this should now be true as the response will be short-circuited
Assert.IsTrue(attempt4.IsResponseShortCircuited);
// this should remain open
Assert.IsTrue(attempt4.IsCircuitBreakerOpen);
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Success));
Assert.AreEqual(1, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown));
Assert.AreEqual(3, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Failure));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure));
Assert.AreEqual(3, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected));
Assert.AreEqual(1, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Timeout));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache));
Assert.AreEqual(100, circuitBreaker.Metrics.GetHealthCounts().ErrorPercentage);
Assert.AreEqual(4, HystrixRequestLog.GetCurrentRequest().ExecutedCommands.Count());
Hystrix.Reset();
}
public void Command_CircuitBreakerAcrossMultipleCommandsAndDifferentDependency()
{
TestCircuitBreaker circuitBreaker_one = new TestCircuitBreaker();
TestCircuitBreaker circuitBreaker_two = new TestCircuitBreaker();
/* fail 3 times, twice on one Hystrix, once on a different Hystrix ... circuit-breaker should NOT open */
// failure 1
KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker_one);
attempt1.Execute();
Assert.IsTrue(attempt1.IsResponseFromFallback);
Assert.IsFalse(attempt1.IsCircuitBreakerOpen);
Assert.IsFalse(attempt1.IsResponseShortCircuited);
// failure 2 with a different HystrixCommand implementation and different Hystrix
KnownFailureTestCommandWithFallback attempt2 = new KnownFailureTestCommandWithFallback(circuitBreaker_two);
attempt2.Execute();
Assert.IsTrue(attempt2.IsResponseFromFallback);
Assert.IsFalse(attempt2.IsCircuitBreakerOpen);
Assert.IsFalse(attempt2.IsResponseShortCircuited);
// failure 3 but only 2nd of the Hystrix.ONE
KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker_one);
attempt3.Execute();
Assert.IsTrue(attempt3.IsResponseFromFallback);
Assert.IsFalse(attempt3.IsResponseShortCircuited);
// it should remain 'closed' since we have only had 2 failures on Hystrix.ONE
Assert.IsFalse(attempt3.IsCircuitBreakerOpen);
// this one should also remain closed as it only had 1 failure for Hystrix.TWO
Assert.IsFalse(attempt2.IsCircuitBreakerOpen);
// attempt 4 (3rd attempt for Hystrix.ONE)
KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker_one);
attempt4.Execute();
// this should NOW flip to true as this is the 3rd failure for Hystrix.ONE
Assert.IsTrue(attempt3.IsCircuitBreakerOpen);
Assert.IsTrue(attempt3.IsResponseFromFallback);
Assert.IsFalse(attempt3.IsResponseShortCircuited);
// Hystrix.TWO should still remain closed
Assert.IsFalse(attempt2.IsCircuitBreakerOpen);
Assert.AreEqual(0, circuitBreaker_one.Metrics.GetRollingCount(HystrixRollingNumberEvent.Success));
Assert.AreEqual(0, circuitBreaker_one.Metrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown));
Assert.AreEqual(3, circuitBreaker_one.Metrics.GetRollingCount(HystrixRollingNumberEvent.Failure));
Assert.AreEqual(0, circuitBreaker_one.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection));
Assert.AreEqual(0, circuitBreaker_one.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure));
Assert.AreEqual(3, circuitBreaker_one.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess));
Assert.AreEqual(0, circuitBreaker_one.Metrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected));
Assert.AreEqual(0, circuitBreaker_one.Metrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited));
Assert.AreEqual(0, circuitBreaker_one.Metrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected));
Assert.AreEqual(0, circuitBreaker_one.Metrics.GetRollingCount(HystrixRollingNumberEvent.Timeout));
Assert.AreEqual(0, circuitBreaker_one.Metrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache));
Assert.AreEqual(100, circuitBreaker_one.Metrics.GetHealthCounts().ErrorPercentage);
Assert.AreEqual(0, circuitBreaker_two.Metrics.GetRollingCount(HystrixRollingNumberEvent.Success));
Assert.AreEqual(0, circuitBreaker_two.Metrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown));
Assert.AreEqual(1, circuitBreaker_two.Metrics.GetRollingCount(HystrixRollingNumberEvent.Failure));
Assert.AreEqual(0, circuitBreaker_two.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection));
Assert.AreEqual(0, circuitBreaker_two.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure));
Assert.AreEqual(1, circuitBreaker_two.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess));
Assert.AreEqual(0, circuitBreaker_two.Metrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected));
Assert.AreEqual(0, circuitBreaker_two.Metrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited));
Assert.AreEqual(0, circuitBreaker_two.Metrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected));
Assert.AreEqual(0, circuitBreaker_two.Metrics.GetRollingCount(HystrixRollingNumberEvent.Timeout));
Assert.AreEqual(0, circuitBreaker_two.Metrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache));
Assert.AreEqual(100, circuitBreaker_two.Metrics.GetHealthCounts().ErrorPercentage);
Assert.AreEqual(4, HystrixRequestLog.GetCurrentRequest().ExecutedCommands.Count());
Hystrix.Reset();
}
public void Command_CircuitBreakerTripsAfterFailuresViaQueue()
{
TestCircuitBreaker circuitBreaker = new TestCircuitBreaker();
try
{
/* fail 3 times and then it should trip the circuit and stop executing */
// failure 1
KnownFailureTestCommandWithFallback attempt1 = new KnownFailureTestCommandWithFallback(circuitBreaker);
attempt1.Queue().Get();
Assert.IsTrue(attempt1.IsResponseFromFallback);
Assert.IsFalse(attempt1.IsCircuitBreakerOpen);
Assert.IsFalse(attempt1.IsResponseShortCircuited);
// failure 2
KnownFailureTestCommandWithFallback attempt2 = new KnownFailureTestCommandWithFallback(circuitBreaker);
attempt2.Queue().Get();
Assert.IsTrue(attempt2.IsResponseFromFallback);
Assert.IsFalse(attempt2.IsCircuitBreakerOpen);
Assert.IsFalse(attempt2.IsResponseShortCircuited);
// failure 3
KnownFailureTestCommandWithFallback attempt3 = new KnownFailureTestCommandWithFallback(circuitBreaker);
attempt3.Queue().Get();
Assert.IsTrue(attempt3.IsResponseFromFallback);
Assert.IsFalse(attempt3.IsResponseShortCircuited);
// it should now be 'open' and prevent further executions
Assert.IsTrue(attempt3.IsCircuitBreakerOpen);
// attempt 4
KnownFailureTestCommandWithFallback attempt4 = new KnownFailureTestCommandWithFallback(circuitBreaker);
attempt4.Queue().Get();
Assert.IsTrue(attempt4.IsResponseFromFallback);
// this should now be true as the response will be short-circuited
Assert.IsTrue(attempt4.IsResponseShortCircuited);
// this should remain open
Assert.IsTrue(attempt4.IsCircuitBreakerOpen);
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Success));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown));
Assert.AreEqual(3, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Failure));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure));
Assert.AreEqual(4, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected));
Assert.AreEqual(1, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Timeout));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache));
Assert.AreEqual(100, circuitBreaker.Metrics.GetHealthCounts().ErrorPercentage);
Assert.AreEqual(4, HystrixRequestLog.GetCurrentRequest().ExecutedCommands.Count());
}
catch (Exception)
{
Assert.Fail("We should have received fallbacks.");
}
Hystrix.Reset();
}
public void Command_QueueFailureWithFallback()
{
TestHystrixCommand<bool> command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker());
try
{
IFuture<bool> future = command.Queue();
Assert.AreEqual(false, future.Get());
}
catch (Exception)
{
Assert.Fail("We should have received a response from the fallback.");
}
Assert.IsTrue(command.ExecutionTimeInMilliseconds > -1);
Assert.IsTrue(command.IsFailedExecution);
Assert.AreEqual(0, command.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.Success));
Assert.AreEqual(0, command.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown));
Assert.AreEqual(1, command.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.Failure));
Assert.AreEqual(0, command.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection));
Assert.AreEqual(0, command.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure));
Assert.AreEqual(1, command.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess));
Assert.AreEqual(0, command.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected));
Assert.AreEqual(0, command.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited));
Assert.AreEqual(0, command.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected));
Assert.AreEqual(0, command.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.Timeout));
Assert.AreEqual(0, command.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache));
Assert.AreEqual(100, command.Builder.Metrics.GetHealthCounts().ErrorPercentage);
Assert.AreEqual(1, HystrixRequestLog.GetCurrentRequest().ExecutedCommands.Count());
Hystrix.Reset();
}
public void Command_ExecutionFailureWithFallbackImplementedButDisabled()
{
TestHystrixCommand<bool> commandEnabled = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker(), true);
try
{
Assert.AreEqual(false, commandEnabled.Execute());
}
catch (Exception)
{
Assert.Fail("We should have received a response from the fallback.");
}
TestHystrixCommand<bool> commandDisabled = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker(), false);
try
{
Assert.AreEqual(false, commandDisabled.Execute());
Assert.Fail("expect exception thrown");
}
catch (Exception)
{
// expected
}
Assert.AreEqual("we failed with a simulated issue", commandDisabled.FailedExecutionException.Message);
Assert.IsTrue(commandDisabled.IsFailedExecution);
Assert.AreEqual(0, commandDisabled.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.Success));
Assert.AreEqual(1, commandDisabled.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown));
Assert.AreEqual(1, commandDisabled.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.Failure));
Assert.AreEqual(0, commandDisabled.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection));
Assert.AreEqual(0, commandDisabled.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure));
Assert.AreEqual(0, commandDisabled.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess));
Assert.AreEqual(0, commandDisabled.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected));
Assert.AreEqual(0, commandDisabled.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited));
Assert.AreEqual(0, commandDisabled.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected));
Assert.AreEqual(0, commandDisabled.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.Timeout));
Assert.AreEqual(0, commandDisabled.Builder.Metrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache));
Assert.AreEqual(100, commandDisabled.Builder.Metrics.GetHealthCounts().ErrorPercentage);
Assert.AreEqual(2, HystrixRequestLog.GetCurrentRequest().ExecutedCommands.Count());
Hystrix.Reset();
}
public void Command_ExecutionHookRunFailureWithFallback()
{
/* test with Execute() */
TestHystrixCommand<bool> command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker());
command.Execute();
// the run() method should run as we're not short-circuited or rejected
Assert.AreEqual(1, command.Builder.ExecutionHook.StartRun);
// we should not have a response from run since run() failed
Assert.IsNull(command.Builder.ExecutionHook.RunSuccessResponse);
// we should have an exception since run() failed
Assert.IsNotNull(command.Builder.ExecutionHook.RunFailureException);
// the fallback() method should be run since run() failed
Assert.AreEqual(1, command.Builder.ExecutionHook.StartFallback);
// a response since fallback is implemented
Assert.IsNotNull(command.Builder.ExecutionHook.FallbackSuccessResponse);
// null since it's implemented and succeeds
Assert.IsNull(command.Builder.ExecutionHook.FallbackFailureException);
// the Execute() method was used
Assert.AreEqual(1, command.Builder.ExecutionHook.StartExecute);
// we should have a response from Execute() since we expect a fallback despite failure of run()
Assert.IsNotNull(command.Builder.ExecutionHook.EndExecuteSuccessResponse);
// we should not have an exception because we expect a fallback
Assert.IsNull(command.Builder.ExecutionHook.EndExecuteFailureException);
// thread execution
Assert.AreEqual(1, command.Builder.ExecutionHook.ThreadStart);
Assert.AreEqual(1, command.Builder.ExecutionHook.ThreadComplete);
/* test with queue() */
command = new KnownFailureTestCommandWithFallback(new TestCircuitBreaker());
try
{
command.Queue().Get();
}
catch (Exception e)
{
throw new Exception("Unexpected exception.", e);
}
// the run() method should run as we're not short-circuited or rejected
Assert.AreEqual(1, command.Builder.ExecutionHook.StartRun);
// we should not have a response from run since run() failed
Assert.IsNull(command.Builder.ExecutionHook.RunSuccessResponse);
// we should have an exception since run() failed
Assert.IsNotNull(command.Builder.ExecutionHook.RunFailureException);
// the fallback() method should be run since run() failed
Assert.AreEqual(1, command.Builder.ExecutionHook.StartFallback);
// a response since fallback is implemented
Assert.IsNotNull(command.Builder.ExecutionHook.FallbackSuccessResponse);
// null since it's implemented and succeeds
Assert.IsNull(command.Builder.ExecutionHook.FallbackFailureException);
// the queue() method was used
Assert.AreEqual(1, command.Builder.ExecutionHook.StartExecute);
// we should have a response from queue() since we expect a fallback despite failure of run()
Assert.IsNotNull(command.Builder.ExecutionHook.EndExecuteSuccessResponse);
// we should not have an exception because we expect a fallback
Assert.IsNull(command.Builder.ExecutionHook.EndExecuteFailureException);
// thread execution
Assert.AreEqual(1, command.Builder.ExecutionHook.ThreadStart);
Assert.AreEqual(1, command.Builder.ExecutionHook.ThreadComplete);
Hystrix.Reset();
}
public void Command_ShortCircuitFallbackCounter()
{
TestCircuitBreaker circuitBreaker = new TestCircuitBreaker().SetForceShortCircuit(true);
try
{
new KnownFailureTestCommandWithFallback(circuitBreaker).Execute();
Assert.AreEqual(1, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited));
KnownFailureTestCommandWithFallback command = new KnownFailureTestCommandWithFallback(circuitBreaker);
command.Execute();
Assert.AreEqual(2, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited));
// will be -1 because it never attempted execution
Assert.IsTrue(command.ExecutionTimeInMilliseconds == -1);
Assert.IsTrue(command.IsResponseShortCircuited);
Assert.IsFalse(command.IsResponseTimedOut);
// because it was short-circuited to a fallback we don't count an error
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Failure));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Success));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Timeout));
}
catch (Exception)
{
Assert.Fail("We should have received a response from the fallback.");
}
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Success));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Failure));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure));
Assert.AreEqual(2, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected));
Assert.AreEqual(2, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Timeout));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache));
Assert.AreEqual(100, circuitBreaker.Metrics.GetHealthCounts().ErrorPercentage);
Assert.AreEqual(2, HystrixRequestLog.GetCurrentRequest().ExecutedCommands.Count());
Hystrix.Reset();
}