public void CircuitBreaker_TripCircuit()
{
try
{
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter();
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
// this should still allow requests as everything has been successful
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public async Task TestTripCircuitOnTimeouts()
{
string key = "cmd-D";
HystrixCommand <bool> cmd1 = new TimeoutCommand(key);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new TimeoutCommand(key);
await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new TimeoutCommand(key);
await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new TimeoutCommand(key);
await cmd4.ExecuteAsync();
// everything has been a timeout so we should not allow any requests
Time.Wait(10);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public void CircuitBreaker_TripCircuitOnTimeouts()
{
try
{
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter();
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// this should start as allowing requests
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
// timeouts
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
// everything has been a timeout so we should not allow any requests
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void TestLowVolumeDoesNotTripCircuit()
{
String key = "cmd-I";
try
{
int sleepWindow = 200;
int lowVolume = 5;
HystrixCommand <Boolean> cmd1 = new FailureCommand(key, 60, sleepWindow, lowVolume);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new FailureCommand(key, 1, sleepWindow, lowVolume);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new FailureCommand(key, 1, sleepWindow, lowVolume);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new FailureCommand(key, 1, sleepWindow, lowVolume);
cmd4.Execute();
// even though it has all failed we won't trip the circuit because the volume is low
Time.Wait(150);
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public async Task TestTripCircuitOnTimeouts()
{
string key = "cmd-D";
HystrixCommand <bool> cmd1 = new TimeoutCommand(key);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new TimeoutCommand(key);
await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new TimeoutCommand(key);
await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new TimeoutCommand(key);
await cmd4.ExecuteAsync();
// Allow window to pass, everything has been a timeout so we should not allow any requests
// Time.Wait(125);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public async void TestCircuitClosedAfterSuccess()
{
String key = "cmd-G";
try
{
int sleepWindow = 100;
HystrixCommand <Boolean> cmd1 = new FailureCommand(key, 1, sleepWindow);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new FailureCommand(key, 1, sleepWindow);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new FailureCommand(key, 1, sleepWindow);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new TimeoutCommand(key, sleepWindow);
cmd4.Execute();
// everything has failed in the test window so we should return false now
Time.Wait(150);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("CircuitBreaker state 1 : " + cmd1.Metrics.Healthcounts);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
// wait for sleepWindow to pass
Time.Wait(sleepWindow + 50);
// but the circuit should still be open
Assert.True(cb.IsOpen);
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand <bool> cmd5 = new SuccessCommand(key, 60, sleepWindow);
IObservable <bool> asyncResult = cmd5.Observe();
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest);
await asyncResult.SingleAsync();
// all requests should be open again
Time.Wait(150);
output.WriteLine("CircuitBreaker state 2 : " + cmd1.Metrics.Healthcounts);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public void CircuitBreaker_TripCircuitOnFailuresAboveThreshold()
{
try
{
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter();
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// this should start as allowing requests
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
// success with high latency
metrics.MarkSuccess(400);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkFailure(10);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkFailure(10);
// this should trip the circuit as the error percentage is above the threshold
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void TestTripCircuitOnTimeouts()
{
String key = "cmd-D";
HystrixCommand <Boolean> cmd1 = new TimeoutCommand(key);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// success with high latency
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new TimeoutCommand(key);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new TimeoutCommand(key);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new TimeoutCommand(key);
cmd4.Execute();
// everything has been a timeout so we should not allow any requests
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
public async Task TestLowVolumeDoesNotTripCircuit()
{
string key = "cmd-I";
int sleepWindow = 200;
int lowVolume = 5;
HystrixCommand <bool> cmd1 = new FailureCommand(key, 60, sleepWindow, lowVolume);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new FailureCommand(key, 1, sleepWindow, lowVolume);
await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 1, sleepWindow, lowVolume);
await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new FailureCommand(key, 1, sleepWindow, lowVolume);
await cmd4.ExecuteAsync();
// even though it has all failed we won't trip the circuit because the volume is low
Time.Wait(150);
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
}
public HystrixCommand(HystrixCommandIdentifier commandIdentifier, IHystrixTimeoutWrapper timeoutWrapper, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandMetrics commandMetrics, IHystrixThreadPoolMetrics threadPoolMetrics, IHystrixConfigurationService configurationService)
{
if (commandIdentifier == null)
{
throw new ArgumentNullException("commandIdentifier");
}
if (timeoutWrapper == null)
{
throw new ArgumentNullException("timeoutWrapper");
}
if (circuitBreaker == null)
{
throw new ArgumentNullException("circuitBreaker");
}
if (commandMetrics == null)
{
throw new ArgumentNullException("commandMetrics");
}
if (threadPoolMetrics == null)
{
throw new ArgumentNullException("threadPoolMetrics");
}
if (configurationService == null)
{
throw new ArgumentNullException("configurationService");
}
this.commandIdentifier = commandIdentifier;
this.timeoutWrapper = timeoutWrapper;
this.circuitBreaker = circuitBreaker;
this.commandMetrics = commandMetrics;
this.threadPoolMetrics = threadPoolMetrics;
this.configurationService = configurationService;
}
public void CircuitBreaker_LowVolumeDoesNotTripCircuit()
{
try
{
int sleepWindow = 200;
int lowVolume = 5;
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithCircuitBreakerRequestVolumeThreshold(lowVolume);
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// even though it has all failed we won't trip the circuit because the volume is low
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void CircuitBreaker_SingleTestOnOpenCircuitAfterTimeWindow()
{
try
{
int sleepWindow = 200;
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public static IHystrixCircuitBreaker GetInstance(IHystrixCommandKey key)
{
IHystrixCircuitBreaker previouslyCached = null;
circuitBreakersByCommand.TryGetValue(key.Name, out previouslyCached);
return(previouslyCached);
}
public void TestTripCircuitOnTimeouts()
{
String key = "cmd-D";
try
{
HystrixCommand <Boolean> cmd1 = new TimeoutCommand(key);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// success with high latency
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new TimeoutCommand(key);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new TimeoutCommand(key);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new TimeoutCommand(key);
cmd4.Execute();
// everything has been a timeout so we should not allow any requests
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public void TestLowVolumeDoesNotTripCircuit()
{
String key = "cmd-I";
int sleepWindow = 200;
int lowVolume = 5;
HystrixCommand <Boolean> cmd1 = new FailureCommand(key, 60, sleepWindow, lowVolume);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new FailureCommand(key, 1, sleepWindow, lowVolume);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new FailureCommand(key, 1, sleepWindow, lowVolume);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new FailureCommand(key, 1, sleepWindow, lowVolume);
cmd4.Execute();
// even though it has all failed we won't trip the circuit because the volume is low
Time.Wait(150);
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
}
public async Task TestLowVolumeDoesNotTripCircuit()
{
string key = "cmd-I";
int sleepWindow = 400;
int lowVolume = 5;
HystrixCommand <bool> cmd1 = new FailureCommand(key, 0, sleepWindow, lowVolume);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new FailureCommand(key, 0, sleepWindow, lowVolume);
await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 0, sleepWindow, lowVolume);
await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new FailureCommand(key, 0, sleepWindow, lowVolume);
await cmd4.ExecuteAsync();
// Allow window to pass, even though it has all failed we won't trip the circuit because the volume is low
// Time.Wait(200);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
}
public async Task TestCircuitClosedAfterSuccess()
{
string key = "cmd-G";
int sleepWindow = 100;
HystrixCommand <bool> cmd1 = new FailureCommand(key, 1, sleepWindow);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
_ = await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new FailureCommand(key, 1, sleepWindow);
_ = await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 1, sleepWindow);
_ = await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new TimeoutCommand(key, sleepWindow);
_ = await cmd4.ExecuteAsync();
// everything has failed in the test window so we should return false now
Time.Wait(sleepWindow);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("CircuitBreaker state 1 : " + cmd1.Metrics.Healthcounts);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
// wait for sleepWindow to pass
Time.Wait(sleepWindow + 50);
// but the circuit should still be open
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand <bool> cmd5 = new SuccessCommand(key, 10, sleepWindow);
IObservable <bool> asyncResult = cmd5.Observe();
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
await asyncResult.SingleAsync();
// all requests should be open again
Time.Wait(sleepWindow + 50);
output.WriteLine("CircuitBreaker state 2 : " + cmd1.Metrics.Healthcounts);
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (1)!");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (2)!");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (3)!");
// and the circuit should be closed again
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
}
public TestCommandBuilder SetCircuitBreaker(TestCircuitBreaker circuitBreaker)
{
this.circuitBreaker = circuitBreaker;
if (circuitBreaker != null)
{
this.metrics = circuitBreaker.metrics;
}
return(this);
}
public async Task TestTripCircuitOnFailuresAboveThreshold()
{
string key = "cmd-B";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
_ = await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 0);
_ = await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 0);
_ = await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0);
_ = await cmd4.ExecuteAsync();
HystrixCommand <bool> cmd5 = new FailureCommand(key, 0);
_ = await cmd5.ExecuteAsync();
HystrixCommand <bool> cmd6 = new SuccessCommand(key, 0);
_ = await cmd6.ExecuteAsync();
HystrixCommand <bool> cmd7 = new FailureCommand(key, 0);
_ = await cmd7.ExecuteAsync();
HystrixCommand <bool> cmd8 = new FailureCommand(key, 0);
_ = await cmd8.ExecuteAsync();
// Let window pass, this should trip the circuit as the error percentage is above the threshold
// Time.Wait(125);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("Current CircuitBreaker Status : " + cmd1.Metrics.Healthcounts);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public async Task TestTripCircuitAsync()
{
string key = "cmd-A";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd3 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
_ = await cmd1.ExecuteAsync();
_ = await cmd2.ExecuteAsync();
_ = await cmd3.ExecuteAsync();
_ = await cmd4.ExecuteAsync();
// this should still allow requests as everything has been successful
// Time.Wait(125);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// fail
HystrixCommand <bool> cmd5 = new FailureCommand(key, 0);
HystrixCommand <bool> cmd6 = new FailureCommand(key, 0);
HystrixCommand <bool> cmd7 = new FailureCommand(key, 0);
HystrixCommand <bool> cmd8 = new FailureCommand(key, 0);
Assert.False(await cmd5.ExecuteAsync());
Assert.False(await cmd6.ExecuteAsync());
Assert.False(await cmd7.ExecuteAsync());
Assert.False(await cmd8.ExecuteAsync());
// make sure window has passed
// Time.Wait(125);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("Current CircuitBreaker Status : " + cmd1.Metrics.Healthcounts);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public HystrixServoMetricsPublisherCommand(
HystrixCommandKey commandKey,
HystrixCommandGroupKey commandGroupKey,
HystrixCommandMetrics metrics,
IHystrixCircuitBreaker circuitBreaker,
IHystrixCommandProperties properties)
{
this.commandKey = commandKey;
this.commandGroupKey = commandGroupKey;
this.metrics = metrics;
this.circuitBreaker = circuitBreaker;
this.properties = properties;
this.servoInstanceTag = new BasicTag("instance", commandKey.Name);
this.servoTypeTag = new BasicTag("origin", "HystrixCommand");
}
public async Task TestTripCircuitOnFailuresAboveThreshold()
{
string key = "cmd-B";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
_ = await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 1);
_ = await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 1);
_ = await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1);
_ = await cmd4.ExecuteAsync();
HystrixCommand <bool> cmd5 = new FailureCommand(key, 1);
_ = await cmd5.ExecuteAsync();
HystrixCommand <bool> cmd6 = new SuccessCommand(key, 1);
_ = await cmd6.ExecuteAsync();
HystrixCommand <bool> cmd7 = new FailureCommand(key, 1);
_ = await cmd7.ExecuteAsync();
HystrixCommand <bool> cmd8 = new FailureCommand(key, 1);
_ = await cmd8.ExecuteAsync();
// this should trip the circuit as the error percentage is above the threshold
Time.Wait(200);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public void CircuitBreaker_CircuitClosedAfterSuccessAndClearsStatisticalWindow()
{
try
{
int statisticalWindow = 200;
int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow);
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// the 'singleTest' succeeds so should cause the circuit to be closed
metrics.MarkSuccess(500);
cb.MarkSuccess();
// all requests should be open again
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
// and the circuit should be closed again
Assert.IsFalse(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public async Task TestTripCircuitOnTimeoutsAboveThreshold()
{
string key = "cmd-E";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd3 = new TimeoutCommand(key);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd5 = new TimeoutCommand(key);
HystrixCommand <bool> cmd6 = new TimeoutCommand(key);
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd8 = new TimeoutCommand(key);
HystrixCommand <bool> cmd9 = new TimeoutCommand(key);
var taskList = new List <Task>
{
cmd1.ExecuteAsync(),
cmd2.ExecuteAsync(),
cmd3.ExecuteAsync(),
cmd4.ExecuteAsync(),
cmd5.ExecuteAsync(),
cmd6.ExecuteAsync(),
cmd7.ExecuteAsync(),
cmd8.ExecuteAsync(),
cmd9.ExecuteAsync(),
};
await Task.WhenAll(taskList);
// Allow window to pass, this should trip the circuit as the error percentage is above the threshold
// Time.Wait(200);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public async Task TestSingleTestOnOpenCircuitAfterTimeWindow()
{
string key = "cmd-F";
HystrixCommand <bool> cmd1 = new FailureCommand(key, 0);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected! (1)");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new FailureCommand(key, 0);
await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 0);
await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new FailureCommand(key, 0);
await cmd4.ExecuteAsync();
// Allow window to pass, everything has failed in the test window so we should return false now
// Time.Wait(200);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
// wait for sleepWindow to pass
Time.Wait(500);
// we should now allow 1 request
Assert.True(cb.AllowRequest, "Request NOT allowed when expected! (2)");
// but the circuit should still be open
Assert.True(cb.IsOpen, "Circuit is closed when it should be open! (2)");
// and further requests are still blocked
Assert.False(cb.AllowRequest, "Request allowed when NOT expected! (2)");
}
public void TestTripCircuit()
{
String key = "cmd-A";
try
{
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd3 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
cmd1.Execute();
cmd2.Execute();
cmd3.Execute();
cmd4.Execute();
// this should still allow requests as everything has been successful
Time.Wait(150);
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// fail
HystrixCommand <bool> cmd5 = new FailureCommand(key, 1);
HystrixCommand <bool> cmd6 = new FailureCommand(key, 1);
HystrixCommand <bool> cmd7 = new FailureCommand(key, 1);
HystrixCommand <bool> cmd8 = new FailureCommand(key, 1);
Assert.False(cmd5.Execute());
Assert.False(cmd6.Execute());
Assert.False(cmd7.Execute());
Assert.False(cmd8.Execute());
// everything has failed in the test window so we should return false now
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public void TestTripCircuitOnTimeoutsAboveThreshold()
{
String key = "cmd-E";
try
{
HystrixCommand <Boolean> cmd1 = new SuccessCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// success with high latency
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new SuccessCommand(key, 1);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new TimeoutCommand(key);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new SuccessCommand(key, 1);
cmd4.Execute();
HystrixCommand <Boolean> cmd5 = new TimeoutCommand(key);
cmd5.Execute();
HystrixCommand <Boolean> cmd6 = new TimeoutCommand(key);
cmd6.Execute();
HystrixCommand <Boolean> cmd7 = new SuccessCommand(key, 1);
cmd7.Execute();
HystrixCommand <Boolean> cmd8 = new TimeoutCommand(key);
cmd8.Execute();
HystrixCommand <Boolean> cmd9 = new TimeoutCommand(key);
cmd9.Execute();
// this should trip the circuit as the error percentage is above the threshold
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public void TestSingleTestOnOpenCircuitAfterTimeWindow()
{
String key = "cmd-F";
try
{
int sleepWindow = 200;
HystrixCommand <Boolean> cmd1 = new FailureCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new FailureCommand(key, 1);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new FailureCommand(key, 1);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new FailureCommand(key, 1);
cmd4.Execute();
// everything has failed in the test window so we should return false now
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
// wait for sleepWindow to pass
Time.Wait(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen);
// and further requests are still blocked
Assert.False(cb.AllowRequest);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public void TestCircuitDoesNotTripOnFailuresBelowThreshold()
{
String key = "cmd-C";
try
{
HystrixCommand <Boolean> cmd1 = new SuccessCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// success with high latency
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new SuccessCommand(key, 1);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new FailureCommand(key, 1);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new SuccessCommand(key, 1);
cmd4.Execute();
HystrixCommand <Boolean> cmd5 = new SuccessCommand(key, 1);
cmd5.Execute();
HystrixCommand <Boolean> cmd6 = new FailureCommand(key, 1);
cmd6.Execute();
HystrixCommand <Boolean> cmd7 = new SuccessCommand(key, 1);
cmd7.Execute();
HystrixCommand <Boolean> cmd8 = new FailureCommand(key, 1);
cmd8.Execute();
// this should remain closed as the failure threshold is below the percentage limit
Time.Wait(150);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("Current CircuitBreaker Status : " + cmd1.Metrics.Healthcounts);
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public void TestTripCircuitOnTimeoutsAboveThreshold()
{
string key = "cmd-E";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// success with high latency
cmd1.Execute();
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 1);
cmd2.Execute();
HystrixCommand <bool> cmd3 = new TimeoutCommand(key);
cmd3.Execute();
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1);
cmd4.Execute();
HystrixCommand <bool> cmd5 = new TimeoutCommand(key);
cmd5.Execute();
HystrixCommand <bool> cmd6 = new TimeoutCommand(key);
cmd6.Execute();
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 1);
cmd7.Execute();
HystrixCommand <bool> cmd8 = new TimeoutCommand(key);
cmd8.Execute();
HystrixCommand <bool> cmd9 = new TimeoutCommand(key);
cmd9.Execute();
// this should trip the circuit as the error percentage is above the threshold
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
public HystrixMetricsPublisherCommandDefault(HystrixCommandKey commandKey, HystrixCommandGroupKey commandGroupKey, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandProperties properties)
{
// do nothing by default
}
public virtual IHystrixMetricsPublisherCommand GetMetricsPublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandGroupKey, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandProperties properties)
{
return new HystrixMetricsPublisherCommandDefault(commandKey, commandGroupKey, metrics, circuitBreaker, properties);
}
public static IHystrixMetricsPublisherCommand CreateOrRetrievePublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandProperties properties)
{
return instance.GetPublisherForCommand(commandKey, commandOwner, metrics, circuitBreaker, properties);
}
public IHystrixMetricsPublisherCommand GetMetricsPublisherForCommand(Elders.Hystrix.NET.HystrixCommandKey commandKey, Elders.Hystrix.NET.HystrixCommandGroupKey commandGroupKey, Elders.Hystrix.NET.HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, Elders.Hystrix.NET.IHystrixCommandProperties properties)
{
return new HystrixDelegateMetricsPublisherCommand(() => this.commandCounter.IncrementAndGet());
}
public IHystrixMetricsPublisherCommand GetPublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandProperties properties)
{
return this.commandPublishers.GetOrAdd(commandKey.Name,
w => this.strategy.GetMetricsPublisherForCommand(commandKey, commandOwner, metrics, circuitBreaker, properties),
w => w.Initialize());
}
