public async void TestCircuitClosedAfterSuccess()
{
String key = "cmd-G";
try
{
int sleepWindow = 100;
HystrixCommand <Boolean> cmd1 = new FailureCommand(key, 1, sleepWindow);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new FailureCommand(key, 1, sleepWindow);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new FailureCommand(key, 1, sleepWindow);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new TimeoutCommand(key, sleepWindow);
cmd4.Execute();
// everything has failed in the test window so we should return false now
Time.Wait(150);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("CircuitBreaker state 1 : " + cmd1.Metrics.Healthcounts);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
// wait for sleepWindow to pass
Time.Wait(sleepWindow + 50);
// but the circuit should still be open
Assert.True(cb.IsOpen);
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand <bool> cmd5 = new SuccessCommand(key, 60, sleepWindow);
IObservable <bool> asyncResult = cmd5.Observe();
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest);
await asyncResult.SingleAsync();
// all requests should be open again
Time.Wait(150);
output.WriteLine("CircuitBreaker state 2 : " + cmd1.Metrics.Healthcounts);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public void ShouldPassAfter2RepeatsAndTimeoutIsResetEachTime()
{
// Rather than testing with sleeps, this tests that the execution will occur in the correct
// order by checking which commands are run when. As the repeat command comes first, the
// timeout will be reset each time it runs
var test = TestBuilder.MakeTestFromMethod(typeof(HelperMethodForTimeoutsClass), nameof(HelperMethodForTimeoutsClass.ShouldPassAfter2RepeatsAndTimeoutIsResetEachTime));
SimpleWorkItem work = TestBuilder.CreateWorkItem(test) as SimpleWorkItem;
var method = typeof(SimpleWorkItem).GetMethod("MakeTestCommand", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly);
TestCommand command = (TestCommand)method.Invoke(work, null);
Assert.That(command, Is.TypeOf(typeof(RepeatAttribute.RepeatedTestCommand)));
RepeatAttribute.RepeatedTestCommand repeatedCommand = (RepeatAttribute.RepeatedTestCommand)command;
command = GetInnerCommand(repeatedCommand);
Assert.That(command, Is.TypeOf(typeof(TimeoutCommand)));
TimeoutCommand timeoutCommand = (TimeoutCommand)command;
command = GetInnerCommand(timeoutCommand);
Assert.That(command, Is.TypeOf(typeof(ApplyChangesToContextCommand)));
ApplyChangesToContextCommand applyChangesToContextCommand = (ApplyChangesToContextCommand)command;
command = GetInnerCommand(applyChangesToContextCommand);
Assert.That(command, Is.TypeOf(typeof(TestMethodCommand)));
}
public void TestTripCircuitOnTimeouts()
{
String key = "cmd-D";
HystrixCommand <Boolean> cmd1 = new TimeoutCommand(key);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// success with high latency
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new TimeoutCommand(key);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new TimeoutCommand(key);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new TimeoutCommand(key);
cmd4.Execute();
// everything has been a timeout so we should not allow any requests
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
public async Task TestTripCircuitOnTimeouts()
{
string key = "cmd-D";
HystrixCommand <bool> cmd1 = new TimeoutCommand(key);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new TimeoutCommand(key);
await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new TimeoutCommand(key);
await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new TimeoutCommand(key);
await cmd4.ExecuteAsync();
// everything has been a timeout so we should not allow any requests
Time.Wait(10);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public async Task TestTripCircuitOnTimeouts()
{
var key = "cmd-D";
HystrixCommand <bool> cmd1 = new TimeoutCommand(key);
var cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new TimeoutCommand(key);
await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new TimeoutCommand(key);
await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new TimeoutCommand(key);
await cmd4.ExecuteAsync();
// Allow window to pass, everything has been a timeout so we should not allow any requests
// Time.Wait(125);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public void TestTripCircuitOnTimeouts()
{
String key = "cmd-D";
try
{
HystrixCommand <Boolean> cmd1 = new TimeoutCommand(key);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// success with high latency
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new TimeoutCommand(key);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new TimeoutCommand(key);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new TimeoutCommand(key);
cmd4.Execute();
// everything has been a timeout so we should not allow any requests
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public static IEnumerable <ITestCommand> Make(ITestClassCommand classCommand,
IMethodInfo method)
{
foreach (var testCommand in classCommand.EnumerateTestCommands(method))
{
ITestCommand wrappedCommand = testCommand;
// Timeout (if they have one) -> Capture -> Timed -> Lifetime (if we need an instance) -> BeforeAfter
wrappedCommand = new BeforeAfterCommand(wrappedCommand, method.MethodInfo);
if (testCommand.ShouldCreateInstance)
{
wrappedCommand = new LifetimeCommand(wrappedCommand, method);
}
wrappedCommand = new TimedCommand(wrappedCommand);
wrappedCommand = new ExceptionCaptureCommand(wrappedCommand, method);
if (wrappedCommand.Timeout > 0)
{
wrappedCommand = new TimeoutCommand(wrappedCommand, wrappedCommand.Timeout, method);
}
yield return(wrappedCommand);
}
}
public void TestGetErrorPercentage()
{
string key = "cmd-metrics-A";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 1);
HystrixCommandMetrics metrics = cmd1._metrics;
cmd1.Execute();
Time.Wait(200);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(0, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd2 = new FailureCommand(key, 1);
cmd2.Execute();
Time.Wait(200);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(50, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd3 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1);
cmd3.Execute();
cmd4.Execute();
Time.Wait(200);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(25, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd5 = new TimeoutCommand(key);
HystrixCommand <bool> cmd6 = new TimeoutCommand(key);
cmd5.Execute();
cmd6.Execute();
Time.Wait(200);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(50, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd8 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd9 = new SuccessCommand(key, 1);
cmd7.Execute();
cmd8.Execute();
cmd9.Execute();
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
// latent
HystrixCommand <bool> cmd10 = new SuccessCommand(key, 60);
cmd10.Execute();
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
// 6 success + 1 latent success + 1 failure + 2 timeout = 10 total
// latent success not considered error
// error percentage = 1 failure + 2 timeout / 10
Time.Wait(200);
Assert.Equal(30, metrics.Healthcounts.ErrorPercentage);
}
public async Task TestCircuitClosedAfterSuccess()
{
string key = "cmd-G";
int sleepWindow = 100;
HystrixCommand <bool> cmd1 = new FailureCommand(key, 1, sleepWindow);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
_ = await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new FailureCommand(key, 1, sleepWindow);
_ = await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 1, sleepWindow);
_ = await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new TimeoutCommand(key, sleepWindow);
_ = await cmd4.ExecuteAsync();
// everything has failed in the test window so we should return false now
Time.Wait(sleepWindow);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("CircuitBreaker state 1 : " + cmd1.Metrics.Healthcounts);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
// wait for sleepWindow to pass
Time.Wait(sleepWindow + 50);
// but the circuit should still be open
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand <bool> cmd5 = new SuccessCommand(key, 10, sleepWindow);
IObservable <bool> asyncResult = cmd5.Observe();
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
await asyncResult.SingleAsync();
// all requests should be open again
Time.Wait(sleepWindow + 50);
output.WriteLine("CircuitBreaker state 2 : " + cmd1.Metrics.Healthcounts);
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (1)!");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (2)!");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (3)!");
// and the circuit should be closed again
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
}
public void OnTimedout(TimeoutCommand timeout)
{
if (timeout.whatToDo == ActionOnTimeout.ExecuteLater)
{
stepInd = timeout.executeLaterStep;
}
else
{
stepInd = -1;
}
}
public void TestFinshedOnTimePassedResult()
{
Mock <ITestCommand> testCommand = new Mock <ITestCommand>();
testCommand.Setup(tc => tc.Execute(null))
.Returns(new PassedResult(GetMethodInfo(), null));
TimeoutCommand command = new TimeoutCommand(testCommand.Object, 10000, GetMethodInfo());
MethodResult result = command.Execute(null);
Assert.IsType <PassedResult>(result);
}
public void TestTookTooLongFailedResult()
{
Mock<ITestCommand> testCommand = new Mock<ITestCommand>();
testCommand
.Setup(tc => tc.Execute(null))
.Callback<object>(_ => Thread.Sleep(500));
TimeoutCommand command = new TimeoutCommand(testCommand.Object, 20, GetMethodInfo());
MethodResult result = command.Execute(null);
FailedResult failedResult = Assert.IsType<FailedResult>(result);
Assert.Equal("Test execution time exceeded: 20ms", failedResult.Message);
}
public async Task <GameState> Handle(Guid tableId, TimeoutCommand command)
{
Task <GameState> loadStateTask = _reader.LoadTableState(tableId);
await loadStateTask;
GameState currentState = loadStateTask.Result;
GameState newState = CommandsHandler.Handle(currentState, command);
await _writer.WriteGameState(newState);
return(newState);
}
public static GameState Handle(GameState state, TimeoutCommand command)
{
GameState newState = state;
if (state.Players.Count > 1)
{
newState = state.Timeout();
}
Log.Info("TimeoutCommand: state: " + state + " command: " + command + " newState: " + newState);
return(newState);
}
public async Task TestTripCircuitOnTimeoutsAboveThreshold()
{
var key = "cmd-E";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0);
var cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd3 = new TimeoutCommand(key);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd5 = new TimeoutCommand(key);
HystrixCommand <bool> cmd6 = new TimeoutCommand(key);
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd8 = new TimeoutCommand(key);
HystrixCommand <bool> cmd9 = new TimeoutCommand(key);
var taskList = new List <Task>
{
cmd1.ExecuteAsync(),
cmd2.ExecuteAsync(),
cmd3.ExecuteAsync(),
cmd4.ExecuteAsync(),
cmd5.ExecuteAsync(),
cmd6.ExecuteAsync(),
cmd7.ExecuteAsync(),
cmd8.ExecuteAsync(),
cmd9.ExecuteAsync(),
};
await Task.WhenAll(taskList);
// Allow window to pass, this should trip the circuit as the error percentage is above the threshold
// Time.Wait(200);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public void TestTripCircuitOnTimeoutsAboveThreshold()
{
string key = "cmd-E";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// success with high latency
cmd1.Execute();
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 1);
cmd2.Execute();
HystrixCommand <bool> cmd3 = new TimeoutCommand(key);
cmd3.Execute();
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1);
cmd4.Execute();
HystrixCommand <bool> cmd5 = new TimeoutCommand(key);
cmd5.Execute();
HystrixCommand <bool> cmd6 = new TimeoutCommand(key);
cmd6.Execute();
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 1);
cmd7.Execute();
HystrixCommand <bool> cmd8 = new TimeoutCommand(key);
cmd8.Execute();
HystrixCommand <bool> cmd9 = new TimeoutCommand(key);
cmd9.Execute();
// this should trip the circuit as the error percentage is above the threshold
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
public void TestTripCircuitOnTimeoutsAboveThreshold()
{
String key = "cmd-E";
try
{
HystrixCommand <Boolean> cmd1 = new SuccessCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
// success with high latency
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new SuccessCommand(key, 1);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new TimeoutCommand(key);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new SuccessCommand(key, 1);
cmd4.Execute();
HystrixCommand <Boolean> cmd5 = new TimeoutCommand(key);
cmd5.Execute();
HystrixCommand <Boolean> cmd6 = new TimeoutCommand(key);
cmd6.Execute();
HystrixCommand <Boolean> cmd7 = new SuccessCommand(key, 1);
cmd7.Execute();
HystrixCommand <Boolean> cmd8 = new TimeoutCommand(key);
cmd8.Execute();
HystrixCommand <Boolean> cmd9 = new TimeoutCommand(key);
cmd9.Execute();
// this should trip the circuit as the error percentage is above the threshold
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
public async Task TestTripCircuitOnTimeoutsAboveThreshold()
{
string key = "cmd-E";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 50);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
// success with high latency
HystrixCommand <bool> cmd2 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd3 = new TimeoutCommand(key);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd5 = new TimeoutCommand(key);
HystrixCommand <bool> cmd6 = new TimeoutCommand(key);
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 1);
HystrixCommand <bool> cmd8 = new TimeoutCommand(key);
HystrixCommand <bool> cmd9 = new TimeoutCommand(key);
var taskList = new List <Task>
{
cmd1.ExecuteAsync(),
cmd2.ExecuteAsync(),
cmd3.ExecuteAsync(),
cmd4.ExecuteAsync(),
cmd5.ExecuteAsync(),
cmd6.ExecuteAsync(),
cmd7.ExecuteAsync(),
cmd8.ExecuteAsync(),
cmd9.ExecuteAsync(),
};
await Task.WhenAll(taskList);
// this should trip the circuit as the error percentage is above the threshold
Time.Wait(200);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
}
public async Task TestMultipleTimeWindowRetriesBeforeClosingCircuit()
{
var key = "cmd-H";
var sleepWindow = 400;
HystrixCommand <bool> cmd1 = new FailureCommand(key, 0);
var cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed when expected!");
Assert.False(cb.IsOpen, Time.CurrentTimeMillis + " Circuit breaker is open when it should be closed!");
_ = await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new FailureCommand(key, 0);
_ = await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 0);
_ = await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new TimeoutCommand(key);
_ = await cmd4.ExecuteAsync();
// everything has failed in the test window so we should return false now
// Allow window to pass,
// Time.Wait(200);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine(Time.CurrentTimeMillis + " !!!! 1 4 failures, circuit will open on recalc");
// Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, Time.CurrentTimeMillis + " Circuit is closed when it should be open!");
// wait for sleepWindow to pass
output.WriteLine(Time.CurrentTimeMillis + " !!!! 2 Sleep window starting where all commands fail-fast");
Time.Wait(sleepWindow + 50);
output.WriteLine(Time.CurrentTimeMillis + " !!!! 3 Sleep window over, should allow singleTest()");
// but the circuit should still be open
Assert.True(cb.IsOpen, Time.CurrentTimeMillis + " Circuit is closed when it should be open!");
// we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open
HystrixCommand <bool> cmd5 = new FailureCommand(key, 50);
var asyncResult5 = cmd5.Observe();
output.WriteLine(Time.CurrentTimeMillis + " !!!! Kicked off the single-test");
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed when NOT expected!");
output.WriteLine(Time.CurrentTimeMillis + " !!!! Confirmed that no other requests go out during single-test");
await asyncResult5.SingleAsync();
output.WriteLine(Time.CurrentTimeMillis + " !!!! SingleTest just completed");
// all requests should still be blocked, because the singleTest failed
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (1) when NOT expected!");
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (2) when NOT expected!");
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (3) when NOT expected!");
// wait for sleepWindow to pass
output.WriteLine(Time.CurrentTimeMillis + " !!!! 2nd sleep window START");
Time.Wait(sleepWindow + 50);
output.WriteLine(Time.CurrentTimeMillis + " !!!! 2nd sleep window over");
// we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open
HystrixCommand <bool> cmd6 = new FailureCommand(key, 50);
var asyncResult6 = cmd6.Observe();
output.WriteLine(Time.CurrentTimeMillis + " 2nd singleTest just kicked off");
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed when NOT expected!");
Assert.False(await asyncResult6.SingleAsync());
output.WriteLine(Time.CurrentTimeMillis + " 2nd singleTest now over");
// all requests should still be blocked, because the singleTest failed
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (1) when NOT expected!");
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (2) when NOT expected!");
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (3) when NOT expected!");
// wait for sleepWindow to pass
Time.Wait(sleepWindow);
// but the circuit should still be open
Assert.True(cb.IsOpen, Time.CurrentTimeMillis + " Circuit is closed when it should be open!");
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 50);
var asyncResult7 = cmd7.Observe();
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed when NOT expected!");
await asyncResult7.SingleAsync();
// all requests should be open again
Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed (1) when expected!");
Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed (2) when expected!");
Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed (3) when expected!");
// and the circuit should be closed again
Assert.False(cb.IsOpen, Time.CurrentTimeMillis + " Circuit breaker is open when it should be closed!");
// and the circuit should be closed again
Assert.False(cb.IsOpen, Time.CurrentTimeMillis + " Circuit breaker is open when it should be closed!");
}
public async Task TestCircuitClosedAfterSuccess()
{
var key = "cmd-G";
var sleepWindow = 400;
HystrixCommand <bool> cmd1 = new FailureCommand(key, 0, sleepWindow);
var cb = (HystrixCircuitBreakerImpl)cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
_ = await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new FailureCommand(key, 0, sleepWindow);
_ = await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 0, sleepWindow);
_ = await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new TimeoutCommand(key, sleepWindow);
_ = await cmd4.ExecuteAsync();
// Allow window to pass, everything has failed in the test window so we should return false now
// Time.Wait(200);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine("ReqLog : " + Time.CurrentTimeMillis + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("CircuitBreaker state 1 : " + Time.CurrentTimeMillis + cmd1.Metrics.Healthcounts);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
// wait for sleepWindow to pass
Time.Wait(sleepWindow + 100);
// but the circuit should still be open
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand <bool> cmd5 = new SuccessCommand(key, 10, sleepWindow);
output.WriteLine("Starting test cmd : " + Time.CurrentTimeMillis + cmd1.Metrics.Healthcounts);
_ = await cmd5.Observe();
// Allow window to pass, all requests should be open again
// Time.Wait(200);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine("CircuitBreaker state 2 : " + Time.CurrentTimeMillis + cmd1.Metrics.Healthcounts);
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (1)!");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (2)!");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (3)!");
// and the circuit should be closed again
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
}
public async void TestMultipleTimeWindowRetriesBeforeClosingCircuit()
{
String key = "cmd-H";
try
{
int sleepWindow = 200;
HystrixCommand <Boolean> cmd1 = new FailureCommand(key, 60);
IHystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen);
cmd1.Execute();
HystrixCommand <Boolean> cmd2 = new FailureCommand(key, 1);
cmd2.Execute();
HystrixCommand <Boolean> cmd3 = new FailureCommand(key, 1);
cmd3.Execute();
HystrixCommand <Boolean> cmd4 = new TimeoutCommand(key);
cmd4.Execute();
// everything has failed in the test window so we should return false now
output.WriteLine("!!!! 1 4 failures, circuit will open on recalc");
Time.Wait(150);
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen);
// wait for sleepWindow to pass
output.WriteLine("!!!! 2 Sleep window starting where all commands fail-fast");
Time.Wait(sleepWindow + 50);
output.WriteLine("!!!! 3 Sleep window over, should allow singleTest()");
// but the circuit should still be open
Assert.True(cb.IsOpen);
// we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open
HystrixCommand <bool> cmd5 = new FailureCommand(key, 60);
IObservable <bool> asyncResult5 = cmd5.Observe();
output.WriteLine("!!!! Kicked off the single-test");
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest);
output.WriteLine("!!!! Confirmed that no other requests go out during single-test");
await asyncResult5.SingleAsync();
output.WriteLine("!!!! SingleTest just completed");
// all requests should still be blocked, because the singleTest failed
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
// wait for sleepWindow to pass
output.WriteLine("!!!! 2nd sleep window START");
Time.Wait(sleepWindow + 50);
output.WriteLine("!!!! 2nd sleep window over");
// we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open
HystrixCommand <bool> cmd6 = new FailureCommand(key, 60);
IObservable <bool> asyncResult6 = cmd6.Observe();
output.WriteLine("2nd singleTest just kicked off");
//and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest);
output.WriteLine("confirmed that 2nd singletest only happened once");
await asyncResult6.SingleAsync();
output.WriteLine("2nd singleTest now over");
// all requests should still be blocked, because the singleTest failed
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
// wait for sleepWindow to pass
Time.Wait(sleepWindow + 50);
// but the circuit should still be open
Assert.True(cb.IsOpen);
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 60);
IObservable <bool> asyncResult7 = cmd7.Observe();
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest);
await asyncResult7.SingleAsync();
// all requests should be open again
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen);
// and the circuit should be closed again
Assert.False(cb.IsOpen);
}
catch (Exception e)
{
output.WriteLine(e.ToString());
Assert.False(true, "Error occurred: " + e.Message);
}
}
/// <summary>
/// Creates a test command for use in running this test.
/// </summary>
/// <returns>A TestCommand</returns>
private TestCommand MakeTestCommand()
{
if (Test.RunState == RunState.Runnable ||
Test.RunState == RunState.Explicit && Filter.IsExplicitMatch(Test))
{
// Command to execute test
TestCommand command = new TestMethodCommand(_testMethod);
var method = _testMethod.Method;
// Add any wrappers to the TestMethodCommand
foreach (IWrapTestMethod wrapper in method.GetCustomAttributes <IWrapTestMethod>(true))
{
command = wrapper.Wrap(command);
}
// Create TestActionCommands using attributes of the method
foreach (ITestAction action in Test.Actions)
{
if (action.Targets == ActionTargets.Default || action.Targets.HasFlag(ActionTargets.Test))
{
command = new TestActionCommand(command, action);
}
}
;
// Try to locate the parent fixture. In current implementations, the test method
// is either one or two levels levels below the TestFixture - if this changes,
// so should the following code.
TestFixture parentFixture = Test.Parent as TestFixture ?? Test.Parent?.Parent as TestFixture;
// In normal operation we should always get the methods from the parent fixture.
// However, some of NUnit's own tests can create a TestMethod without a parent
// fixture. Most likely, we should stop doing this, but it affects 100s of cases.
var setUpMethods = parentFixture?.SetUpMethods ?? Reflect.GetMethodsWithAttribute(Test.TypeInfo.Type, typeof(SetUpAttribute), true);
var tearDownMethods = parentFixture?.TearDownMethods ?? Reflect.GetMethodsWithAttribute(Test.TypeInfo.Type, typeof(TearDownAttribute), true);
// Wrap in SetUpTearDownCommands
var setUpTearDownList = BuildSetUpTearDownList(setUpMethods, tearDownMethods);
foreach (var item in setUpTearDownList)
{
command = new SetUpTearDownCommand(command, item);
}
// In the current implementation, upstream actions only apply to tests. If that should change in the future,
// then actions would have to be tested for here. For now we simply assert it in Debug. We allow
// ActionTargets.Default, because it is passed down by ParameterizedMethodSuite.
int index = Context.UpstreamActions.Count;
while (--index >= 0)
{
ITestAction action = Context.UpstreamActions[index];
System.Diagnostics.Debug.Assert(
action.Targets == ActionTargets.Default || action.Targets.HasFlag(ActionTargets.Test),
"Invalid target on upstream action: " + action.Targets.ToString());
command = new TestActionCommand(command, action);
}
// Add wrappers that apply before setup and after teardown
foreach (ICommandWrapper decorator in method.GetCustomAttributes <IWrapSetUpTearDown>(true))
{
command = decorator.Wrap(command);
}
// Add command to set up context using attributes that implement IApplyToContext
foreach (var attr in method.GetCustomAttributes <IApplyToContext>(true))
{
command = new ApplyChangesToContextCommand(command, attr);
}
// If a timeout is specified, create a TimeoutCommand
#if !NETSTANDARD1_6
// Timeout set at a higher level
int timeout = Context.TestCaseTimeout;
// Timeout set on this test
if (Test.Properties.ContainsKey(PropertyNames.Timeout))
{
timeout = (int)Test.Properties.Get(PropertyNames.Timeout);
}
if (timeout > 0)
{
command = new TimeoutCommand(command, timeout);
}
#endif
return(command);
}
else
{
return(new SkipCommand(_testMethod));
}
}
public void TestGetErrorPercentage()
{
string key = "cmd-metrics-A";
HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0);
HystrixCommandMetrics metrics = cmd1._metrics;
Assert.True(WaitForHealthCountToUpdate(key, 1000), "Health count stream took to long");
cmd1.Execute();
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(0, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd2 = new FailureCommand(key, 0);
cmd2.Execute();
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(50, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd3 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0);
cmd3.Execute();
cmd4.Execute();
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(25, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd5 = new TimeoutCommand(key);
HystrixCommand <bool> cmd6 = new TimeoutCommand(key);
cmd5.Execute();
cmd6.Execute();
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(50, metrics.Healthcounts.ErrorPercentage);
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd8 = new SuccessCommand(key, 0);
HystrixCommand <bool> cmd9 = new SuccessCommand(key, 0);
cmd7.Execute();
cmd8.Execute();
cmd9.Execute();
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
// latent
HystrixCommand <bool> cmd10 = new SuccessCommand(key, 60);
cmd10.Execute();
output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
// 6 success + 1 latent success + 1 failure + 2 timeout = 10 total
// latent success not considered error
// error percentage = 1 failure + 2 timeout / 10
Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long");
Assert.Equal(30, metrics.Healthcounts.ErrorPercentage);
}
/// <summary> Copy Constructor</summary>
/// <param name="other"></param>
public TimeoutCommand(TimeoutCommand other)
: base(other)
{
}
