public async void TestCircuitClosedAfterSuccess() { String key = "cmd-G"; try { int sleepWindow = 100; HystrixCommand <Boolean> cmd1 = new FailureCommand(key, 1, sleepWindow); IHystrixCircuitBreaker cb = cmd1.circuitBreaker; // this should start as allowing requests Assert.True(cb.AllowRequest); Assert.False(cb.IsOpen); cmd1.Execute(); HystrixCommand <Boolean> cmd2 = new FailureCommand(key, 1, sleepWindow); cmd2.Execute(); HystrixCommand <Boolean> cmd3 = new FailureCommand(key, 1, sleepWindow); cmd3.Execute(); HystrixCommand <Boolean> cmd4 = new TimeoutCommand(key, sleepWindow); cmd4.Execute(); // everything has failed in the test window so we should return false now Time.Wait(150); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); output.WriteLine("CircuitBreaker state 1 : " + cmd1.Metrics.Healthcounts); Assert.False(cb.AllowRequest); Assert.True(cb.IsOpen); // wait for sleepWindow to pass Time.Wait(sleepWindow + 50); // but the circuit should still be open Assert.True(cb.IsOpen); // we should now allow 1 request, and upon success, should cause the circuit to be closed HystrixCommand <bool> cmd5 = new SuccessCommand(key, 60, sleepWindow); IObservable <bool> asyncResult = cmd5.Observe(); // and further requests are still blocked while the singleTest command is in flight Assert.False(cb.AllowRequest); await asyncResult.SingleAsync(); // all requests should be open again Time.Wait(150); output.WriteLine("CircuitBreaker state 2 : " + cmd1.Metrics.Healthcounts); Assert.True(cb.AllowRequest); Assert.True(cb.AllowRequest); Assert.True(cb.AllowRequest); // and the circuit should be closed again Assert.False(cb.IsOpen); } catch (Exception e) { output.WriteLine(e.ToString()); Assert.False(true, "Error occurred: " + e.Message); } }
public async Task TestCircuitClosedAfterSuccess() { string key = "cmd-G"; int sleepWindow = 100; HystrixCommand <bool> cmd1 = new FailureCommand(key, 1, sleepWindow); IHystrixCircuitBreaker cb = cmd1._circuitBreaker; // this should start as allowing requests Assert.True(cb.AllowRequest, "Request NOT allowed when expected!"); Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!"); _ = await cmd1.ExecuteAsync(); HystrixCommand <bool> cmd2 = new FailureCommand(key, 1, sleepWindow); _ = await cmd2.ExecuteAsync(); HystrixCommand <bool> cmd3 = new FailureCommand(key, 1, sleepWindow); _ = await cmd3.ExecuteAsync(); HystrixCommand <bool> cmd4 = new TimeoutCommand(key, sleepWindow); _ = await cmd4.ExecuteAsync(); // everything has failed in the test window so we should return false now Time.Wait(sleepWindow); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); output.WriteLine("CircuitBreaker state 1 : " + cmd1.Metrics.Healthcounts); Assert.False(cb.AllowRequest, "Request allowed when NOT expected!"); Assert.True(cb.IsOpen, "Circuit is closed when it should be open!"); // wait for sleepWindow to pass Time.Wait(sleepWindow + 50); // but the circuit should still be open Assert.True(cb.IsOpen, "Circuit is closed when it should be open!"); // we should now allow 1 request, and upon success, should cause the circuit to be closed HystrixCommand <bool> cmd5 = new SuccessCommand(key, 10, sleepWindow); IObservable <bool> asyncResult = cmd5.Observe(); // and further requests are still blocked while the singleTest command is in flight Assert.False(cb.AllowRequest, "Request allowed when NOT expected!"); await asyncResult.SingleAsync(); // all requests should be open again Time.Wait(sleepWindow + 50); output.WriteLine("CircuitBreaker state 2 : " + cmd1.Metrics.Healthcounts); Assert.True(cb.AllowRequest, "Request NOT allowed when expected (1)!"); Assert.True(cb.AllowRequest, "Request NOT allowed when expected (2)!"); Assert.True(cb.AllowRequest, "Request NOT allowed when expected (3)!"); // and the circuit should be closed again Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!"); }
public void TestCurrentConcurrentExecutionCount() { String key = "cmd-metrics-C"; HystrixCommandMetrics metrics = null; List <IObservable <bool> > cmdResults = new List <IObservable <bool> >(); int NUM_CMDS = 8; for (int i = 0; i < NUM_CMDS; i++) { HystrixCommand <Boolean> cmd = new SuccessCommand(key, 900); if (metrics == null) { metrics = cmd.metrics; } IObservable <bool> eagerObservable = cmd.Observe(); cmdResults.Add(eagerObservable); } try { Time.Wait(200); } catch (Exception ie) { Assert.True(false, ie.Message); } output.WriteLine("ReqLog: " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(NUM_CMDS, metrics.CurrentConcurrentExecutionCount); CountdownEvent latch = new CountdownEvent(1); Observable.Merge(cmdResults).Subscribe( (n) => { }, (e) => { output.WriteLine("Error duing command execution"); output.WriteLine(e.ToString()); latch.SignalEx(); }, () => { output.WriteLine("All commands done"); latch.SignalEx(); }); latch.Wait(10000); Assert.Equal(0, metrics.CurrentConcurrentExecutionCount); }
public async Task TestMultipleTimeWindowRetriesBeforeClosingCircuit() { var key = "cmd-H"; var sleepWindow = 400; HystrixCommand <bool> cmd1 = new FailureCommand(key, 0); var cb = cmd1._circuitBreaker; var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions); Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start"); // this should start as allowing requests Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed when expected!"); Assert.False(cb.IsOpen, Time.CurrentTimeMillis + " Circuit breaker is open when it should be closed!"); _ = await cmd1.ExecuteAsync(); HystrixCommand <bool> cmd2 = new FailureCommand(key, 0); _ = await cmd2.ExecuteAsync(); HystrixCommand <bool> cmd3 = new FailureCommand(key, 0); _ = await cmd3.ExecuteAsync(); HystrixCommand <bool> cmd4 = new TimeoutCommand(key); _ = await cmd4.ExecuteAsync(); // everything has failed in the test window so we should return false now // Allow window to pass, // Time.Wait(200); Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update"); output.WriteLine(Time.CurrentTimeMillis + " !!!! 1 4 failures, circuit will open on recalc"); // Assert.False(cb.AllowRequest, "Request allowed when NOT expected!"); Assert.True(cb.IsOpen, Time.CurrentTimeMillis + " Circuit is closed when it should be open!"); // wait for sleepWindow to pass output.WriteLine(Time.CurrentTimeMillis + " !!!! 2 Sleep window starting where all commands fail-fast"); Time.Wait(sleepWindow + 50); output.WriteLine(Time.CurrentTimeMillis + " !!!! 3 Sleep window over, should allow singleTest()"); // but the circuit should still be open Assert.True(cb.IsOpen, Time.CurrentTimeMillis + " Circuit is closed when it should be open!"); // we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open HystrixCommand <bool> cmd5 = new FailureCommand(key, 50); var asyncResult5 = cmd5.Observe(); output.WriteLine(Time.CurrentTimeMillis + " !!!! Kicked off the single-test"); // and further requests are still blocked while the singleTest command is in flight Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed when NOT expected!"); output.WriteLine(Time.CurrentTimeMillis + " !!!! Confirmed that no other requests go out during single-test"); await asyncResult5.SingleAsync(); output.WriteLine(Time.CurrentTimeMillis + " !!!! SingleTest just completed"); // all requests should still be blocked, because the singleTest failed Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (1) when NOT expected!"); Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (2) when NOT expected!"); Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (3) when NOT expected!"); // wait for sleepWindow to pass output.WriteLine(Time.CurrentTimeMillis + " !!!! 2nd sleep window START"); Time.Wait(sleepWindow + 50); output.WriteLine(Time.CurrentTimeMillis + " !!!! 2nd sleep window over"); // we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open HystrixCommand <bool> cmd6 = new FailureCommand(key, 50); var asyncResult6 = cmd6.Observe(); output.WriteLine(Time.CurrentTimeMillis + " 2nd singleTest just kicked off"); // and further requests are still blocked while the singleTest command is in flight Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed when NOT expected!"); Assert.False(await asyncResult6.SingleAsync()); output.WriteLine(Time.CurrentTimeMillis + " 2nd singleTest now over"); // all requests should still be blocked, because the singleTest failed Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (1) when NOT expected!"); Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (2) when NOT expected!"); Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (3) when NOT expected!"); // wait for sleepWindow to pass Time.Wait(sleepWindow); // but the circuit should still be open Assert.True(cb.IsOpen, Time.CurrentTimeMillis + " Circuit is closed when it should be open!"); // we should now allow 1 request, and upon success, should cause the circuit to be closed HystrixCommand <bool> cmd7 = new SuccessCommand(key, 50); var asyncResult7 = cmd7.Observe(); // and further requests are still blocked while the singleTest command is in flight Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed when NOT expected!"); await asyncResult7.SingleAsync(); // all requests should be open again Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed (1) when expected!"); Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed (2) when expected!"); Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed (3) when expected!"); // and the circuit should be closed again Assert.False(cb.IsOpen, Time.CurrentTimeMillis + " Circuit breaker is open when it should be closed!"); // and the circuit should be closed again Assert.False(cb.IsOpen, Time.CurrentTimeMillis + " Circuit breaker is open when it should be closed!"); }
public async Task TestCircuitClosedAfterSuccess() { var key = "cmd-G"; var sleepWindow = 400; HystrixCommand <bool> cmd1 = new FailureCommand(key, 0, sleepWindow); var cb = (HystrixCircuitBreakerImpl)cmd1._circuitBreaker; var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions); Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start"); // this should start as allowing requests Assert.True(cb.AllowRequest, "Request NOT allowed when expected!"); Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!"); _ = await cmd1.ExecuteAsync(); HystrixCommand <bool> cmd2 = new FailureCommand(key, 0, sleepWindow); _ = await cmd2.ExecuteAsync(); HystrixCommand <bool> cmd3 = new FailureCommand(key, 0, sleepWindow); _ = await cmd3.ExecuteAsync(); HystrixCommand <bool> cmd4 = new TimeoutCommand(key, sleepWindow); _ = await cmd4.ExecuteAsync(); // Allow window to pass, everything has failed in the test window so we should return false now // Time.Wait(200); Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update"); output.WriteLine("ReqLog : " + Time.CurrentTimeMillis + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); output.WriteLine("CircuitBreaker state 1 : " + Time.CurrentTimeMillis + cmd1.Metrics.Healthcounts); Assert.False(cb.AllowRequest, "Request allowed when NOT expected!"); Assert.True(cb.IsOpen, "Circuit is closed when it should be open!"); // wait for sleepWindow to pass Time.Wait(sleepWindow + 100); // but the circuit should still be open Assert.True(cb.IsOpen, "Circuit is closed when it should be open!"); // we should now allow 1 request, and upon success, should cause the circuit to be closed HystrixCommand <bool> cmd5 = new SuccessCommand(key, 10, sleepWindow); output.WriteLine("Starting test cmd : " + Time.CurrentTimeMillis + cmd1.Metrics.Healthcounts); _ = await cmd5.Observe(); // Allow window to pass, all requests should be open again // Time.Wait(200); Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update"); output.WriteLine("CircuitBreaker state 2 : " + Time.CurrentTimeMillis + cmd1.Metrics.Healthcounts); Assert.True(cb.AllowRequest, "Request NOT allowed when expected (1)!"); Assert.True(cb.AllowRequest, "Request NOT allowed when expected (2)!"); Assert.True(cb.AllowRequest, "Request NOT allowed when expected (3)!"); // and the circuit should be closed again Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!"); }
public async void TestMultipleTimeWindowRetriesBeforeClosingCircuit() { String key = "cmd-H"; try { int sleepWindow = 200; HystrixCommand <Boolean> cmd1 = new FailureCommand(key, 60); IHystrixCircuitBreaker cb = cmd1.circuitBreaker; // this should start as allowing requests Assert.True(cb.AllowRequest); Assert.False(cb.IsOpen); cmd1.Execute(); HystrixCommand <Boolean> cmd2 = new FailureCommand(key, 1); cmd2.Execute(); HystrixCommand <Boolean> cmd3 = new FailureCommand(key, 1); cmd3.Execute(); HystrixCommand <Boolean> cmd4 = new TimeoutCommand(key); cmd4.Execute(); // everything has failed in the test window so we should return false now output.WriteLine("!!!! 1 4 failures, circuit will open on recalc"); Time.Wait(150); Assert.False(cb.AllowRequest); Assert.True(cb.IsOpen); // wait for sleepWindow to pass output.WriteLine("!!!! 2 Sleep window starting where all commands fail-fast"); Time.Wait(sleepWindow + 50); output.WriteLine("!!!! 3 Sleep window over, should allow singleTest()"); // but the circuit should still be open Assert.True(cb.IsOpen); // we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open HystrixCommand <bool> cmd5 = new FailureCommand(key, 60); IObservable <bool> asyncResult5 = cmd5.Observe(); output.WriteLine("!!!! Kicked off the single-test"); // and further requests are still blocked while the singleTest command is in flight Assert.False(cb.AllowRequest); output.WriteLine("!!!! Confirmed that no other requests go out during single-test"); await asyncResult5.SingleAsync(); output.WriteLine("!!!! SingleTest just completed"); // all requests should still be blocked, because the singleTest failed Assert.False(cb.AllowRequest); Assert.False(cb.AllowRequest); Assert.False(cb.AllowRequest); // wait for sleepWindow to pass output.WriteLine("!!!! 2nd sleep window START"); Time.Wait(sleepWindow + 50); output.WriteLine("!!!! 2nd sleep window over"); // we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open HystrixCommand <bool> cmd6 = new FailureCommand(key, 60); IObservable <bool> asyncResult6 = cmd6.Observe(); output.WriteLine("2nd singleTest just kicked off"); //and further requests are still blocked while the singleTest command is in flight Assert.False(cb.AllowRequest); output.WriteLine("confirmed that 2nd singletest only happened once"); await asyncResult6.SingleAsync(); output.WriteLine("2nd singleTest now over"); // all requests should still be blocked, because the singleTest failed Assert.False(cb.AllowRequest); Assert.False(cb.AllowRequest); Assert.False(cb.AllowRequest); // wait for sleepWindow to pass Time.Wait(sleepWindow + 50); // but the circuit should still be open Assert.True(cb.IsOpen); // we should now allow 1 request, and upon success, should cause the circuit to be closed HystrixCommand <bool> cmd7 = new SuccessCommand(key, 60); IObservable <bool> asyncResult7 = cmd7.Observe(); // and further requests are still blocked while the singleTest command is in flight Assert.False(cb.AllowRequest); await asyncResult7.SingleAsync(); // all requests should be open again Assert.True(cb.AllowRequest); Assert.True(cb.AllowRequest); Assert.True(cb.AllowRequest); // and the circuit should be closed again Assert.False(cb.IsOpen); // and the circuit should be closed again Assert.False(cb.IsOpen); } catch (Exception e) { output.WriteLine(e.ToString()); Assert.False(true, "Error occurred: " + e.Message); } }