public void TestFallbackMissing() { IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-K"); stream = HealthCountsStream.GetInstance(key, 10, 100); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); CommandStreamTest.Command cmd = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_MISSING); cmd.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(1L, stream.Latest.ErrorCount); Assert.Equal(1L, stream.Latest.TotalRequests); }
public void TestRequestFromCache() { IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-F"); stream = HealthCountsStream.GetInstance(key, 10, 100); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20); Command cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE); Command cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE); cmd1.Observe(); cmd2.Observe(); cmd3.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(0L, stream.Latest.ErrorCount); Assert.Equal(1L, stream.Latest.TotalRequests); // responses from cache should not show up here }
public override void Dispose() { base.Dispose(); stream.Unsubscribe(); HealthCountsStream.Reset(); }
public async Task TestRequestFromCache() { var key = HystrixCommandKeyDefault.AsKey("CMD-Health-F"); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = HealthCountsStream.GetInstance(key, 10, 100); var cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0); var cmd2 = Command.From(GroupKey, key, HystrixEventType.RESPONSE_FROM_CACHE); var cmd3 = Command.From(GroupKey, key, HystrixEventType.RESPONSE_FROM_CACHE); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); await cmd1.Observe(); await cmd2.Observe(); await cmd3.Observe(); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); Assert.Equal(0L, stream.Latest.ErrorCount); Assert.Equal(1L, stream.Latest.TotalRequests); // responses from cache should not show up here }
public async Task TestMultipleEventsOverTimeGetStoredAndAgeOut() { var key = HystrixCommandKeyDefault.AsKey("CMD-Health-M"); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = HealthCountsStream.GetInstance(key, 10, 100); var cmd1 = CommandStreamTest.Command.From(GroupKey, key, HystrixEventType.SUCCESS, 20); var cmd2 = CommandStreamTest.Command.From(GroupKey, key, HystrixEventType.FAILURE, 10); // by doing a take(30), we ensure that all rolling counts go back to 0 latchSubscription = stream.Observe().Take(30 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); await cmd1.Observe(); await cmd2.Observe(); Assert.True(latch.Wait(10000), "CountdownEvent was not set!"); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(0L, stream.Latest.ErrorCount); Assert.Equal(0L, stream.Latest.TotalRequests); }
public void TestMultipleEventsOverTimeGetStoredAndAgeOut() { IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-M"); stream = HealthCountsStream.GetInstance(key, 10, 100); // by doing a take(30), we ensure that all rolling counts go back to 0 CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(30).Subscribe(new LatchedObserver(output, latch)); CommandStreamTest.Command cmd1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 20); CommandStreamTest.Command cmd2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 10); cmd1.Observe(); cmd2.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(0L, stream.Latest.ErrorCount); Assert.Equal(0L, stream.Latest.TotalRequests); }
public async Task TestLowVolumeDoesNotTripCircuit() { var key = "cmd-I"; var sleepWindow = 400; var lowVolume = 5; HystrixCommand <bool> cmd1 = new FailureCommand(key, 0, sleepWindow, lowVolume); var cb = cmd1._circuitBreaker; var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions); Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start"); // this should start as allowing requests Assert.True(cb.AllowRequest, "Request NOT allowed when expected!"); Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!"); await cmd1.ExecuteAsync(); HystrixCommand <bool> cmd2 = new FailureCommand(key, 0, sleepWindow, lowVolume); await cmd2.ExecuteAsync(); HystrixCommand <bool> cmd3 = new FailureCommand(key, 0, sleepWindow, lowVolume); await cmd3.ExecuteAsync(); HystrixCommand <bool> cmd4 = new FailureCommand(key, 0, sleepWindow, lowVolume); await cmd4.ExecuteAsync(); // Allow window to pass, even though it has all failed we won't trip the circuit because the volume is low // Time.Wait(200); Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update"); Assert.True(cb.AllowRequest, "Request NOT allowed when expected!"); Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!"); }
public async Task TestTripCircuitOnTimeouts() { var key = "cmd-D"; HystrixCommand <bool> cmd1 = new TimeoutCommand(key); var cb = cmd1._circuitBreaker; var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions); Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start"); // this should start as allowing requests Assert.True(cb.AllowRequest, "Request NOT allowed when expected!"); Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!"); // success with high latency await cmd1.ExecuteAsync(); HystrixCommand <bool> cmd2 = new TimeoutCommand(key); await cmd2.ExecuteAsync(); HystrixCommand <bool> cmd3 = new TimeoutCommand(key); await cmd3.ExecuteAsync(); HystrixCommand <bool> cmd4 = new TimeoutCommand(key); await cmd4.ExecuteAsync(); // Allow window to pass, everything has been a timeout so we should not allow any requests // Time.Wait(125); Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update"); Assert.False(cb.AllowRequest, "Request allowed when NOT expected!"); Assert.True(cb.IsOpen, "Circuit is closed when it should be open!"); }
public void TestSharedSourceStream() { var key = HystrixCommandKeyDefault.AsKey("CMD-Health-N"); stream = HealthCountsStream.GetInstance(key, 10, 100); var latch = new CountdownEvent(1); var allEqual = new AtomicBoolean(false); var o1 = stream .Observe() .Take(10) .ObserveOn(TaskPoolScheduler.Default); var o2 = stream .Observe() .Take(10) .ObserveOn(TaskPoolScheduler.Default); var zipped = Observable.Zip(o1, o2, (healthCounts, healthCounts2) => { return(healthCounts == healthCounts2); // we want object equality }); var reduced = zipped.Aggregate(true, (a, b) => { return(a && b); }).Select(n => n); var rdisp = reduced.Subscribe( (b) => { output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " Reduced OnNext : " + b); allEqual.Value = b; }, (e) => { output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " Reduced OnError : " + e); output.WriteLine(e.ToString()); latch.SignalEx(); }, () => { output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " Reduced OnCompleted"); latch.SignalEx(); }); for (var i = 0; i < 10; i++) { HystrixCommand <int> cmd = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 20); cmd.Execute(); } Assert.True(latch.Wait(10000), "CountdownEvent was not set!"); Assert.True(allEqual.Value); rdisp.Dispose(); // we should be getting the same object from both streams. this ensures that multiple subscribers don't induce extra work }
public override void Dispose() { latchSubscription?.Dispose(); stream?.Unsubscribe(); latchSubscription = null; stream = null; base.Dispose(); }
/* package */ internal void ResetStream() { lock (_syncLock) { healthCountsStream.Unsubscribe(); HealthCountsStream.RemoveByKey(key); healthCountsStream = HealthCountsStream.GetInstance(key, properties); } }
public void TestSemaphoreRejected() { IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-H"); stream = HealthCountsStream.GetInstance(key, 10, 100); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); // 10 commands will saturate semaphore when called from different threads. // submit 2 more requests and they should be SEMAPHORE_REJECTED // should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes List <Command> saturators = new List <Command>(); for (int i = 0; i < 10; i++) { saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 400, ExecutionIsolationStrategy.SEMAPHORE)); } Command rejected1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE); Command rejected2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE); foreach (Command c in saturators) { new Thread(new ThreadStart(() => c.Observe())).Start(); } try { Time.Wait(100); } catch (Exception ie) { Assert.True(false, ie.Message); } rejected1.Observe(); rejected2.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.True(rejected1.IsResponseSemaphoreRejected); Assert.True(rejected2.IsResponseSemaphoreRejected); // should only see failures here, not SHORT-CIRCUITS Assert.Equal(2L, stream.Latest.ErrorCount); Assert.Equal(12L, stream.Latest.TotalRequests); }
/* package */ internal void ResetStream() { lock (_syncLock) { _healthCountsStream.Unsubscribe(); HealthCountsStream.RemoveByKey(CommandKey); _healthCountsStream = HealthCountsStream.GetInstance(CommandKey, Properties); } }
public void TestThreadPoolRejected() { IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-I"); stream = HealthCountsStream.GetInstance(key, 10, 100); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); // 10 commands will saturate threadpools when called concurrently. // submit 2 more requests and they should be THREADPOOL_REJECTED // should see 10 SUCCESSes, 2 THREADPOOL_REJECTED and 2 FALLBACK_SUCCESSes List <CommandStreamTest.Command> saturators = new List <CommandStreamTest.Command>(); for (int i = 0; i < 10; i++) { saturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 400)); } CommandStreamTest.Command rejected1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0); CommandStreamTest.Command rejected2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0); foreach (CommandStreamTest.Command saturator in saturators) { saturator.Observe(); } try { Time.Wait(100); } catch (Exception ie) { Assert.True(false, ie.Message); } rejected1.Observe(); rejected2.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.False(true, "Interrupted ex"); } output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.True(rejected1.IsResponseThreadPoolRejected); Assert.True(rejected2.IsResponseThreadPoolRejected); Assert.Equal(2L, stream.Latest.ErrorCount); Assert.Equal(12L, stream.Latest.TotalRequests); }
public void TestFallbackRejection() { IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-L"); stream = HealthCountsStream.GetInstance(key, 10, 100); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); //fallback semaphore size is 5. So let 5 commands saturate that semaphore, then //let 2 more commands go to fallback. they should get rejected by the fallback-semaphore List <CommandStreamTest.Command> fallbackSaturators = new List <CommandStreamTest.Command>(); for (int i = 0; i < 5; i++) { fallbackSaturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 400)); } CommandStreamTest.Command rejection1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0); CommandStreamTest.Command rejection2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0); foreach (CommandStreamTest.Command saturator in fallbackSaturators) { saturator.Observe(); } try { Time.Wait(70); } catch (Exception ex) { Assert.True(false, ex.Message); } rejection1.Observe(); rejection2.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(7L, stream.Latest.ErrorCount); Assert.Equal(7L, stream.Latest.TotalRequests); }
public virtual bool WaitForHealthCountToUpdate(string commandKey, int numberOfUpdates, int maxTimeToWait, ITestOutputHelper output = null) { var stream = HealthCountsStream.GetInstance(commandKey); if (stream == null) { return(false); } return(WaitForObservableToUpdate(stream.Observe(), numberOfUpdates, maxTimeToWait, output)); }
public async Task TestTripCircuitOnFailuresAboveThreshold() { var key = "cmd-B"; HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0); var cb = cmd1._circuitBreaker; var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions); Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start"); // this should start as allowing requests Assert.True(cb.AllowRequest, "Request NOT allowed when expected!"); Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!"); // success with high latency _ = await cmd1.ExecuteAsync(); HystrixCommand <bool> cmd2 = new SuccessCommand(key, 0); _ = await cmd2.ExecuteAsync(); HystrixCommand <bool> cmd3 = new FailureCommand(key, 0); _ = await cmd3.ExecuteAsync(); HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0); _ = await cmd4.ExecuteAsync(); HystrixCommand <bool> cmd5 = new FailureCommand(key, 0); _ = await cmd5.ExecuteAsync(); HystrixCommand <bool> cmd6 = new SuccessCommand(key, 0); _ = await cmd6.ExecuteAsync(); HystrixCommand <bool> cmd7 = new FailureCommand(key, 0); _ = await cmd7.ExecuteAsync(); HystrixCommand <bool> cmd8 = new FailureCommand(key, 0); _ = await cmd8.ExecuteAsync(); // Let window pass, this should trip the circuit as the error percentage is above the threshold // Time.Wait(125); Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update"); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); output.WriteLine("Current CircuitBreaker Status : " + cmd1.Metrics.Healthcounts); Assert.False(cb.AllowRequest, "Request allowed when NOT expected!"); Assert.True(cb.IsOpen, "Circuit is closed when it should be open!"); }
public void TestShortCircuited() { IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-G"); stream = HealthCountsStream.GetInstance(key, 10, 100); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); // 3 failures in a row will trip circuit. let bucket roll once then submit 2 requests. // should see 3 FAILUREs and 2 SHORT_CIRCUITs and then 5 FALLBACK_SUCCESSes Command failure1 = Command.From(groupKey, key, HystrixEventType.FAILURE, 20); Command failure2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 20); Command failure3 = Command.From(groupKey, key, HystrixEventType.FAILURE, 20); Command shortCircuit1 = Command.From(groupKey, key, HystrixEventType.SUCCESS); Command shortCircuit2 = Command.From(groupKey, key, HystrixEventType.SUCCESS); failure1.Observe(); failure2.Observe(); failure3.Observe(); try { Time.Wait(500); } catch (Exception ie) { Assert.False(true, ie.Message); } shortCircuit1.Observe(); shortCircuit2.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } Assert.True(shortCircuit1.IsResponseShortCircuited); Assert.True(shortCircuit2.IsResponseShortCircuited); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); // should only see failures here, not SHORT-CIRCUITS Assert.Equal(3L, stream.Latest.ErrorCount); Assert.Equal(3L, stream.Latest.TotalRequests); }
public void TestEmptyStreamProducesZeros() { var key = HystrixCommandKeyDefault.AsKey("CMD-Health-A"); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = HealthCountsStream.GetInstance(key, 10, 100); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); Assert.Equal(0L, stream.Latest.ErrorCount); Assert.Equal(0L, stream.Latest.TotalRequests); }
HystrixCommandMetrics(IHystrixCommandKey key, IHystrixCommandGroupKey commandGroup, IHystrixThreadPoolKey threadPoolKey, IHystrixCommandOptions properties, HystrixEventNotifier eventNotifier) : base(null) { this.key = key; this.group = commandGroup; this.threadPoolKey = threadPoolKey; this.properties = properties; healthCountsStream = HealthCountsStream.GetInstance(key, properties); rollingCommandEventCounterStream = RollingCommandEventCounterStream.GetInstance(key, properties); cumulativeCommandEventCounterStream = CumulativeCommandEventCounterStream.GetInstance(key, properties); rollingCommandLatencyDistributionStream = RollingCommandLatencyDistributionStream.GetInstance(key, properties); rollingCommandUserLatencyDistributionStream = RollingCommandUserLatencyDistributionStream.GetInstance(key, properties); rollingCommandMaxConcurrencyStream = RollingCommandMaxConcurrencyStream.GetInstance(key, properties); }
public static void ShutdownThreads() { CumulativeCommandEventCounterStream.Reset(); CumulativeThreadPoolEventCounterStream.Reset(); RollingCommandEventCounterStream.Reset(); RollingThreadPoolEventCounterStream.Reset(); RollingCollapserEventCounterStream.Reset(); RollingCollapserEventCounterStream.Reset(); HealthCountsStream.Reset(); RollingCollapserBatchSizeDistributionStream.Reset(); RollingCommandLatencyDistributionStream.Reset(); RollingCommandUserLatencyDistributionStream.Reset(); RollingCommandMaxConcurrencyStream.Reset(); RollingThreadPoolMaxConcurrencyStream.Reset(); }
public void TestEmptyStreamProducesZeros() { IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-A"); stream = HealthCountsStream.GetInstance(key, 10, 100); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); // no writes Assert.True(latch.Wait(10000), "CountdownEvent was not set!"); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(0L, stream.Latest.ErrorCount); Assert.Equal(0L, stream.Latest.TotalRequests); }
public async Task TestCircuitDoesNotTripOnFailuresBelowThreshold() { string key = "cmd-C"; HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0); ICircuitBreaker cb = cmd1._circuitBreaker; var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions); Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start"); // this should start as allowing requests Assert.True(cb.AllowRequest, "Request NOT allowed when expected!"); Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!"); // success with high latency await cmd1.ExecuteAsync(); HystrixCommand <bool> cmd2 = new SuccessCommand(key, 0); await cmd2.ExecuteAsync(); HystrixCommand <bool> cmd3 = new FailureCommand(key, 0); await cmd3.ExecuteAsync(); HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0); await cmd4.ExecuteAsync(); HystrixCommand <bool> cmd5 = new SuccessCommand(key, 0); await cmd5.ExecuteAsync(); HystrixCommand <bool> cmd6 = new FailureCommand(key, 0); await cmd6.ExecuteAsync(); HystrixCommand <bool> cmd7 = new SuccessCommand(key, 0); await cmd7.ExecuteAsync(); HystrixCommand <bool> cmd8 = new FailureCommand(key, 0); await cmd8.ExecuteAsync(); // Allow window to pass, this should remain closed as the failure threshold is below the percentage limit // Time.Wait(125); Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update"); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); output.WriteLine("Current CircuitBreaker Status : " + cmd1.Metrics.Healthcounts); Assert.True(cb.AllowRequest, "Request NOT allowed when expected!"); Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!"); }
internal static void Reset() { foreach (HystrixCommandMetrics metricsInstance in GetInstances()) { metricsInstance.UnsubscribeAll(); } RollingCommandEventCounterStream.Reset(); CumulativeCommandEventCounterStream.Reset(); RollingCommandLatencyDistributionStream.Reset(); RollingCommandUserLatencyDistributionStream.Reset(); RollingCommandMaxConcurrencyStream.Reset(); HystrixThreadEventStream.Reset(); HealthCountsStream.Reset(); Metrics.Clear(); }
public async Task TestTripCircuitOnTimeoutsAboveThreshold() { var key = "cmd-E"; HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0); var cb = cmd1._circuitBreaker; var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions); Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start"); // this should start as allowing requests Assert.True(cb.AllowRequest, "Request NOT allowed when expected!"); Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!"); // success with high latency HystrixCommand <bool> cmd2 = new SuccessCommand(key, 0); HystrixCommand <bool> cmd3 = new TimeoutCommand(key); HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0); HystrixCommand <bool> cmd5 = new TimeoutCommand(key); HystrixCommand <bool> cmd6 = new TimeoutCommand(key); HystrixCommand <bool> cmd7 = new SuccessCommand(key, 0); HystrixCommand <bool> cmd8 = new TimeoutCommand(key); HystrixCommand <bool> cmd9 = new TimeoutCommand(key); var taskList = new List <Task> { cmd1.ExecuteAsync(), cmd2.ExecuteAsync(), cmd3.ExecuteAsync(), cmd4.ExecuteAsync(), cmd5.ExecuteAsync(), cmd6.ExecuteAsync(), cmd7.ExecuteAsync(), cmd8.ExecuteAsync(), cmd9.ExecuteAsync(), }; await Task.WhenAll(taskList); // Allow window to pass, this should trip the circuit as the error percentage is above the threshold // Time.Wait(200); Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update"); output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.False(cb.AllowRequest, "Request allowed when NOT expected!"); Assert.True(cb.IsOpen, "Circuit is closed when it should be open!"); }
public async Task TestSemaphoreRejected() { var key = HystrixCommandKeyDefault.AsKey("CMD-Health-H"); var saturators = new List <Command>(); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = HealthCountsStream.GetInstance(key, 10, 100); for (var i = 0; i < 10; i++) { saturators.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 500, ExecutionIsolationStrategy.SEMAPHORE)); } var rejected1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE); var rejected2 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); // 10 commands will saturate semaphore when called from different threads. // submit 2 more requests and they should be SEMAPHORE_REJECTED // should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes var tasks = new List <Task>(); foreach (var saturator in saturators) { tasks.Add(Task.Run(() => saturator.Execute())); } await Task.Delay(50); tasks.Add(Task.Run(() => rejected1.Execute())); tasks.Add(Task.Run(() => rejected2.Execute())); Task.WaitAll(tasks.ToArray()); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); Assert.True(rejected1.IsResponseSemaphoreRejected, "rejected1 not rejected"); Assert.True(rejected2.IsResponseSemaphoreRejected, "rejected2 not rejected"); // should only see failures here, not SHORT-CIRCUITS Assert.Equal(2L, stream.Latest.ErrorCount); Assert.Equal(12L, stream.Latest.TotalRequests); }
public void TestSingleBadRequest() { IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-E"); stream = HealthCountsStream.GetInstance(key, 10, 100); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); Command cmd = Command.From(groupKey, key, HystrixEventType.BAD_REQUEST); cmd.Observe(); Assert.True(latch.Wait(10000), "CountdownEvent was not set!"); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(0L, stream.Latest.ErrorCount); Assert.Equal(0L, stream.Latest.TotalRequests); }
public async Task TestSingleTestOnOpenCircuitAfterTimeWindow() { var key = "cmd-F"; HystrixCommand <bool> cmd1 = new FailureCommand(key, 0); var cb = cmd1._circuitBreaker; var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions); Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start"); // this should start as allowing requests Assert.True(cb.AllowRequest, "Request NOT allowed when expected! (1)"); Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!"); await cmd1.ExecuteAsync(); HystrixCommand <bool> cmd2 = new FailureCommand(key, 0); await cmd2.ExecuteAsync(); HystrixCommand <bool> cmd3 = new FailureCommand(key, 0); await cmd3.ExecuteAsync(); HystrixCommand <bool> cmd4 = new FailureCommand(key, 0); await cmd4.ExecuteAsync(); // Allow window to pass, everything has failed in the test window so we should return false now // Time.Wait(200); Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update"); Assert.False(cb.AllowRequest, "Request allowed when NOT expected!"); Assert.True(cb.IsOpen, "Circuit is closed when it should be open!"); // wait for sleepWindow to pass Time.Wait(500); // we should now allow 1 request Assert.True(cb.AllowRequest, "Request NOT allowed when expected! (2)"); // but the circuit should still be open Assert.True(cb.IsOpen, "Circuit is closed when it should be open! (2)"); // and further requests are still blocked Assert.False(cb.AllowRequest, "Request allowed when NOT expected! (2)"); }
public async Task TestFallbackRejection() { var key = HystrixCommandKeyDefault.AsKey("CMD-Health-L"); var fallbackSaturators = new List <Command>(); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = HealthCountsStream.GetInstance(key, 10, 100); for (var i = 0; i < 5; i++) { fallbackSaturators.Add(CommandStreamTest.Command.From(GroupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 500)); } var rejection1 = CommandStreamTest.Command.From(GroupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 0); var rejection2 = CommandStreamTest.Command.From(GroupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 0); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); // fallback semaphore size is 5. So let 5 commands saturate that semaphore, then // let 2 more commands go to fallback. they should get rejected by the fallback-semaphore var tasks = new List <Task>(); foreach (var saturator in fallbackSaturators) { tasks.Add(saturator.ExecuteAsync()); } await Task.Delay(50); output.WriteLine("ReqLog1 @ " + Time.CurrentTimeMillis + " " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await rejection1.Observe()); await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await rejection2.Observe()); output.WriteLine("ReqLog2 @ " + Time.CurrentTimeMillis + " " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Task.WaitAll(tasks.ToArray()); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); Assert.Equal(7L, stream.Latest.ErrorCount); Assert.Equal(7L, stream.Latest.TotalRequests); }
public async Task TestShortCircuited() { var key = HystrixCommandKeyDefault.AsKey("CMD-Health-G"); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = HealthCountsStream.GetInstance(key, 10, 100); var failure1 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0); var failure2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0); var failure3 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0); var shortCircuit1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS); var shortCircuit2 = Command.From(GroupKey, key, HystrixEventType.SUCCESS); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); // 3 failures in a row will trip circuit. let bucket roll once then submit 2 requests. // should see 3 FAILUREs and 2 SHORT_CIRCUITs and then 5 FALLBACK_SUCCESSes await failure1.Observe(); await failure2.Observe(); await failure3.Observe(); output.WriteLine(Time.CurrentTimeMillis + " Waiting for health window to change"); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); output.WriteLine(Time.CurrentTimeMillis + " Running short circuits"); await shortCircuit1.Observe(); await shortCircuit2.Observe(); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); Assert.True(shortCircuit1.IsResponseShortCircuited); Assert.True(shortCircuit2.IsResponseShortCircuited); // should only see failures here, not SHORT-CIRCUITS Assert.Equal(3L, stream.Latest.ErrorCount); Assert.Equal(3L, stream.Latest.TotalRequests); }