public void TestSemaphoreRejected() { IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-H"); IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-H"); IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-H"); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); stream.StartCachingStreamValuesIfUnstarted(); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); //10 commands will saturate semaphore when called from different threads. //submit 2 more requests and they should be SEMAPHORE_REJECTED //should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes List <Command> saturators = new List <Command>(); for (int i = 0; i < 10; i++) { saturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 300, ExecutionIsolationStrategy.SEMAPHORE)); } CommandStreamTest.Command rejected1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE); CommandStreamTest.Command rejected2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE); foreach (Command c in saturators) { Task t = new Task(() => { c.Observe(); }, CancellationToken.None, TaskCreationOptions.LongRunning); t.Start(); Task.Run(() => c.Observe()); } Time.Wait(10); Task.Run(() => rejected1.Observe()); Task.Run(() => rejected2.Observe()); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.True(rejected1.IsResponseSemaphoreRejected); Assert.True(rejected2.IsResponseSemaphoreRejected); Assert.Equal(2, stream.Latest.Length); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public async Task TestMultipleEventsOverTimeGetStoredAndDoNotAgeOut() { var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-M"); var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-M"); var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-M"); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20); var cmd2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 10); latchSubscription = stream.Observe().Take(20 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); await cmd1.Observe(); await cmd2.Observe(); Assert.True(latch.Wait(20000), "CountdownEvent was not set!"); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); // all commands should not have aged out Assert.Equal(2, stream.Latest.Length); Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public void TestFallbackMissing() { IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-K"); IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-K"); IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-K"); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); stream.StartCachingStreamValuesIfUnstarted(); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); Command cmd = Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_MISSING); cmd.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(2, stream.Latest.Length); Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public void TestSingleBadRequest() { IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-E"); IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-E"); IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-E"); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); stream.StartCachingStreamValuesIfUnstarted(); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); Command cmd = Command.From(groupKey, key, HystrixEventType.BAD_REQUEST); cmd.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } Assert.Equal(2, stream.Latest.Length); Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public override void Dispose() { base.Dispose(); stream.Unsubscribe(); CumulativeThreadPoolEventCounterStream.Reset(); }
public async Task TestRequestFromCache() { var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-F"); var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-F"); var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-F"); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0); var cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE); var cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); await cmd1.Observe(); await cmd2.Observe(); await cmd3.Observe(); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); // RESPONSE_FROM_CACHE should not show up at all in thread pool counters - just the success Assert.Equal(2, stream.Latest.Length); Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public void TestRequestFromCache() { IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-F"); IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-F"); IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-F"); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); stream.StartCachingStreamValuesIfUnstarted(); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20); Command cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE); Command cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE); cmd1.Observe(); cmd2.Observe(); cmd3.Observe(); Assert.True(latch.Wait(10000), "CountdownEvent was not set!"); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); // RESPONSE_FROM_CACHE should not show up at all in thread pool counters - just the success Assert.Equal(2, stream.Latest.Length); Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public void TestMultipleEventsOverTimeGetStoredAndDoNotAgeOut() { IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-M"); IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-M"); IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-M"); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); stream.StartCachingStreamValuesIfUnstarted(); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(30).Subscribe(new LatchedObserver(output, latch)); Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20); Command cmd2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 10); cmd1.Observe(); cmd2.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); // all commands should have aged out Assert.Equal(2, stream.Latest.Length); Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public void TestEmptyStreamProducesZeros() { IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-A"); IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-A"); IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-A"); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); stream.StartCachingStreamValuesIfUnstarted(); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); // no writes try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } Assert.Equal(2, stream.Latest.Length); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public void Before() { context = HystrixRequestContext.InitializeContext(); HystrixCommandMetrics.Reset(); HystrixThreadPoolMetrics.Reset(); HystrixCollapserMetrics.Reset(); // clear collapsers RequestCollapserFactory.Reset(); // clear circuit breakers HystrixCircuitBreakerFactory.Reset(); HystrixPlugins.Reset(); HystrixOptionsFactory.Reset(); // clear up all streams CumulativeCollapserEventCounterStream.Reset(); CumulativeCommandEventCounterStream.Reset(); CumulativeThreadPoolEventCounterStream.Reset(); RollingCollapserBatchSizeDistributionStream.Reset(); RollingCollapserEventCounterStream.Reset(); RollingCommandEventCounterStream.Reset(); RollingCommandLatencyDistributionStream.Reset(); RollingCommandMaxConcurrencyStream.Reset(); RollingCommandUserLatencyDistributionStream.Reset(); RollingThreadPoolEventCounterStream.Reset(); RollingThreadPoolMaxConcurrencyStream.Reset(); }
public override void Dispose() { latchSubscription?.Dispose(); stream?.Unsubscribe(); latchSubscription = null; stream = null; base.Dispose(); }
internal static void Reset() { RollingThreadPoolEventCounterStream.Reset(); CumulativeThreadPoolEventCounterStream.Reset(); RollingThreadPoolMaxConcurrencyStream.Reset(); Metrics.Clear(); }
public void TestFallbackRejection() { IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-L"); IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-L"); IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-L"); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); stream.StartCachingStreamValuesIfUnstarted(); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); // fallback semaphore size is 5. So let 5 commands saturate that semaphore, then // let 2 more commands go to fallback. they should get rejected by the fallback-semaphore List <CommandStreamTest.Command> fallbackSaturators = new List <CommandStreamTest.Command>(); for (int i = 0; i < 5; i++) { fallbackSaturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 400)); } Command rejection1 = Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0); Command rejection2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0); foreach (Command saturator in fallbackSaturators) { saturator.Observe(); } try { Time.Wait(70); } catch (Exception ie) { Assert.True(false, ie.Message); } rejection1.Observe(); rejection2.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); // all 7 commands executed on-thread, so should be executed according to thread-pool metrics Assert.Equal(2, stream.Latest.Length); Assert.Equal(7, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
private HystrixThreadPoolMetrics(IHystrixThreadPoolKey threadPoolKey, IHystrixTaskScheduler threadPool, IHystrixThreadPoolOptions properties) : base(null) { this.threadPoolKey = threadPoolKey; this.threadPool = threadPool; this.properties = properties; rollingCounterStream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, properties); cumulativeCounterStream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, properties); rollingThreadPoolMaxConcurrencyStream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, properties); }
public void TestThreadPoolRejected() { IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-I"); IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-I"); IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-I"); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); stream.StartCachingStreamValuesIfUnstarted(); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); //10 commands will saturate threadpools when called concurrently. //submit 2 more requests and they should be THREADPOOL_REJECTED //should see 10 SUCCESSes, 2 THREADPOOL_REJECTED and 2 FALLBACK_SUCCESSes List <Command> saturators = new List <Command>(); for (int i = 0; i < 10; i++) { saturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 700)); } CommandStreamTest.Command rejected1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0); CommandStreamTest.Command rejected2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0); foreach (Command c in saturators) { c.Observe(); } rejected1.Observe(); rejected2.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.True(rejected1.IsResponseThreadPoolRejected); Assert.True(rejected2.IsResponseThreadPoolRejected); Assert.Equal(2, stream.Latest.Length); Assert.Equal(10, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public async Task TestFallbackRejection() { var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-L"); var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-L"); var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-L"); var fallbackSaturators = new List <Command>(); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); for (var i = 0; i < 5; i++) { fallbackSaturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 500)); } var rejection1 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 0); var rejection2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 0); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); // fallback semaphore size is 5. So let 5 commands saturate that semaphore, then // let 2 more commands go to fallback. they should get rejected by the fallback-semaphore var tasks = new List <Task>(); foreach (var saturator in fallbackSaturators) { tasks.Add(saturator.ExecuteAsync()); } await Task.Delay(50); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await rejection1.Observe()); await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await rejection2.Observe()); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Task.WaitAll(tasks.ToArray()); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); // all 7 commands executed on-thread, so should be executed according to thread-pool metrics Assert.Equal(2, stream.Latest.Length); Assert.Equal(7, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public void TestShortCircuited() { IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-G"); IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-G"); IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-G"); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); stream.StartCachingStreamValuesIfUnstarted(); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); //3 failures in a row will trip circuit. let bucket roll once then submit 2 requests. //should see 3 FAILUREs and 2 SHORT_CIRCUITs and each should see a FALLBACK_SUCCESS CommandStreamTest.Command failure1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20); CommandStreamTest.Command failure2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20); CommandStreamTest.Command failure3 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20); CommandStreamTest.Command shortCircuit1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS); CommandStreamTest.Command shortCircuit2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS); failure1.Observe(); failure2.Observe(); failure3.Observe(); Time.Wait(150); shortCircuit1.Observe(); shortCircuit2.Observe(); try { Assert.True(latch.Wait(10000)); } catch (Exception) { Assert.True(false, "Interrupted ex"); } output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.True(shortCircuit1.IsResponseShortCircuited); Assert.True(shortCircuit2.IsResponseShortCircuited); //only the FAILUREs should show up in thread pool counters Assert.Equal(2, stream.Latest.Length); Assert.Equal(3, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public static void ShutdownThreads() { CumulativeCommandEventCounterStream.Reset(); CumulativeThreadPoolEventCounterStream.Reset(); RollingCommandEventCounterStream.Reset(); RollingThreadPoolEventCounterStream.Reset(); RollingCollapserEventCounterStream.Reset(); RollingCollapserEventCounterStream.Reset(); HealthCountsStream.Reset(); RollingCollapserBatchSizeDistributionStream.Reset(); RollingCommandLatencyDistributionStream.Reset(); RollingCommandUserLatencyDistributionStream.Reset(); RollingCommandMaxConcurrencyStream.Reset(); RollingThreadPoolMaxConcurrencyStream.Reset(); }
public void TestThreadPoolRejected() { var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-I"); var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-I"); var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-I"); var saturators = new List <Command>(); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); for (var i = 0; i < 10; i++) { saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 500)); } var rejected1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0); var rejected2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); // 10 commands will saturate threadpools when called concurrently. // submit 2 more requests and they should be THREADPOOL_REJECTED // should see 10 SUCCESSes, 2 THREADPOOL_REJECTED and 2 FALLBACK_SUCCESSes var tasks = new List <Task>(); foreach (var c in saturators) { tasks.Add(c.ExecuteAsync()); } Time.Wait(50); tasks.Add(rejected1.ExecuteAsync()); tasks.Add(rejected2.ExecuteAsync()); Task.WaitAll(tasks.ToArray()); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); Assert.True(rejected1.IsResponseThreadPoolRejected); Assert.True(rejected2.IsResponseThreadPoolRejected); Assert.Equal(2, stream.Latest.Length); Assert.Equal(10, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public async Task TestSemaphoreRejected() { var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-H"); var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-H"); var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-H"); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); var saturators = new List <Command>(); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); for (var i = 0; i < 10; i++) { saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 500, ExecutionIsolationStrategy.SEMAPHORE)); } var rejected1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE); var rejected2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); // 10 commands will saturate semaphore when called from different threads. // submit 2 more requests and they should be SEMAPHORE_REJECTED // should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes var tasks = new List <Task>(); foreach (var saturator in saturators) { tasks.Add(Task.Run(() => saturator.Execute())); } await Task.Delay(50); tasks.Add(Task.Run(() => rejected1.Execute())); tasks.Add(Task.Run(() => rejected2.Execute())); Task.WaitAll(tasks.ToArray()); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); Assert.True(rejected1.IsResponseSemaphoreRejected); Assert.True(rejected2.IsResponseSemaphoreRejected); Assert.Equal(2, stream.Latest.Length); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public async Task TestShortCircuited() { var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-G"); var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-G"); var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-G"); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); var failure1 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0); var failure2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0); var failure3 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0); var shortCircuit1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS); var shortCircuit2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); // 3 failures in a row will trip circuit. let bucket roll once then submit 2 requests. // should see 3 FAILUREs and 2 SHORT_CIRCUITs and each should see a FALLBACK_SUCCESS await failure1.Observe(); await failure2.Observe(); await failure3.Observe(); Assert.True(WaitForHealthCountToUpdate(key.Name, 500, output), "health count took to long to update"); output.WriteLine(Time.CurrentTimeMillis + " running failures"); await shortCircuit1.Observe(); await shortCircuit2.Observe(); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); Assert.True(shortCircuit1.IsResponseShortCircuited); Assert.True(shortCircuit2.IsResponseShortCircuited); // only the FAILUREs should show up in thread pool counters Assert.Equal(2, stream.Latest.Length); Assert.Equal(3, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public void TestEmptyStreamProducesZeros() { var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-A"); var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-A"); var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-A"); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); stream.StartCachingStreamValuesIfUnstarted(); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); Assert.Equal(2, stream.Latest.Length); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public void TestSingleTimeout() { IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-D"); IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-D"); IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-D"); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); stream.StartCachingStreamValuesIfUnstarted(); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); Command cmd = Command.From(groupKey, key, HystrixEventType.TIMEOUT); cmd.Observe(); Assert.True(latch.Wait(10000), "CountdownEvent was not set!"); Assert.Equal(2, stream.Latest.Length); Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public async void TestSingleBadRequest() { IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-E"); IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-E"); IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-E"); CountdownEvent latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); Command cmd = Command.From(groupKey, key, HystrixEventType.BAD_REQUEST); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); await Assert.ThrowsAsync <HystrixBadRequestException>(async() => await cmd.Observe()); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); Assert.Equal(2, stream.Latest.Length); Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public async Task TestSingleSuccess() { var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-B"); var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-B"); var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-B"); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); var cmd = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); await cmd.Observe(); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); Assert.Equal(2, stream.Latest.Length); Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }
public void TestSingleFailure() { IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-C"); IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-C"); IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-C"); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); stream.StartCachingStreamValuesIfUnstarted(); CountdownEvent latch = new CountdownEvent(1); stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch)); Command cmd = Command.From(groupKey, key, HystrixEventType.FAILURE, 20); cmd.Observe(); Assert.True(latch.Wait(10000), "CountdownEvent was not set!"); Assert.Equal(2, stream.Latest.Length); Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); }
public async Task TestFallbackMissing() { var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-K"); var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-K"); var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-K"); var latch = new CountdownEvent(1); var observer = new LatchedObserver(output, latch); stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100); var cmd = Command.From(groupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_MISSING); latchSubscription = stream.Observe().Subscribe(observer); Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start"); await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await cmd.Observe()); Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update"); Assert.Equal(2, stream.Latest.Length); Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED)); Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED)); }