public void TestOneCommandCarriesOverToNextBucket()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-D");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-D");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-D");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 560);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 75);
cmd1.Observe();
Time.Wait(150); // bucket roll
cmd2.Observe();
Time.Wait(1);
cmd3.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(3, stream.LatestRollingMax);
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-A");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-A");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-A");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
// no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Assert.Equal(0, stream.LatestRollingMax);
}
public void Before()
{
context = HystrixRequestContext.InitializeContext();
HystrixCommandMetrics.Reset();
HystrixThreadPoolMetrics.Reset();
HystrixCollapserMetrics.Reset();
// clear collapsers
RequestCollapserFactory.Reset();
// clear circuit breakers
HystrixCircuitBreakerFactory.Reset();
HystrixPlugins.Reset();
HystrixOptionsFactory.Reset();
// clear up all streams
CumulativeCollapserEventCounterStream.Reset();
CumulativeCommandEventCounterStream.Reset();
CumulativeThreadPoolEventCounterStream.Reset();
RollingCollapserBatchSizeDistributionStream.Reset();
RollingCollapserEventCounterStream.Reset();
RollingCommandEventCounterStream.Reset();
RollingCommandLatencyDistributionStream.Reset();
RollingCommandMaxConcurrencyStream.Reset();
RollingCommandUserLatencyDistributionStream.Reset();
RollingThreadPoolEventCounterStream.Reset();
RollingThreadPoolMaxConcurrencyStream.Reset();
}
public void TestStartsAndEndsInSameBucketProduceValue()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-B");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-B");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-B");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
cmd1.Observe();
Time.Wait(1);
cmd2.Observe();
Assert.True(latch.Wait(10000));
Assert.Equal(2, stream.LatestRollingMax);
}
public void TestConcurrencyStreamProperlyFiltersOutResponseFromCache()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-G");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-G");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-G");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
Command cmd4 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
cmd1.Observe();
Time.Wait(5);
cmd2.Observe();
cmd3.Observe();
cmd4.Observe();
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(cmd2.IsResponseFromCache);
Assert.True(cmd3.IsResponseFromCache);
Assert.True(cmd4.IsResponseFromCache);
Assert.Equal(1, stream.LatestRollingMax);
}
public void TestMultipleCommandsCarryOverMultipleBucketsAndThenAgeOut()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-F");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-F");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-F");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Take(20 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
var cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
var cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
var cmd4 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
Task t1 = cmd1.ExecuteAsync();
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t2 = cmd2.ExecuteAsync();
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t3 = cmd3.ExecuteAsync();
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t4 = cmd4.ExecuteAsync();
Task.WaitAll(t1, t2, t3, t4);
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestStartsAndEndsInSameBucketSemaphoreIsolated()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-C");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-C");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-C");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10, ExecutionIsolationStrategy.SEMAPHORE);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 14, ExecutionIsolationStrategy.SEMAPHORE);
cmd1.Observe();
Time.Wait(1);
cmd2.Observe();
Assert.True(latch.Wait(10000));
// since commands run in semaphore isolation, they are not tracked by threadpool metrics
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestMultipleCommandsCarryOverMultipleBucketsAndThenAgeOut()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-F");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-F");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-F");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(30).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd4 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
cmd1.Observe();
Time.Wait(100); // bucket roll
cmd2.Observe();
Time.Wait(100);
cmd3.Observe();
Time.Wait(100);
cmd4.Observe();
Assert.True(latch.Wait(10000));
Assert.Equal(0, stream.LatestRollingMax);
}
public RollingThreadPoolMaxConcurrencyStreamTest(ITestOutputHelper output) : base()
{
this.output = output;
HystrixThreadPoolStartStream.Reset();
RollingThreadPoolMaxConcurrencyStream.Reset();
}
public void TestConcurrencyStreamProperlyFiltersOutResponseFromCache()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-G");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-G");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-G");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
var cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
var cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
var cmd4 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
cmd1.Execute();
cmd2.Execute();
cmd3.Execute();
cmd4.Execute();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.True(cmd2.IsResponseFromCache);
Assert.True(cmd3.IsResponseFromCache);
Assert.True(cmd4.IsResponseFromCache);
Assert.Equal(1, stream.LatestRollingMax);
}
public void TestConcurrencyStreamProperlyFiltersOutSemaphoreRejections()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-I");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-I");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-I");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
//10 commands executed concurrently on different caller threads should saturate semaphore
//once these are in-flight, execute 10 more concurrently on new caller threads.
//since these are semaphore-rejected, the max concurrency should be 10
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 400, ExecutionIsolationStrategy.SEMAPHORE));
}
List <Command> rejected = new List <Command>();
for (int i = 0; i < 10; i++)
{
rejected.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 100, ExecutionIsolationStrategy.SEMAPHORE));
}
foreach (Command saturatingCmd in saturators)
{
Task t = new Task(() =>
{
saturatingCmd.Observe();
}, CancellationToken.None, TaskCreationOptions.LongRunning);
t.Start();
}
Time.Wait(30);
foreach (Command rejectedCmd in rejected)
{
Task.Run(() => rejectedCmd.Observe());
}
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
foreach (Command rejectedCmd in rejected)
{
Assert.True(rejectedCmd.IsResponseSemaphoreRejected || rejectedCmd.IsResponseShortCircuited);
}
//should be 0 since all are executed in a semaphore
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestOneCommandCarriesOverToNextBucket()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-D");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-D");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-D");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 560);
var cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
var cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 75);
Task t1 = cmd1.ExecuteAsync();
// Time.Wait(150); // bucket roll
Assert.True(WaitForObservableToUpdate(stream.Observe(), 1, 500, output), "Stream update took to long");
Task t2 = cmd2.ExecuteAsync();
Task t3 = cmd3.ExecuteAsync();
Task.WaitAll(t1, t2, t3);
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(3, stream.LatestRollingMax);
}
internal static void Reset()
{
RollingThreadPoolEventCounterStream.Reset();
CumulativeThreadPoolEventCounterStream.Reset();
RollingThreadPoolMaxConcurrencyStream.Reset();
Metrics.Clear();
}
public override void Dispose()
{
latchSubscription?.Dispose();
stream?.Unsubscribe();
latchSubscription = null;
stream = null;
base.Dispose();
}
private HystrixThreadPoolMetrics(IHystrixThreadPoolKey threadPoolKey, IHystrixTaskScheduler threadPool, IHystrixThreadPoolOptions properties) : base(null)
{
this.threadPoolKey = threadPoolKey;
this.threadPool = threadPool;
this.properties = properties;
rollingCounterStream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, properties);
cumulativeCounterStream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, properties);
rollingThreadPoolMaxConcurrencyStream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, properties);
}
public void TestConcurrencyStreamProperlyFiltersOutThreadPoolRejections()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-J");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-J");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-J");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
//10 commands executed concurrently should saturate the Hystrix threadpool
//once these are in-flight, execute 10 more concurrently
//since these are threadpool-rejected, the max concurrency should be 10
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 400));
}
List <Command> rejected = new List <Command>();
for (int i = 0; i < 10; i++)
{
rejected.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 100));
}
foreach (Command saturatingCmd in saturators)
{
saturatingCmd.Observe();
}
Time.Wait(30);
foreach (Command rejectedCmd in rejected)
{
rejectedCmd.Observe();
}
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
foreach (Command rejectedCmd in rejected)
{
Assert.True(rejectedCmd.IsResponseThreadPoolRejected);
}
//this should not count rejected commands
Assert.Equal(10, stream.LatestRollingMax);
}
public async Task TestConcurrencyStreamProperlyFiltersOutSemaphoreRejections()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-I");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-I");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-I");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 10 commands executed concurrently on different caller threads should saturate semaphore
// once these are in-flight, execute 10 more concurrently on new caller threads.
// since these are semaphore-rejected, the max concurrency should be 10
var saturators = new List <Command>();
for (var i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 500, ExecutionIsolationStrategy.SEMAPHORE));
}
var rejected = new List <Command>();
for (var i = 0; i < 10; i++)
{
rejected.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE));
}
var tasks = new List <Task>();
foreach (var saturatingCmd in saturators)
{
tasks.Add(Task.Run(() => saturatingCmd.Execute()));
}
await Task.Delay(50);
foreach (var rejectedCmd in rejected)
{
await Task.Run(() => rejectedCmd.Execute());
}
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
foreach (var rejectedCmd in rejected)
{
Assert.True(rejectedCmd.IsResponseSemaphoreRejected || rejectedCmd.IsResponseShortCircuited);
}
// should be 0 since all are executed in a semaphore
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestConcurrencyStreamProperlyFiltersOutThreadPoolRejections()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-J");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-J");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-J");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Take(10 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 10 commands executed concurrently should saturate the Hystrix threadpool
// once these are in-flight, execute 10 more concurrently
// since these are threadpool-rejected, the max concurrency should be 10
var saturators = new List <Command>();
for (var i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 400));
}
var rejected = new List <Command>();
for (var i = 0; i < 10; i++)
{
rejected.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 100));
}
var tasks = new List <Task>();
foreach (var saturatingCmd in saturators)
{
tasks.Add(saturatingCmd.ExecuteAsync());
}
Time.Wait(30);
foreach (var rejectedCmd in rejected)
{
rejectedCmd.Observe();
}
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
foreach (var rejectedCmd in rejected)
{
Assert.True(rejectedCmd.IsResponseThreadPoolRejected);
}
// this should not count rejected commands
Assert.Equal(10, stream.LatestRollingMax);
}
public void TestEmptyStreamProducesZeros()
{
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-A");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestConcurrencyStreamProperlyFiltersOutShortCircuits()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-H");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-H");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-H");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// after 3 failures, next command should short-circuit.
// to prove short-circuited commands don't contribute to concurrency, execute 3 FAILURES in the first bucket sequentially
// then when circuit is open, execute 20 concurrent commands. they should all get short-circuited, and max concurrency should be 1
var failure1 = Command.From(groupKey, key, HystrixEventType.FAILURE);
var failure2 = Command.From(groupKey, key, HystrixEventType.FAILURE);
var failure3 = Command.From(groupKey, key, HystrixEventType.FAILURE);
var shortCircuited = new List <Command>();
for (var i = 0; i < 20; i++)
{
shortCircuited.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 0));
}
failure1.Execute();
failure2.Execute();
failure3.Execute();
Assert.True(WaitForHealthCountToUpdate(key.Name, 500, output), "Health count stream update took to long");
var shorts = new List <Task <int> >();
foreach (var cmd in shortCircuited)
{
shorts.Add(cmd.ExecuteAsync());
}
Task.WaitAll(shorts.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
foreach (var cmd in shortCircuited)
{
Assert.True(cmd.IsResponseShortCircuited);
}
Assert.Equal(1, stream.LatestRollingMax);
}
public static void ShutdownThreads()
{
CumulativeCommandEventCounterStream.Reset();
CumulativeThreadPoolEventCounterStream.Reset();
RollingCommandEventCounterStream.Reset();
RollingThreadPoolEventCounterStream.Reset();
RollingCollapserEventCounterStream.Reset();
RollingCollapserEventCounterStream.Reset();
HealthCountsStream.Reset();
RollingCollapserBatchSizeDistributionStream.Reset();
RollingCommandLatencyDistributionStream.Reset();
RollingCommandUserLatencyDistributionStream.Reset();
RollingCommandMaxConcurrencyStream.Reset();
RollingThreadPoolMaxConcurrencyStream.Reset();
}
public void TestConcurrencyStreamProperlyFiltersOutShortCircuits()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-H");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-H");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-H");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
// after 3 failures, next command should short-circuit.
// to prove short-circuited commands don't contribute to concurrency, execute 3 FAILURES in the first bucket sequentially
// then when circuit is open, execute 20 concurrent commands. they should all get short-circuited, and max concurrency should be 1
Command failure1 = Command.From(groupKey, key, HystrixEventType.FAILURE);
Command failure2 = Command.From(groupKey, key, HystrixEventType.FAILURE);
Command failure3 = Command.From(groupKey, key, HystrixEventType.FAILURE);
List <Command> shortCircuited = new List <Command>();
for (int i = 0; i < 20; i++)
{
shortCircuited.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 100));
}
failure1.Execute();
failure2.Execute();
failure3.Execute();
Time.Wait(150);
foreach (Command cmd in shortCircuited)
{
cmd.Observe();
}
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
foreach (Command cmd in shortCircuited)
{
Assert.True(cmd.IsResponseShortCircuited);
}
Assert.Equal(1, stream.LatestRollingMax);
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-A");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-A");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-A");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
// no writes
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestStartsAndEndsInSameBucketProduceValue()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-B");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-B");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-B");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
var cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
Task t1 = cmd1.ExecuteAsync();
Task t2 = cmd2.ExecuteAsync();
Task.WaitAll(t1, t2);
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(2, stream.LatestRollingMax);
}
public void TestStartsAndEndsInSameBucketSemaphoreIsolated()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-C");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-C");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-C");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10, ExecutionIsolationStrategy.SEMAPHORE);
var cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 14, ExecutionIsolationStrategy.SEMAPHORE);
Task t1 = cmd1.ExecuteAsync();
Task t2 = cmd2.ExecuteAsync();
Task.WaitAll(t1, t2);
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
// since commands run in semaphore isolation, they are not tracked by threadpool metrics
Assert.Equal(0, stream.LatestRollingMax);
}
