public void TestConcurrencyStreamProperlyFiltersOutThreadPoolRejections()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-J");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-J");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-J");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
//10 commands executed concurrently should saturate the Hystrix threadpool
//once these are in-flight, execute 10 more concurrently
//since these are threadpool-rejected, the max concurrency should be 10
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 400));
}
List <Command> rejected = new List <Command>();
for (int i = 0; i < 10; i++)
{
rejected.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 100));
}
foreach (Command saturatingCmd in saturators)
{
saturatingCmd.Observe();
}
Time.Wait(30);
foreach (Command rejectedCmd in rejected)
{
rejectedCmd.Observe();
}
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
foreach (Command rejectedCmd in rejected)
{
Assert.True(rejectedCmd.IsResponseThreadPoolRejected);
}
//this should not count rejected commands
Assert.Equal(10, stream.LatestRollingMax);
}
public void TestFallbackRejection()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-L");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-L");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-L");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(GetSubscriber(output, latch));
// fallback semaphore size is 5. So let 5 commands saturate that semaphore, then
// let 2 more commands go to fallback. they should get rejected by the fallback-semaphore
List <CommandStreamTest.Command> fallbackSaturators = new List <CommandStreamTest.Command>();
for (int i = 0; i < 5; i++)
{
fallbackSaturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 400));
}
CommandStreamTest.Command rejection1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0);
CommandStreamTest.Command rejection2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0);
foreach (CommandStreamTest.Command saturator in fallbackSaturators)
{
saturator.Observe();
}
try
{
Time.Wait(70);
}
catch (Exception ie)
{
Assert.False(true, ie.Message);
}
rejection1.Observe();
rejection2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
// all 7 commands executed on-thread, so should be executed according to thread-pool metrics
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(7, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestThreadPoolRejected()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-I");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-I");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-I");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
//10 commands will saturate threadpools when called concurrently.
//submit 2 more requests and they should be THREADPOOL_REJECTED
//should see 10 SUCCESSes, 2 THREADPOOL_REJECTED and 2 FALLBACK_SUCCESSes
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 700));
}
CommandStreamTest.Command rejected1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
CommandStreamTest.Command rejected2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
foreach (Command c in saturators)
{
c.Observe();
}
rejected1.Observe();
rejected2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(rejected1.IsResponseThreadPoolRejected);
Assert.True(rejected2.IsResponseThreadPoolRejected);
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(10, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public async Task TestConcurrencyStreamProperlyFiltersOutSemaphoreRejections()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-I");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-I");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-I");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 10 commands executed concurrently on different caller threads should saturate semaphore
// once these are in-flight, execute 10 more concurrently on new caller threads.
// since these are semaphore-rejected, the max concurrency should be 10
var saturators = new List <Command>();
for (var i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 500, ExecutionIsolationStrategy.SEMAPHORE));
}
var rejected = new List <Command>();
for (var i = 0; i < 10; i++)
{
rejected.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE));
}
var tasks = new List <Task>();
foreach (var saturatingCmd in saturators)
{
tasks.Add(Task.Run(() => saturatingCmd.Execute()));
}
await Task.Delay(50);
foreach (var rejectedCmd in rejected)
{
await Task.Run(() => rejectedCmd.Execute());
}
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
foreach (var rejectedCmd in rejected)
{
Assert.True(rejectedCmd.IsResponseSemaphoreRejected || rejectedCmd.IsResponseShortCircuited);
}
// should be 0 since all are executed in a semaphore
Assert.Equal(0, stream.LatestRollingMax);
}
protected static HystrixCommandOptions Options(string groupKey, string commandKey, int requestVolumeThreshold, int sleepWindow)
{
var opts = HystrixCommandOptionsTest.GetUnitTestOptions();
opts.GroupKey = HystrixCommandGroupKeyDefault.AsKey(groupKey);
opts.CommandKey = HystrixCommandKeyDefault.AsKey(commandKey);
opts.ExecutionTimeoutInMilliseconds = 500;
opts.CircuitBreakerRequestVolumeThreshold = requestVolumeThreshold;
opts.CircuitBreakerSleepWindowInMilliseconds = sleepWindow;
return(opts);
}
public void TestNoRequestContextOnSimpleConcurencyStrategyWithoutException()
{
Dispose();
var opts = new HystrixCommandOptions()
{
RequestLogEnabled = false,
GroupKey = HystrixCommandGroupKeyDefault.AsKey("SimpleCommand")
};
new SimpleCommand(output, opts).Execute();
}
public void TestSemaphoreRejected()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-H");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-H");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-H");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
// 10 commands will saturate semaphore when called from different threads.
// submit 2 more requests and they should be SEMAPHORE_REJECTED
// should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 300, ExecutionIsolationStrategy.SEMAPHORE));
}
Command rejected1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
Command rejected2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
foreach (Command c in saturators)
{
Task t = new Task(
() =>
{
c.Observe();
}, CancellationToken.None,
TaskCreationOptions.LongRunning);
t.Start();
Task.Run(() => c.Observe());
}
Time.Wait(10);
Task.Run(() => rejected1.Observe());
Task.Run(() => rejected2.Observe());
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(rejected1.IsResponseSemaphoreRejected);
Assert.True(rejected2.IsResponseSemaphoreRejected);
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestConcurrencyStreamProperlyFiltersOutThreadPoolRejections()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-J");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-J");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-J");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Take(10 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 10 commands executed concurrently should saturate the Hystrix threadpool
// once these are in-flight, execute 10 more concurrently
// since these are threadpool-rejected, the max concurrency should be 10
var saturators = new List <Command>();
for (var i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 400));
}
var rejected = new List <Command>();
for (var i = 0; i < 10; i++)
{
rejected.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 100));
}
var tasks = new List <Task>();
foreach (var saturatingCmd in saturators)
{
tasks.Add(saturatingCmd.ExecuteAsync());
}
Time.Wait(30);
foreach (var rejectedCmd in rejected)
{
rejectedCmd.Observe();
}
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
foreach (var rejectedCmd in rejected)
{
Assert.True(rejectedCmd.IsResponseThreadPoolRejected);
}
// this should not count rejected commands
Assert.Equal(10, stream.LatestRollingMax);
}
public ExecutionReport PlaceOrder(ExecutionReport order)
{
var options = new HystrixCommandOptions(HystrixCommandGroupKeyDefault.AsKey("OMS"), HystrixCommandKeyDefault.AsKey("OMS.NewOrder"));
var cmd = new HystrixCommand <ExecutionReport>(options,
run: () => PlaceOrderRun(order),
fallback: () => PlaceOrderFallback(order));
// Thread.Sleep(1000);
var result = cmd.Execute();
return(result);
}
public void TestThreadPoolRejected()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-I");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-I");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-I");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(new LatchedObserver(output, latch));
// 10 commands will saturate threadpools when called concurrently.
// submit 2 more requests and they should be THREADPOOL_REJECTED
// should see 10 SUCCESSes, 2 THREADPOOL_REJECTED and 2 FALLBACK_SUCCESSes
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 200));
}
Command rejected1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
Command rejected2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
foreach (Command saturator in saturators)
{
saturator.Observe();
}
try
{
Time.Wait(100);
}
catch (Exception ie)
{
Assert.False(true, ie.Message);
}
rejected1.Observe();
rejected2.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(rejected1.IsResponseThreadPoolRejected, "Command1 IsResponseThreadPoolRejected");
Assert.True(rejected2.IsResponseThreadPoolRejected, "Command2 IsResponseThreadPoolRejected");
// none of these got executed on a thread-pool, so thread pool metrics should be 0
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(10, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public async Task TestFallbackRejection()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-L");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-L");
var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-L");
var fallbackSaturators = new List <Command>();
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
for (var i = 0; i < 5; i++)
{
fallbackSaturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 500));
}
var rejection1 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 0);
var rejection2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 0);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// fallback semaphore size is 5. So let 5 commands saturate that semaphore, then
// let 2 more commands go to fallback. they should get rejected by the fallback-semaphore
var tasks = new List <Task>();
foreach (var saturator in fallbackSaturators)
{
tasks.Add(saturator.ExecuteAsync());
}
await Task.Delay(50);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await rejection1.Observe());
await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await rejection2.Observe());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
// all 7 commands executed on-thread, so should be executed according to thread-pool metrics
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(7, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestShortCircuited()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-G");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-G");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-G");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(GetSubscriber(output, latch));
//3 failures in a row will trip circuit. let bucket roll once then submit 2 requests.
//should see 3 FAILUREs and 2 SHORT_CIRCUITs and each should see a FALLBACK_SUCCESS
CommandStreamTest.Command failure1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20);
CommandStreamTest.Command failure2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20);
CommandStreamTest.Command failure3 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20);
CommandStreamTest.Command shortCircuit1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS);
CommandStreamTest.Command shortCircuit2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS);
failure1.Observe();
failure2.Observe();
failure3.Observe();
Time.Wait(150);
shortCircuit1.Observe();
shortCircuit2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(shortCircuit1.IsResponseShortCircuited);
Assert.True(shortCircuit2.IsResponseShortCircuited);
//only the FAILUREs should show up in thread pool counters
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(3, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestNoRequestContextOnSimpleConcurencyStrategyWithoutException()
{
base.Dispose();
//ConfigurationManager.getConfigInstance().setProperty("hystrix.command.default.requestLog.enabled", "false");
var opts = new HystrixCommandOptions()
{
RequestLogEnabled = false,
GroupKey = HystrixCommandGroupKeyDefault.AsKey("SimpleCommand")
};
new SimpleCommand(output, opts).Execute();
//Assert.True("We are able to run the simple command without a context initialization error.", true);
}
public TestCommand(string commandName, string value, bool fail, bool failOnFallback)
: base(new HystrixCommandOptions()
{
GroupKey = HystrixCommandGroupKeyDefault.AsKey("RequestLogTestCommand"),
CommandKey = HystrixCommandKeyDefault.AsKey(commandName)
})
{
this.value = value;
this.fail = fail;
this.failOnFallback = failOnFallback;
this.timeout = false;
this.useFallback = true;
this.useCache = true;
}
public TestCommand(string value, bool fail, bool failOnFallback, bool timeout)
: base(new HystrixCommandOptions()
{
GroupKey = HystrixCommandGroupKeyDefault.AsKey("RequestLogTestCommand"),
ExecutionTimeoutInMilliseconds = 500
})
{
this.value = value;
this.fail = fail;
this.failOnFallback = failOnFallback;
this.timeout = timeout;
this.useFallback = false;
this.useCache = false;
}
private static IHystrixCommandOptions GetOptions()
{
HystrixCommandOptions opts = new HystrixCommandOptions()
{
GroupKey = HystrixCommandGroupKeyDefault.AsKey("testTimeoutConcurrency"),
CommandKey = HystrixCommandKeyDefault.AsKey("testTimeoutConcurrencyCommand"),
ExecutionTimeoutInMilliseconds = 3,
CircuitBreakerEnabled = false,
FallbackIsolationSemaphoreMaxConcurrentRequests = NUM_CONCURRENT_COMMANDS,
ThreadPoolOptions = GetThreadPoolOptions()
};
return(opts);
}
public void TestConcurrencyStreamProperlyFiltersOutShortCircuits()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-H");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-H");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-H");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// after 3 failures, next command should short-circuit.
// to prove short-circuited commands don't contribute to concurrency, execute 3 FAILURES in the first bucket sequentially
// then when circuit is open, execute 20 concurrent commands. they should all get short-circuited, and max concurrency should be 1
var failure1 = Command.From(groupKey, key, HystrixEventType.FAILURE);
var failure2 = Command.From(groupKey, key, HystrixEventType.FAILURE);
var failure3 = Command.From(groupKey, key, HystrixEventType.FAILURE);
var shortCircuited = new List <Command>();
for (var i = 0; i < 20; i++)
{
shortCircuited.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 0));
}
failure1.Execute();
failure2.Execute();
failure3.Execute();
Assert.True(WaitForHealthCountToUpdate(key.Name, 500, output), "Health count stream update took to long");
var shorts = new List <Task <int> >();
foreach (var cmd in shortCircuited)
{
shorts.Add(cmd.ExecuteAsync());
}
Task.WaitAll(shorts.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
foreach (var cmd in shortCircuited)
{
Assert.True(cmd.IsResponseShortCircuited);
}
Assert.Equal(1, stream.LatestRollingMax);
}
public async void TestThreadPoolRejected()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-I");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-I");
var key = HystrixCommandKeyDefault.AsKey("RollingCounter-I");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 2000), "Stream failed to start");
// 10 commands will saturate threadpools when called concurrently.
// submit 2 more requests and they should be THREADPOOL_REJECTED
// should see 10 SUCCESSes, 2 THREADPOOL_REJECTED and 2 FALLBACK_SUCCESSes
var saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 500));
}
var rejected1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
var rejected2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
var tasks = new List <Task>();
foreach (var saturator in saturators)
{
tasks.Add(saturator.ExecuteAsync());
}
await Task.Delay(50);
await rejected1.Observe();
await rejected2.Observe();
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 2000, output), "Latch took to long to update");
Assert.True(rejected1.IsResponseThreadPoolRejected, "Command1 IsResponseThreadPoolRejected");
Assert.True(rejected2.IsResponseThreadPoolRejected, "Command2 IsResponseThreadPoolRejected");
// none of these got executed on a thread-pool, so thread pool metrics should be 0
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(10, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public async void TestSemaphoreRejected()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-H");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-H");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-H");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 2000), "Stream failed to start");
// 10 commands will saturate semaphore when called from different threads.
// submit 2 more requests and they should be SEMAPHORE_REJECTED
// should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 500, ExecutionIsolationStrategy.SEMAPHORE));
}
Command rejected1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
Command rejected2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
List <Task> tasks = new List <Task>();
foreach (Command saturator in saturators)
{
tasks.Add(Task.Run(() => saturator.Execute()));
}
await Task.Delay(50);
await Task.Run(() => rejected1.Execute());
await Task.Run(() => rejected2.Execute());
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 2000, output), "Latch took to long to update");
Assert.True(rejected1.IsResponseSemaphoreRejected, "rejected1 not rejected");
Assert.True(rejected2.IsResponseSemaphoreRejected, "rejected2 not rejected");
// none of these got executed on a thread-pool, so thread pool metrics should be 0
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public GetCatalogCommand(Func <int, int, int?, int?, Task <Catalog> > getCatalogFn,
Func <int, int, int?, int?, Task <Catalog> > getCatalogFallbackFn,
int pageIndex,
int itemsPage,
int?brandId,
int?typeId)
: base(HystrixCommandGroupKeyDefault.AsKey("CatalogGroup"))
{
PageIndex = pageIndex;
ItemsPage = itemsPage;
BrandId = brandId;
TypeId = typeId;
GetCatalogFn = getCatalogFn;
GetCatalogFallbackFn = getCatalogFallbackFn;
}
public void TestThreadPoolRejected()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-I");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-I");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-I");
List <Command> saturators = new List <Command>();
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 500));
}
Command rejected1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
Command rejected2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 10 commands will saturate threadpools when called concurrently.
// submit 2 more requests and they should be THREADPOOL_REJECTED
// should see 10 SUCCESSes, 2 THREADPOOL_REJECTED and 2 FALLBACK_SUCCESSes
List <Task> tasks = new List <Task>();
foreach (Command c in saturators)
{
tasks.Add(c.ExecuteAsync());
}
Time.Wait(50);
tasks.Add(rejected1.ExecuteAsync());
tasks.Add(rejected2.ExecuteAsync());
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.True(rejected1.IsResponseThreadPoolRejected);
Assert.True(rejected2.IsResponseThreadPoolRejected);
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(10, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
private async Task SetColorNotifyObservers(ColorChangeResponse response, bool?notify = true, double?duration = 1)
{
var hystrixOptions = new HystrixCommandOptions(HystrixCommandKeyDefault.AsKey("SetColor"));
hystrixOptions.GroupKey = HystrixCommandGroupKeyDefault.AsKey("SetColorGroup");
hystrixOptions.ExecutionTimeoutEnabled = false;
SetColorCommand command = new SetColorCommand(hystrixOptions, _lifxKey, response.HexColor, duration);
await command.ExecuteAsync();
if (notify == true)
{
await _hubContext.Clients.All.SendAsync("Messages", new List <ColorChangeResponse> {
response
});
}
}
public void TestConcurrencyStreamProperlyFiltersOutShortCircuits()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-H");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-H");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-H");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
// after 3 failures, next command should short-circuit.
// to prove short-circuited commands don't contribute to concurrency, execute 3 FAILURES in the first bucket sequentially
// then when circuit is open, execute 20 concurrent commands. they should all get short-circuited, and max concurrency should be 1
Command failure1 = Command.From(groupKey, key, HystrixEventType.FAILURE);
Command failure2 = Command.From(groupKey, key, HystrixEventType.FAILURE);
Command failure3 = Command.From(groupKey, key, HystrixEventType.FAILURE);
List <Command> shortCircuited = new List <Command>();
for (int i = 0; i < 20; i++)
{
shortCircuited.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 100));
}
failure1.Execute();
failure2.Execute();
failure3.Execute();
Time.Wait(150);
foreach (Command cmd in shortCircuited)
{
cmd.Observe();
}
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
foreach (Command cmd in shortCircuited)
{
Assert.True(cmd.IsResponseShortCircuited);
}
Assert.Equal(1, stream.LatestRollingMax);
}
private IHystrixCommandOptions GetCommandOptions(string serviceName, string methodName)
{
var strategy = HystrixPlugins.OptionsStrategy;
var dynOpts = strategy.GetDynamicOptions(_configuration);
var commandKey = HystrixCommandKeyDefault.AsKey($"{serviceName}.{methodName}");
var groupKey = HystrixCommandGroupKeyDefault.AsKey($"{serviceName}Group");
IHystrixCommandOptions opts = new HystrixCommandOptions(groupKey, commandKey, null, dynOpts)
{
ThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey($"{serviceName}Group")
};
opts.ThreadPoolOptions = new HystrixThreadPoolOptions(opts.ThreadPoolKey, null, dynOpts);
return(opts);
}
public async Task TestShortCircuited()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-G");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-G");
var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-G");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
var failure1 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0);
var failure2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0);
var failure3 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0);
var shortCircuit1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS);
var shortCircuit2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 3 failures in a row will trip circuit. let bucket roll once then submit 2 requests.
// should see 3 FAILUREs and 2 SHORT_CIRCUITs and each should see a FALLBACK_SUCCESS
await failure1.Observe();
await failure2.Observe();
await failure3.Observe();
Assert.True(WaitForHealthCountToUpdate(key.Name, 500, output), "health count took to long to update");
output.WriteLine(Time.CurrentTimeMillis + " running failures");
await shortCircuit1.Observe();
await shortCircuit2.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.True(shortCircuit1.IsResponseShortCircuited);
Assert.True(shortCircuit2.IsResponseShortCircuited);
// only the FAILUREs should show up in thread pool counters
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(3, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public static IHystrixCommandOptions GetCommandOptions(string serviceName, string methodName)
{
var configuration = new ConfigurationBuilder().AddInMemoryCollection().Build();
var strategy = HystrixPlugins.OptionsStrategy;
var dynOpts = strategy.GetDynamicOptions(configuration);
var commandKey = HystrixCommandKeyDefault.AsKey($"{serviceName}.{methodName}");
var groupKey = HystrixCommandGroupKeyDefault.AsKey($"{serviceName}Group");
IHystrixCommandOptions opts = new HystrixCommandOptions(groupKey, commandKey, null, dynOpts)
{
ThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey($"{serviceName}Group")
};
opts.ThreadPoolOptions = new HystrixThreadPoolOptions(opts.ThreadPoolKey, null, dynOpts);
return(opts);
}
private IHystrixCommandOptions CreateCommandOptions(MethodBase method)
{
var groupKeyName = _clientName;
var commandKeyName = GetCommandKey(_pryxyType, method);
var groupKey = HystrixCommandGroupKeyDefault.AsKey(groupKeyName);
var commandKey = HystrixCommandKeyDefault.AsKey(commandKeyName);
var configuration = _services.GetService <IConfiguration>();
var strategy = HystrixPlugins.OptionsStrategy;
var dynOpts = strategy.GetDynamicOptions(configuration);
var opts = new HystrixCommandOptions(commandKey, null, dynOpts)
{
GroupKey = groupKey
};
return(opts);
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-A");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-A");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-A");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(new LatchedObserver(output, latch));
// no writes
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED) + stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-A");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-A");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-A");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
// no writes
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(0, stream.LatestRollingMax);
}
public static void RegisterHystrixCommand <TService, TImplementation>(this ContainerBuilder container, string groupKey, IConfiguration config)
where TService : class where TImplementation : class, TService
{
if (container == null)
{
throw new ArgumentNullException(nameof(container));
}
if (string.IsNullOrEmpty(groupKey))
{
throw new ArgumentNullException(nameof(groupKey));
}
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
RegisterHystrixCommand <TService, TImplementation>(container, HystrixCommandGroupKeyDefault.AsKey(groupKey), config);
}
