internal HystrixCommandMetrics(IHystrixCommandKey key, IHystrixCommandGroupKey commandGroup, IHystrixThreadPoolKey threadPoolKey, IHystrixCommandOptions properties, HystrixEventNotifier eventNotifier)
: base(null)
{
this.key = key;
this.group = commandGroup;
this.threadPoolKey = threadPoolKey;
this.properties = properties;
healthCountsStream = HealthCountsStream.GetInstance(key, properties);
rollingCommandEventCounterStream = RollingCommandEventCounterStream.GetInstance(key, properties);
cumulativeCommandEventCounterStream = CumulativeCommandEventCounterStream.GetInstance(key, properties);
rollingCommandLatencyDistributionStream = RollingCommandLatencyDistributionStream.GetInstance(key, properties);
rollingCommandUserLatencyDistributionStream = RollingCommandUserLatencyDistributionStream.GetInstance(key, properties);
rollingCommandMaxConcurrencyStream = RollingCommandMaxConcurrencyStream.GetInstance(key, properties);
}
public static HystrixCommandMetrics GetInstance(IHystrixCommandKey key, IHystrixCommandGroupKey commandGroup, IHystrixThreadPoolKey threadPoolKey, IHystrixCommandOptions properties)
{
// attempt to retrieve from cache first
IHystrixThreadPoolKey nonNullThreadPoolKey;
if (threadPoolKey == null)
{
nonNullThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey(commandGroup.Name);
}
else
{
nonNullThreadPoolKey = threadPoolKey;
}
return(Metrics.GetOrAddEx(key.Name, (k) => new HystrixCommandMetrics(key, commandGroup, nonNullThreadPoolKey, properties, HystrixPlugins.EventNotifier)));
}
public void TestConcurrencyStreamProperlyFiltersOutShortCircuits()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-H");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-H");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-H");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
// after 3 failures, next command should short-circuit.
// to prove short-circuited commands don't contribute to concurrency, execute 3 FAILURES in the first bucket sequentially
// then when circuit is open, execute 20 concurrent commands. they should all get short-circuited, and max concurrency should be 1
Command failure1 = Command.From(groupKey, key, HystrixEventType.FAILURE);
Command failure2 = Command.From(groupKey, key, HystrixEventType.FAILURE);
Command failure3 = Command.From(groupKey, key, HystrixEventType.FAILURE);
List <Command> shortCircuited = new List <Command>();
for (int i = 0; i < 20; i++)
{
shortCircuited.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 100));
}
failure1.Execute();
failure2.Execute();
failure3.Execute();
Time.Wait(150);
foreach (Command cmd in shortCircuited)
{
cmd.Observe();
}
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
foreach (Command cmd in shortCircuited)
{
Assert.True(cmd.IsResponseShortCircuited);
}
Assert.Equal(1, stream.LatestRollingMax);
}
public async void TestShortCircuited()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-G");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-G");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-G");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
Command failure1 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0);
Command failure2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0);
Command failure3 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0);
Command shortCircuit1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS);
Command shortCircuit2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 3 failures in a row will trip circuit. let bucket roll once then submit 2 requests.
// should see 3 FAILUREs and 2 SHORT_CIRCUITs and each should see a FALLBACK_SUCCESS
await failure1.Observe();
await failure2.Observe();
await failure3.Observe();
Assert.True(WaitForHealthCountToUpdate(key.Name, 500, output), "health count took to long to update");
output.WriteLine(Time.CurrentTimeMillis + " running failures");
await shortCircuit1.Observe();
await shortCircuit2.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.True(shortCircuit1.IsResponseShortCircuited);
Assert.True(shortCircuit2.IsResponseShortCircuited);
// only the FAILUREs should show up in thread pool counters
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(3, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public async void TestFallbackRejection()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-L");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-L");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-L");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 2000), "Stream failed to start");
// fallback semaphore size is 5. So let 5 commands saturate that semaphore, then
// let 2 more commands go to fallback. they should get rejected by the fallback-semaphore
List <Command> fallbackSaturators = new List <Command>();
for (int i = 0; i < 5; i++)
{
fallbackSaturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 500));
}
Command rejection1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 0);
Command rejection2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 0);
List <Task> tasks = new List <Task>();
foreach (Command saturator in fallbackSaturators)
{
tasks.Add(saturator.ExecuteAsync());
}
await Task.Delay(50);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await rejection1.Observe());
await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await rejection2.Observe());
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 2000, output), "Latch took to long to update");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(7, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-A");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-A");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-A");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(new LatchedObserver(output, latch));
// no writes
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED) + stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-A");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-A");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-A");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
// no writes
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestThreadPoolRejected()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-I");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-I");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-I");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
// 10 commands will saturate threadpools when called concurrently.
// submit 2 more requests and they should be THREADPOOL_REJECTED
// should see 10 SUCCESSes, 2 THREADPOOL_REJECTED and 2 FALLBACK_SUCCESSes
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 700));
}
Command rejected1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
Command rejected2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
foreach (Command c in saturators)
{
c.Observe();
}
rejected1.Observe();
rejected2.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(rejected1.IsResponseThreadPoolRejected);
Assert.True(rejected2.IsResponseThreadPoolRejected);
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(10, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-A");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-A");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-A");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestShortCircuited()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-G");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-G");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-G");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
// 3 failures in a row will trip circuit. let bucket roll once then submit 2 requests.
// should see 3 FAILUREs and 2 SHORT_CIRCUITs and each should see a FALLBACK_SUCCESS
Command failure1 = Command.From(groupKey, key, HystrixEventType.FAILURE, 20);
Command failure2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 20);
Command failure3 = Command.From(groupKey, key, HystrixEventType.FAILURE, 20);
Command shortCircuit1 = Command.From(groupKey, key, HystrixEventType.SUCCESS);
Command shortCircuit2 = Command.From(groupKey, key, HystrixEventType.SUCCESS);
failure1.Observe();
failure2.Observe();
failure3.Observe();
Time.Wait(150);
shortCircuit1.Observe();
shortCircuit2.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(shortCircuit1.IsResponseShortCircuited);
Assert.True(shortCircuit2.IsResponseShortCircuited);
// only the FAILUREs should show up in thread pool counters
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(3, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestSingleTimeout()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-D");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-D");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-D");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(groupKey, key, HystrixEventType.TIMEOUT);
cmd.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public HystrixThreadPoolConfiguration(
IHystrixThreadPoolKey threadPoolKey,
int coreSize,
int maximumSize,
int maxQueueSize,
int queueRejectionThreshold,
int keepAliveTimeInMinutes,
bool allowMaximumSizeToDivergeFromCoreSize,
int rollingCounterNumberOfBuckets,
int rollingCounterBucketSizeInMilliseconds)
{
this.threadPoolKey = threadPoolKey;
this.coreSize = coreSize;
this.maximumSize = maximumSize;
this.maxQueueSize = maxQueueSize;
this.queueRejectionThreshold = queueRejectionThreshold;
this.keepAliveTimeInMinutes = keepAliveTimeInMinutes;
this.allowMaximumSizeToDivergeFromCoreSize = allowMaximumSizeToDivergeFromCoreSize;
this.rollingCounterNumberOfBuckets = rollingCounterNumberOfBuckets;
this.rollingCounterBucketSizeInMilliseconds = rollingCounterBucketSizeInMilliseconds;
}
public async void TestSingleSuccess()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-B");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-B");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-B");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 2000), "Stream failed to start");
Command cmd = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
await cmd.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 2000, output), "Latch took to long to update");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public async void TestSingleBadRequest()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-E");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-E");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-E");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
Command cmd = Command.From(groupKey, key, HystrixEventType.BAD_REQUEST);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
await Assert.ThrowsAsync <HystrixBadRequestException>(async() => await cmd.Observe());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestSingleSuccess()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-B");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-B");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-B");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
cmd.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestStartsAndEndsInSameBucketProduceValue()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-B");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-B");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-B");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
Task t1 = cmd1.ExecuteAsync();
Task t2 = cmd2.ExecuteAsync();
Task.WaitAll(t1, t2);
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(2, stream.LatestRollingMax);
}
public void TestRequestFromCache()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-F");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-F");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-F");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(GetSubscriber(output, latch));
CommandStreamTest.Command cmd1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
CommandStreamTest.Command cmd2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
CommandStreamTest.Command cmd3 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
cmd1.Observe();
cmd2.Observe();
cmd3.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
//RESPONSE_FROM_CACHE should not show up at all in thread pool counters - just the success
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestSingleFailure()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-C");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-C");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-C");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(groupKey, key, HystrixEventType.FAILURE, 20);
cmd.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
}
public HystrixCommand(IHystrixCommandGroupKey group, IHystrixCommandKey key, IHystrixThreadPoolKey threadPoolKey, IHystrixCircuitBreaker circuitBreaker, IHystrixThreadPool threadPool,
IHystrixCommandOptions commandOptionsDefaults, IHystrixThreadPoolOptions threadPoolOptionsDefaults, HystrixCommandMetrics metrics, SemaphoreSlim fallbackSemaphore, SemaphoreSlim executionSemaphore,
HystrixOptionsStrategy optionsStrategy, HystrixCommandExecutionHook executionHook, Action run, Action fallback, ILogger logger = null) :
base(group, key, threadPoolKey, circuitBreaker, threadPool, commandOptionsDefaults, threadPoolOptionsDefaults, metrics, fallbackSemaphore, executionSemaphore, optionsStrategy, executionHook, null, null, logger)
{
if (run == null)
{
_run = () => Run();
}
else
{
_run = run;
}
if (fallback == null)
{
_fallback = () => RunFallback();
}
else
{
_fallback = fallback;
}
}
private static void SerializeUtilization(JsonTextWriter json, HystrixUtilization utilization)
{
json.WriteStartObject();
json.WriteStringField("type", "HystrixUtilization");
json.WriteObjectFieldStart("commands");
foreach (var entry in utilization.CommandUtilizationMap)
{
IHystrixCommandKey key = entry.Key;
HystrixCommandUtilization commandUtilization = entry.Value;
WriteCommandUtilizationJson(json, key, commandUtilization);
}
json.WriteEndObject();
json.WriteObjectFieldStart("threadpools");
foreach (var entry in utilization.ThreadPoolUtilizationMap)
{
IHystrixThreadPoolKey threadPoolKey = entry.Key;
HystrixThreadPoolUtilization threadPoolUtilization = entry.Value;
WriteThreadPoolUtilizationJson(json, threadPoolKey, threadPoolUtilization);
}
json.WriteEndObject();
json.WriteEndObject();
}
public void TestStartsAndEndsInSameBucketProduceValue()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-B");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-B");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-B");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
cmd1.Observe();
Time.Wait(1);
cmd2.Observe();
Assert.True(latch.Wait(10000));
Assert.Equal(2, stream.LatestRollingMax);
}
public void TestMultipleCommandsCarryOverMultipleBucketsForMultipleThreadPools()
{
IHystrixCommandGroupKey groupKeyX = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-X");
IHystrixCommandGroupKey groupKeyY = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-Y");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-X");
IHystrixCommandKey keyX = HystrixCommandKeyDefault.AsKey("RollingConcurrency-X");
IHystrixCommandKey keyY = HystrixCommandKeyDefault.AsKey("RollingConcurrency-Y");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Take(10 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd1 = Command.From(groupKeyX, keyX, HystrixEventType.SUCCESS, 300);
Command cmd2 = Command.From(groupKeyY, keyY, HystrixEventType.SUCCESS, 300);
Command cmd3 = Command.From(groupKeyX, keyY, HystrixEventType.SUCCESS, 10);
Command cmd4 = Command.From(groupKeyX, keyY, HystrixEventType.SUCCESS, 10);
Task t1 = cmd1.ExecuteAsync();
// Time.Wait(100); // bucket roll
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t2 = cmd2.ExecuteAsync();
// Time.Wait(100); // bucket roll
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t3 = cmd3.ExecuteAsync();
// Time.Wait(100); // bucket roll
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t4 = cmd4.ExecuteAsync();
Task.WaitAll(t1, t2, t3, t4);
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Assert.Equal(2, stream.LatestRollingMax);
}
public void TestMultipleEventsOverTimeGetStoredAndDoNotAgeOut()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-M");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-M");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-M");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(30).Subscribe(new LatchedObserver(output, latch));
CommandStreamTest.Command cmd1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
CommandStreamTest.Command cmd2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 10);
cmd1.Observe();
cmd2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
//all commands should have aged out
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public static HystrixCommandConfiguration Sample(
IHystrixCommandKey commandKey,
IHystrixThreadPoolKey threadPoolKey,
IHystrixCommandGroupKey groupKey,
IHystrixCommandOptions commandProperties)
{
var executionConfig = new HystrixCommandExecutionConfig(
commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests,
commandProperties.ExecutionIsolationStrategy,
false,
commandProperties.ExecutionIsolationThreadPoolKeyOverride,
commandProperties.ExecutionTimeoutEnabled,
commandProperties.ExecutionTimeoutInMilliseconds,
commandProperties.FallbackEnabled,
commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests,
commandProperties.RequestCacheEnabled,
commandProperties.RequestLogEnabled);
var circuitBreakerConfig = new HystrixCommandCircuitBreakerConfig(
commandProperties.CircuitBreakerEnabled,
commandProperties.CircuitBreakerErrorThresholdPercentage,
commandProperties.CircuitBreakerForceClosed,
commandProperties.CircuitBreakerForceOpen,
commandProperties.CircuitBreakerRequestVolumeThreshold,
commandProperties.CircuitBreakerSleepWindowInMilliseconds);
var metricsConfig = new HystrixCommandMetricsConfig(
commandProperties.MetricsHealthSnapshotIntervalInMilliseconds,
commandProperties.MetricsRollingPercentileEnabled,
commandProperties.MetricsRollingPercentileWindowBuckets,
commandProperties.MetricsRollingPercentileWindowInMilliseconds,
commandProperties.MetricsRollingStatisticalWindowBuckets,
commandProperties.MetricsRollingStatisticalWindowInMilliseconds);
return(new HystrixCommandConfiguration(
commandKey, threadPoolKey, groupKey, executionConfig, circuitBreakerConfig, metricsConfig));
}
public void TestStartsAndEndsInSameBucketSemaphoreIsolated()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-C");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-C");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-C");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10, ExecutionIsolationStrategy.SEMAPHORE);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 14, ExecutionIsolationStrategy.SEMAPHORE);
Task t1 = cmd1.ExecuteAsync();
Task t2 = cmd2.ExecuteAsync();
Task.WaitAll(t1, t2);
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
// since commands run in semaphore isolation, they are not tracked by threadpool metrics
Assert.Equal(0, stream.LatestRollingMax);
}
public override IHystrixMetricsPublisherThreadPool GetMetricsPublisherForThreadPool(IHystrixThreadPoolKey threadPoolKey, HystrixThreadPoolMetrics metrics, IHystrixThreadPoolOptions properties)
{
return(new MyHystrixMetricsPublisherThreadPool(threadCounter));
}
public HystrixCommandExecutionStarted(IHystrixCommandKey commandKey, IHystrixThreadPoolKey threadPoolKey, ExecutionIsolationStrategy isolationStrategy, int currentConcurrency)
: base(commandKey, threadPoolKey)
{
this.isolationStrategy = isolationStrategy;
this.currentConcurrency = currentConcurrency;
}
public override string GetThreadPoolOptionsCacheKey(IHystrixThreadPoolKey threadPoolKey, IHystrixThreadPoolOptions builder)
{
return(null);
}
public static HystrixCommandCompletion From(ExecutionResult executionResult, IHystrixCommandKey commandKey, IHystrixThreadPoolKey threadPoolKey, HystrixRequestContext requestContext)
{
return(new HystrixCommandCompletion(executionResult, commandKey, threadPoolKey, requestContext));
}
public static HystrixCommandCompletion From(ExecutionResult executionResult, IHystrixCommandKey commandKey, IHystrixThreadPoolKey threadPoolKey)
{
return(From(executionResult, commandKey, threadPoolKey, HystrixRequestContext.ContextForCurrentThread));
}
