public HystrixDashboardStream.DashboardData GetTestData()
{
var commandKey = new HystrixCommandKeyDefault("command");
var tpKey = new HystrixThreadPoolKeyDefault("threadPool");
var collapserKey = new HystrixCollapserKeyDefault("collapser");
var commandMetric = new HystrixCommandMetrics(
commandKey,
new HystrixCommandGroupKeyDefault("group"),
tpKey,
new HystrixCommandOptions(),
HystrixEventNotifierDefault.GetInstance());
var threadPoolMetric = HystrixThreadPoolMetrics.GetInstance(
tpKey,
new HystrixSyncTaskScheduler(new HystrixThreadPoolOptions()),
new HystrixThreadPoolOptions());
var commandMetrics = new List <HystrixCommandMetrics>()
{
commandMetric
};
var collapserOptions = new HystrixCollapserOptions(collapserKey);
var threadPoolMetrics = new List <HystrixThreadPoolMetrics>()
{
threadPoolMetric
};
var collapserMetrics = new List <HystrixCollapserMetrics>()
{
HystrixCollapserMetrics.GetInstance(collapserKey, collapserOptions)
};
return(new HystrixDashboardStream.DashboardData(commandMetrics, threadPoolMetrics, collapserMetrics));
}
public void TestMultipleEventsOverTimeGetStoredAndAgeOut()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-M");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-M");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-M");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 250);
stream.StartCachingStreamValuesIfUnstarted();
// by doing a take(20), we ensure that all rolling counts go back to 0
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(20).Subscribe(GetSubscriber(output, latch));
CommandStreamTest.Command cmd1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
CommandStreamTest.Command cmd2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 10);
cmd1.Observe();
cmd2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
// all commands should have aged out
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestSemaphoreRejected()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-H");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-H");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-H");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
//10 commands will saturate semaphore when called from different threads.
//submit 2 more requests and they should be SEMAPHORE_REJECTED
//should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 300, ExecutionIsolationStrategy.SEMAPHORE));
}
CommandStreamTest.Command rejected1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
CommandStreamTest.Command rejected2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
foreach (Command c in saturators)
{
Task t = new Task(() =>
{
c.Observe();
}, CancellationToken.None, TaskCreationOptions.LongRunning);
t.Start();
Task.Run(() => c.Observe());
}
Time.Wait(10);
Task.Run(() => rejected1.Observe());
Task.Run(() => rejected2.Observe());
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(rejected1.IsResponseSemaphoreRejected);
Assert.True(rejected2.IsResponseSemaphoreRejected);
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestSetNeitherCoreNorMaximumSize()
{
HystrixThreadPoolOptions properties = new HystrixThreadPoolOptions(HystrixThreadPoolKeyDefault.AsKey("TEST"));
Assert.Equal(HystrixThreadPoolOptions.Default_CoreSize, properties.CoreSize);
Assert.Equal(HystrixThreadPoolOptions.Default_MaximumSize, properties.MaximumSize);
}
public void EnsureThreadPoolInstanceIsTheOneRegisteredWithMetricsPublisherAndThreadPoolCache()
{
HystrixPlugins.RegisterMetricsPublisher(new MyHystrixMetricsPublisher());
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("threadPoolFactoryConcurrencyTest");
IHystrixThreadPool poolOne = new HystrixThreadPoolDefault(
threadPoolKey, HystrixThreadPoolOptionsTest.GetUnitTestPropertiesBuilder());
IHystrixThreadPool poolTwo = new HystrixThreadPoolDefault(
threadPoolKey, HystrixThreadPoolOptionsTest.GetUnitTestPropertiesBuilder());
Assert.Equal(poolOne.GetScheduler(), poolTwo.GetScheduler()); // Now that we get the threadPool from the metrics object, this will always be equal
HystrixMetricsPublisherThreadPoolContainer hystrixMetricsPublisherThreadPool =
(HystrixMetricsPublisherThreadPoolContainer)HystrixMetricsPublisherFactory
.CreateOrRetrievePublisherForThreadPool(threadPoolKey, null, null);
IHystrixTaskScheduler threadPoolExecutor = hystrixMetricsPublisherThreadPool.HystrixThreadPoolMetrics.TaskScheduler;
// assert that both HystrixThreadPools share the same ThreadPoolExecutor as the one in HystrixMetricsPublisherThreadPool
Assert.True(threadPoolExecutor.Equals(poolOne.GetScheduler()) && threadPoolExecutor.Equals(poolTwo.GetScheduler()));
Assert.False(threadPoolExecutor.IsShutdown);
// Now the HystrixThreadPool ALWAYS has the same reference to the ThreadPoolExecutor so that it no longer matters which
// wins to be inserted into the HystrixThreadPool.Factory.threadPools cache.
poolOne.Dispose();
poolTwo.Dispose();
}
public async Task TestMultipleEventsOverTimeGetStoredAndDoNotAgeOut()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-M");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-M");
var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-M");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
var cmd2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 10);
latchSubscription = stream.Observe().Take(20 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
await cmd1.Observe();
await cmd2.Observe();
Assert.True(latch.Wait(20000), "CountdownEvent was not set!");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
// all commands should not have aged out
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestMultipleCommandsCarryOverMultipleBucketsAndThenAgeOut()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-F");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-F");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-F");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Take(20 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
var cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
var cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
var cmd4 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
Task t1 = cmd1.ExecuteAsync();
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t2 = cmd2.ExecuteAsync();
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t3 = cmd3.ExecuteAsync();
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t4 = cmd4.ExecuteAsync();
Task.WaitAll(t1, t2, t3, t4);
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(0, stream.LatestRollingMax);
}
public static Command From(IHystrixCommandGroupKey groupKey, IHystrixCommandKey key, HystrixEventType desiredEventType, int latency,
ExecutionIsolationStrategy isolationStrategy,
HystrixEventType desiredFallbackEventType, int fallbackLatency)
{
HystrixThreadPoolOptions topts = new HystrixThreadPoolOptions()
{
CoreSize = 10,
MaxQueueSize = -1,
ThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey(groupKey.Name)
};
HystrixCommandOptions setter = new HystrixCommandOptions()
{
GroupKey = groupKey,
CommandKey = key,
ExecutionTimeoutInMilliseconds = 600,
ExecutionIsolationStrategy = isolationStrategy,
CircuitBreakerEnabled = true,
CircuitBreakerRequestVolumeThreshold = 3,
MetricsHealthSnapshotIntervalInMilliseconds = 100,
MetricsRollingStatisticalWindowInMilliseconds = 1000,
MetricsRollingStatisticalWindowBuckets = 10,
RequestCacheEnabled = true,
RequestLogEnabled = true,
FallbackIsolationSemaphoreMaxConcurrentRequests = 5,
ThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey(groupKey.Name),
ThreadPoolOptions = topts
};
String uniqueArg;
switch (desiredEventType)
{
case HystrixEventType.SUCCESS:
uniqueArg = uniqueId.IncrementAndGet() + "";
return(new Command(setter, HystrixEventType.SUCCESS, latency, uniqueArg, desiredFallbackEventType, 0));
case HystrixEventType.FAILURE:
uniqueArg = uniqueId.IncrementAndGet() + "";
return(new Command(setter, HystrixEventType.FAILURE, latency, uniqueArg, desiredFallbackEventType, fallbackLatency));
case HystrixEventType.TIMEOUT:
uniqueArg = uniqueId.IncrementAndGet() + "";
return(new Command(setter, HystrixEventType.SUCCESS, 1000, uniqueArg, desiredFallbackEventType, fallbackLatency));
case HystrixEventType.BAD_REQUEST:
uniqueArg = uniqueId.IncrementAndGet() + "";
return(new Command(setter, HystrixEventType.BAD_REQUEST, latency, uniqueArg, desiredFallbackEventType, 0));
case HystrixEventType.RESPONSE_FROM_CACHE:
String arg = uniqueId.Value + "";
return(new Command(setter, HystrixEventType.SUCCESS, 0, arg, desiredFallbackEventType, 0));
default:
throw new Exception("not supported yet");
}
}
public void TestConcurrencyStreamProperlyFiltersOutResponseFromCache()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-G");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-G");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-G");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
var cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
var cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
var cmd4 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
cmd1.Execute();
cmd2.Execute();
cmd3.Execute();
cmd4.Execute();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.True(cmd2.IsResponseFromCache);
Assert.True(cmd3.IsResponseFromCache);
Assert.True(cmd4.IsResponseFromCache);
Assert.Equal(1, stream.LatestRollingMax);
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-A");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-A");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-A");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
// no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestMultipleCommandsCarryOverMultipleBucketsAndThenAgeOut()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-F");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-F");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-F");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(30).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd4 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
cmd1.Observe();
Time.Wait(100); // bucket roll
cmd2.Observe();
Time.Wait(100);
cmd3.Observe();
Time.Wait(100);
cmd4.Observe();
Assert.True(latch.Wait(10000));
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestConcurrencyStreamProperlyFiltersOutResponseFromCache()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-G");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-G");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-G");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
Command cmd4 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
cmd1.Observe();
Time.Wait(5);
cmd2.Observe();
cmd3.Observe();
cmd4.Observe();
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(cmd2.IsResponseFromCache);
Assert.True(cmd3.IsResponseFromCache);
Assert.True(cmd4.IsResponseFromCache);
Assert.Equal(1, stream.LatestRollingMax);
}
public void TestConcurrencyStreamProperlyFiltersOutSemaphoreRejections()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-I");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-I");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-I");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
//10 commands executed concurrently on different caller threads should saturate semaphore
//once these are in-flight, execute 10 more concurrently on new caller threads.
//since these are semaphore-rejected, the max concurrency should be 10
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 400, ExecutionIsolationStrategy.SEMAPHORE));
}
List <Command> rejected = new List <Command>();
for (int i = 0; i < 10; i++)
{
rejected.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 100, ExecutionIsolationStrategy.SEMAPHORE));
}
foreach (Command saturatingCmd in saturators)
{
Task t = new Task(() =>
{
saturatingCmd.Observe();
}, CancellationToken.None, TaskCreationOptions.LongRunning);
t.Start();
}
Time.Wait(30);
foreach (Command rejectedCmd in rejected)
{
Task.Run(() => rejectedCmd.Observe());
}
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
foreach (Command rejectedCmd in rejected)
{
Assert.True(rejectedCmd.IsResponseSemaphoreRejected || rejectedCmd.IsResponseShortCircuited);
}
//should be 0 since all are executed in a semaphore
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestStartsAndEndsInSameBucketSemaphoreIsolated()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-C");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-C");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-C");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10, ExecutionIsolationStrategy.SEMAPHORE);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 14, ExecutionIsolationStrategy.SEMAPHORE);
cmd1.Observe();
Time.Wait(1);
cmd2.Observe();
Assert.True(latch.Wait(10000));
// since commands run in semaphore isolation, they are not tracked by threadpool metrics
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestStartsAndEndsInSameBucketProduceValue()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-B");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-B");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-B");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
cmd1.Observe();
Time.Wait(1);
cmd2.Observe();
Assert.True(latch.Wait(10000));
Assert.Equal(2, stream.LatestRollingMax);
}
public void TestOneCommandCarriesOverToNextBucket()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-D");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-D");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-D");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 560);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 75);
cmd1.Observe();
Time.Wait(150); // bucket roll
cmd2.Observe();
Time.Wait(1);
cmd3.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(3, stream.LatestRollingMax);
}
public void TestRequestFromCache()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-F");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-F");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-F");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
cmd1.Observe();
cmd2.Observe();
cmd3.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
// RESPONSE_FROM_CACHE should not show up at all in thread pool counters - just the success
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestSingleBadRequest()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-E");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-E");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-E");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(groupKey, key, HystrixEventType.BAD_REQUEST);
cmd.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-A");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-A");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-A");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
// no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestFallbackMissing()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-K");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-K");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-K");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_MISSING);
cmd.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestThreadPoolRejected()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-I");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-I");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-I");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(GetSubscriber(output, latch));
// 10 commands will saturate threadpools when called concurrently.
// submit 2 more requests and they should be THREADPOOL_REJECTED
// should see 10 SUCCESSes, 2 THREADPOOL_REJECTED and 2 FALLBACK_SUCCESSes
List <CommandStreamTest.Command> saturators = new List <CommandStreamTest.Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 200));
}
CommandStreamTest.Command rejected1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
CommandStreamTest.Command rejected2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
foreach (CommandStreamTest.Command saturator in saturators)
{
saturator.Observe();
}
try
{
Time.Wait(100);
}
catch (Exception ie)
{
Assert.False(true, ie.Message);
}
rejected1.Observe();
rejected2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(rejected1.IsResponseThreadPoolRejected);
Assert.True(rejected2.IsResponseThreadPoolRejected);
// none of these got executed on a thread-pool, so thread pool metrics should be 0
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(10, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestMultipleEventsOverTimeGetStoredAndDoNotAgeOut()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-M");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-M");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-M");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(30).Subscribe(new LatchedObserver(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 10);
cmd1.Observe();
cmd2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
// all commands should have aged out
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestFallbackMissing()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-K");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-K");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-K");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(GetSubscriber(output, latch));
CommandStreamTest.Command cmd = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_MISSING);
cmd.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestOneCommandCarriesOverToNextBucket()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-D");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-D");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-D");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 560);
var cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
var cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 75);
Task t1 = cmd1.ExecuteAsync();
// Time.Wait(150); // bucket roll
Assert.True(WaitForObservableToUpdate(stream.Observe(), 1, 500, output), "Stream update took to long");
Task t2 = cmd2.ExecuteAsync();
Task t3 = cmd3.ExecuteAsync();
Task.WaitAll(t1, t2, t3);
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(3, stream.LatestRollingMax);
}
public async Task TestRequestFromCache()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-F");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-F");
var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-F");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
var cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
var cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
await cmd1.Observe();
await cmd2.Observe();
await cmd3.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
// RESPONSE_FROM_CACHE should not show up at all in thread pool counters - just the success
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestSemaphoreRejected()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-H");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-H");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-H");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(new LatchedObserver(output, latch));
// 10 commands will saturate semaphore when called from different threads.
// submit 2 more requests and they should be SEMAPHORE_REJECTED
// should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 500, ExecutionIsolationStrategy.SEMAPHORE));
}
Command rejected1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
Command rejected2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
foreach (Command saturator in saturators)
{
Task t = new Task(
() =>
{
saturator.Observe();
}, CancellationToken.None,
TaskCreationOptions.LongRunning);
t.Start();
}
try
{
Time.Wait(100);
}
catch (Exception ie)
{
Assert.False(true, ie.Message);
}
rejected1.Observe();
rejected2.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(rejected1.IsResponseSemaphoreRejected);
Assert.True(rejected2.IsResponseSemaphoreRejected);
// none of these got executed on a thread-pool, so thread pool metrics should be 0
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestFallbackRejection()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-L");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-L");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-L");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
// fallback semaphore size is 5. So let 5 commands saturate that semaphore, then
// let 2 more commands go to fallback. they should get rejected by the fallback-semaphore
List <CommandStreamTest.Command> fallbackSaturators = new List <CommandStreamTest.Command>();
for (int i = 0; i < 5; i++)
{
fallbackSaturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 400));
}
Command rejection1 = Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0);
Command rejection2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0);
foreach (Command saturator in fallbackSaturators)
{
saturator.Observe();
}
try
{
Time.Wait(70);
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
rejection1.Observe();
rejection2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
// all 7 commands executed on-thread, so should be executed according to thread-pool metrics
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(7, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestSetCoreSizeEqualToMaximumSize()
{
var properties = new HystrixThreadPoolOptions(HystrixThreadPoolKeyDefault.AsKey("TEST"), new HystrixThreadPoolOptions()
{
CoreSize = 7, MaximumSize = 7
});
Assert.Equal(7, properties.CoreSize);
Assert.Equal(7, properties.MaximumSize);
}
public void TestSetCoreSizeLessThanMaximumSize()
{
var properties = new HystrixThreadPoolOptions(HystrixThreadPoolKeyDefault.AsKey("TEST"), new HystrixThreadPoolOptions()
{
CoreSize = 2, MaximumSize = 8
});
Assert.Equal(2, properties.CoreSize);
Assert.Equal(8, properties.MaximumSize);
}
public void TestSetMaximumSizeOnlyGreaterThanDefaultCoreSize()
{
var properties = new HystrixThreadPoolOptions(HystrixThreadPoolKeyDefault.AsKey("TEST"), new HystrixThreadPoolOptions()
{
MaximumSize = 21
});
Assert.Equal(HystrixThreadPoolOptions.Default_CoreSize, properties.CoreSize);
Assert.Equal(21, properties.MaximumSize);
}
