public async void TestSingleTimeout()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-D");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = HealthCountsStream.GetInstance(key, 10, 100);
Command cmd = Command.From(groupKey, key, HystrixEventType.TIMEOUT); // Timeout 1000
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
await cmd.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(1L, stream.Latest.ErrorCount);
Assert.Equal(1L, stream.Latest.TotalRequests);
}
public void TestFallbackFailure()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-J");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-J");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-J");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_FAILURE);
cmd.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestSingleBadRequest()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-CumulativeCounter-E");
stream = CumulativeCommandEventCounterStream.GetInstance(key, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(GroupKey, key, HystrixEventType.BAD_REQUEST);
cmd.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.BAD_REQUEST] = 1;
expected[(int)HystrixEventType.EXCEPTION_THROWN] = 1;
Assert.Equal(expected, stream.Latest);
}
public async void TestSingleBucketGetsStored()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-B");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.TIMEOUT); // latency = 600
await cmd1.Observe();
await cmd2.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
AssertBetween(100, 400, stream.LatestMean);
AssertBetween(10, 100, stream.GetLatestPercentile(0.0));
AssertBetween(300, 800, stream.GetLatestPercentile(100.0));
}
public void TestStartsAndEndsInSameBucketProduceValue()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-B");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 100);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 100);
cmd1.Observe();
Time.Wait(1);
cmd2.Observe();
Assert.True(latch.Wait(10000));
Assert.Equal(2, stream.LatestRollingMax);
}
public void TestResponseFromCacheDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-G");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// should get 1 SUCCESS and 1 RESPONSE_FROM_CACHE
List <Command> commands = Command.GetCommandsWithResponseFromCache(GroupKey, key);
foreach (Command cmd in commands)
{
_ = cmd.ExecuteAsync();
}
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(1, stream.Latest.GetTotalCount());
AssertBetween(0, 75, stream.LatestMean);
}
public async void TestSingleBadRequest()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-CumulativeCounter-E");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeCommandEventCounterStream.GetInstance(key, 10, 500);
Command cmd = Command.From(GroupKey, key, HystrixEventType.BAD_REQUEST);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 2000), "Stream failed to start");
await Assert.ThrowsAsync <HystrixBadRequestException>(async() => await cmd.Observe());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 2000, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.BAD_REQUEST] = 1;
expected[(int)HystrixEventType.EXCEPTION_THROWN] = 1;
Assert.Equal(expected, stream.Latest);
}
public async void TestSingleSuccess()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-CumulativeCounter-B");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeCommandEventCounterStream.GetInstance(key, 10, 500); // Stream should start
Command cmd = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 2000), "Stream failed to start");
await cmd.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 2000, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)(int)HystrixEventType.SUCCESS] = 1;
Assert.Equal(expected, stream.Latest);
}
public void TestShortCircuited()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-CumulativeCounter-G");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeCommandEventCounterStream.GetInstance(key, 10, 500);
Command failure1 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0);
Command failure2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0);
Command failure3 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0);
Command shortCircuit1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
Command shortCircuit2 = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 2000), "Stream failed to start");
// 3 failures in a row will trip circuit. let bucket roll once then submit 2 requests.
// should see 3 FAILUREs and 2 SHORT_CIRCUITs and then 5 FALLBACK_SUCCESSes
failure1.Execute();
failure2.Execute();
failure3.Execute();
Assert.True(WaitForHealthCountToUpdate(key.Name, 500, output), "health count took to long to update");
output.WriteLine(Time.CurrentTimeMillis + " running failures");
shortCircuit1.Execute();
shortCircuit2.Execute();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 2000, output), "Latch took to long to update");
Assert.True(shortCircuit1.IsResponseShortCircuited);
Assert.True(shortCircuit2.IsResponseShortCircuited);
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.FAILURE] = 3;
expected[(int)HystrixEventType.SHORT_CIRCUITED] = 2;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 5;
Assert.Equal(expected, stream.Latest);
}
public async void TestFallbackMissing()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-K");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-K");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-K");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 2000), "Stream failed to start");
Command cmd = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_MISSING);
await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await cmd.Observe());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 2000, output), "Latch took to long to update");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestRequestFromCache()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-F");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-F");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-F");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(GetSubscriber(output, latch));
CommandStreamTest.Command cmd1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
CommandStreamTest.Command cmd2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
CommandStreamTest.Command cmd3 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
cmd1.Observe();
cmd2.Observe();
cmd3.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
//RESPONSE_FROM_CACHE should not show up at all in thread pool counters - just the success
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestEmptyStreamProducesEmptyDistributions()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-A");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + DateTime.Now.Ticks / 10000 + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + Thread.CurrentThread.ManagedThreadId);
Assert.Equal(0, distribution.GetTotalCount());
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
//no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(0, stream.Latest.GetTotalCount());
}
public async void TestSemaphoreRejectedCommandDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-F");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 10 commands with latency should occupy all semaphores. execute those, then wait for bucket to roll
// next command should be a semaphore rejection
List <Command> commands = new List <Command>();
for (int i = 0; i < 10; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 500, ExecutionIsolationStrategy.SEMAPHORE));
}
Command semaphoreRejected = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
List <Task> satTasks = new List <Task>();
foreach (Command saturator in commands)
{
satTasks.Add(Task.Run(() => saturator.Execute()));
}
await Task.Delay(50);
await Task.Run(() => semaphoreRejected.Execute());
Task.WaitAll(satTasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(10, stream.Latest.GetTotalCount());
AssertBetween(500, 750, stream.LatestMean);
Assert.True(semaphoreRejected.IsResponseSemaphoreRejected);
}
private static void WriteCommandConfigJson(JsonTextWriter json, IHystrixCommandKey key, HystrixCommandConfiguration commandConfig)
{
json.WriteObjectFieldStart(key.Name);
json.WriteStringField("threadPoolKey", commandConfig.ThreadPoolKey.Name);
json.WriteStringField("groupKey", commandConfig.GroupKey.Name);
json.WriteObjectFieldStart("execution");
HystrixCommandConfiguration.HystrixCommandExecutionConfig executionConfig = commandConfig.ExecutionConfig;
json.WriteStringField("isolationStrategy", executionConfig.IsolationStrategy.ToString());
json.WriteStringField("threadPoolKeyOverride", executionConfig.ThreadPoolKeyOverride);
json.WriteBooleanField("requestCacheEnabled", executionConfig.IsRequestCacheEnabled);
json.WriteBooleanField("requestLogEnabled", executionConfig.IsRequestLogEnabled);
json.WriteBooleanField("timeoutEnabled", executionConfig.IsTimeoutEnabled);
json.WriteBooleanField("fallbackEnabled", executionConfig.IsFallbackEnabled);
json.WriteIntegerField("timeoutInMilliseconds", executionConfig.TimeoutInMilliseconds);
json.WriteIntegerField("semaphoreSize", executionConfig.SemaphoreMaxConcurrentRequests);
json.WriteIntegerField("fallbackSemaphoreSize", executionConfig.FallbackMaxConcurrentRequest);
json.WriteBooleanField("threadInterruptOnTimeout", executionConfig.IsThreadInterruptOnTimeout);
json.WriteEndObject();
json.WriteObjectFieldStart("metrics");
HystrixCommandConfiguration.HystrixCommandMetricsConfig metricsConfig = commandConfig.MetricsConfig;
json.WriteIntegerField("healthBucketSizeInMs", metricsConfig.HealthIntervalInMilliseconds);
json.WriteIntegerField("percentileBucketSizeInMilliseconds", metricsConfig.RollingPercentileBucketSizeInMilliseconds);
json.WriteIntegerField("percentileBucketCount", metricsConfig.RollingCounterNumberOfBuckets);
json.WriteBooleanField("percentileEnabled", metricsConfig.IsRollingPercentileEnabled);
json.WriteIntegerField("counterBucketSizeInMilliseconds", metricsConfig.RollingCounterBucketSizeInMilliseconds);
json.WriteIntegerField("counterBucketCount", metricsConfig.RollingCounterNumberOfBuckets);
json.WriteEndObject();
json.WriteObjectFieldStart("circuitBreaker");
HystrixCommandConfiguration.HystrixCommandCircuitBreakerConfig circuitBreakerConfig = commandConfig.CircuitBreakerConfig;
json.WriteBooleanField("enabled", circuitBreakerConfig.IsEnabled);
json.WriteBooleanField("isForcedOpen", circuitBreakerConfig.IsForceOpen);
json.WriteBooleanField("isForcedClosed", circuitBreakerConfig.IsForceOpen);
json.WriteIntegerField("requestVolumeThreshold", circuitBreakerConfig.RequestVolumeThreshold);
json.WriteIntegerField("errorPercentageThreshold", circuitBreakerConfig.ErrorThresholdPercentage);
json.WriteIntegerField("sleepInMilliseconds", circuitBreakerConfig.SleepWindowInMilliseconds);
json.WriteEndObject();
json.WriteEndObject();
}
public void TestRequestFromCache()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-F");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(this.output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
cmd1.Observe();
cmd2.Observe();
cmd3.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 1;
expected[(int)HystrixEventType.RESPONSE_FROM_CACHE] = 2;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal <long[]>(expected, stream.Latest);
}
public void TestSingleFailure()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-C");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-C");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-C");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(groupKey, key, HystrixEventType.FAILURE, 20);
cmd.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
}
public async void TestMultipleEventsOverTimeGetStoredAndAgeOut()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-M");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Take(30 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 20);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 10);
await cmd1.Observe();
await cmd2.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
Assert.Equal(expected, stream.Latest);
}
public async void TestFallbackFailure()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-J");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_FAILURE);
await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await cmd.Observe());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.FAILURE] = 1;
expected[(int)HystrixEventType.FALLBACK_FAILURE] = 1;
expected[(int)HystrixEventType.EXCEPTION_THROWN] = 1;
Assert.Equal(expected, stream.Latest);
}
private static void SerializeUtilization(JsonTextWriter json, HystrixUtilization utilization)
{
json.WriteStartObject();
json.WriteStringField("type", "HystrixUtilization");
json.WriteObjectFieldStart("commands");
foreach (var entry in utilization.CommandUtilizationMap)
{
IHystrixCommandKey key = entry.Key;
HystrixCommandUtilization commandUtilization = entry.Value;
WriteCommandUtilizationJson(json, key, commandUtilization);
}
json.WriteEndObject();
json.WriteObjectFieldStart("threadpools");
foreach (var entry in utilization.ThreadPoolUtilizationMap)
{
IHystrixThreadPoolKey threadPoolKey = entry.Key;
HystrixThreadPoolUtilization threadPoolUtilization = entry.Value;
WriteThreadPoolUtilizationJson(json, threadPoolKey, threadPoolUtilization);
}
json.WriteEndObject();
json.WriteEndObject();
}
public HystrixCommand(IHystrixCommandGroupKey group, IHystrixCommandKey key, IHystrixThreadPoolKey threadPoolKey, IHystrixCircuitBreaker circuitBreaker, IHystrixThreadPool threadPool,
IHystrixCommandOptions commandOptionsDefaults, IHystrixThreadPoolOptions threadPoolOptionsDefaults, HystrixCommandMetrics metrics, SemaphoreSlim fallbackSemaphore, SemaphoreSlim executionSemaphore,
HystrixOptionsStrategy optionsStrategy, HystrixCommandExecutionHook executionHook, Action run, Action fallback, ILogger logger = null) :
base(group, key, threadPoolKey, circuitBreaker, threadPool, commandOptionsDefaults, threadPoolOptionsDefaults, metrics, fallbackSemaphore, executionSemaphore, optionsStrategy, executionHook, null, null, logger)
{
if (run == null)
{
_run = () => Run();
}
else
{
_run = run;
}
if (fallback == null)
{
_fallback = () => RunFallback();
}
else
{
_fallback = fallback;
}
}
public void TestStartsAndEndsInSameBucketProduceValue()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-B");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-B");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-B");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
cmd1.Observe();
Time.Wait(1);
cmd2.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(2, stream.LatestRollingMax);
}
public async void TestSingleFailure()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-C");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd = Command.From(GroupKey, key, HystrixEventType.FAILURE, 20);
await cmd.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.FAILURE] = 1;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 1;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(expected, stream.Latest);
}
public void TestSingleTimeout()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-D");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 600);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(GroupKey, key, HystrixEventType.TIMEOUT);
cmd.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.TIMEOUT] = 1;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 1;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(expected, stream.Latest);
}
public void TestResponseFromCacheDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-G");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
// should get 1 SUCCESS and 1 RESPONSE_FROM_CACHE
List <Command> commands = Command.GetCommandsWithResponseFromCache(GroupKey, key);
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
Assert.True(distribution.GetTotalCount() <= 1);
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
foreach (Command cmd in commands)
{
cmd.Observe();
}
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(1, stream.Latest.GetTotalCount());
AssertBetween(0, 30, stream.LatestMean);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
}
public void TestMultipleEventsOverTimeGetStoredAndDoNotAgeOut()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-M");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-M");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-M");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(30).Subscribe(new LatchedObserver(output, latch));
CommandStreamTest.Command cmd1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
CommandStreamTest.Command cmd2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 10);
cmd1.Observe();
cmd2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
//all commands should have aged out
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public static HystrixCommandConfiguration Sample(
IHystrixCommandKey commandKey,
IHystrixThreadPoolKey threadPoolKey,
IHystrixCommandGroupKey groupKey,
IHystrixCommandOptions commandProperties)
{
var executionConfig = new HystrixCommandExecutionConfig(
commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests,
commandProperties.ExecutionIsolationStrategy,
false,
commandProperties.ExecutionIsolationThreadPoolKeyOverride,
commandProperties.ExecutionTimeoutEnabled,
commandProperties.ExecutionTimeoutInMilliseconds,
commandProperties.FallbackEnabled,
commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests,
commandProperties.RequestCacheEnabled,
commandProperties.RequestLogEnabled);
var circuitBreakerConfig = new HystrixCommandCircuitBreakerConfig(
commandProperties.CircuitBreakerEnabled,
commandProperties.CircuitBreakerErrorThresholdPercentage,
commandProperties.CircuitBreakerForceClosed,
commandProperties.CircuitBreakerForceOpen,
commandProperties.CircuitBreakerRequestVolumeThreshold,
commandProperties.CircuitBreakerSleepWindowInMilliseconds);
var metricsConfig = new HystrixCommandMetricsConfig(
commandProperties.MetricsHealthSnapshotIntervalInMilliseconds,
commandProperties.MetricsRollingPercentileEnabled,
commandProperties.MetricsRollingPercentileWindowBuckets,
commandProperties.MetricsRollingPercentileWindowInMilliseconds,
commandProperties.MetricsRollingStatisticalWindowBuckets,
commandProperties.MetricsRollingStatisticalWindowInMilliseconds);
return(new HystrixCommandConfiguration(
commandKey, threadPoolKey, groupKey, executionConfig, circuitBreakerConfig, metricsConfig));
}
public void TestMultipleCommandsCarryOverMultipleBucketsForMultipleThreadPools()
{
IHystrixCommandGroupKey groupKeyX = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-X");
IHystrixCommandGroupKey groupKeyY = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-Y");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-X");
IHystrixCommandKey keyX = HystrixCommandKeyDefault.AsKey("RollingConcurrency-X");
IHystrixCommandKey keyY = HystrixCommandKeyDefault.AsKey("RollingConcurrency-Y");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Take(10 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd1 = Command.From(groupKeyX, keyX, HystrixEventType.SUCCESS, 300);
Command cmd2 = Command.From(groupKeyY, keyY, HystrixEventType.SUCCESS, 300);
Command cmd3 = Command.From(groupKeyX, keyY, HystrixEventType.SUCCESS, 10);
Command cmd4 = Command.From(groupKeyX, keyY, HystrixEventType.SUCCESS, 10);
Task t1 = cmd1.ExecuteAsync();
// Time.Wait(100); // bucket roll
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t2 = cmd2.ExecuteAsync();
// Time.Wait(100); // bucket roll
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t3 = cmd3.ExecuteAsync();
// Time.Wait(100); // bucket roll
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t4 = cmd4.ExecuteAsync();
Task.WaitAll(t1, t2, t3, t4);
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Assert.Equal(2, stream.LatestRollingMax);
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-A");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
var disposable = stream.Observe().Take(10).Subscribe(new LatchedObserver(this.output, latch));
//no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
Assert.False(HasData(stream.Latest));
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-A");
stream = HealthCountsStream.GetInstance(key, 10, 100);
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
// no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(0L, stream.Latest.ErrorCount);
Assert.Equal(0L, stream.Latest.TotalRequests);
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-CumulativeCounter-A");
stream = CumulativeCommandEventCounterStream.GetInstance(key, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(new LatchedObserver(output, latch));
//no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
Assert.False(HasData(stream.Latest));
}
