public void TestCollapsed()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("BatchCommand");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(this.output, latch));
for (int i = 0; i < 3; i++)
{
CommandStreamTest.Collapser.From(output, i).Observe();
}
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 1;
expected[(int)HystrixEventType.COLLAPSED] = 3;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal <long[]>(expected, stream.Latest);
}
public async void TestRequestFromCache()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-F");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 20);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
Command cmd3 = Command.From(GroupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
await cmd1.Observe();
await cmd2.Observe();
await cmd3.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 1;
expected[(int)HystrixEventType.RESPONSE_FROM_CACHE] = 2;
Assert.Equal(expected, stream.Latest);
}
public void TestMultipleEventsOverTimeGetStoredAndAgeOut()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-M");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
//by doing a take(30), we ensure that all rolling counts go back to 0
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(30).Subscribe(new LatchedObserver(this.output, latch));
CommandStreamTest.Command cmd1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
CommandStreamTest.Command cmd2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 10);
cmd1.Observe();
cmd2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal <long[]>(expected, stream.Latest);
}
public RollingCommandEventCounterStreamTest(ITestOutputHelper output)
: base()
{
RollingCommandEventCounterStream.Reset();
HystrixCommandCompletionStream.Reset();
this.output = output;
}
public void TestRequestFromCache()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-F");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 20);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
Command cmd3 = Command.From(GroupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
cmd1.Observe();
cmd2.Observe();
cmd3.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 1;
expected[(int)HystrixEventType.RESPONSE_FROM_CACHE] = 2;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(expected, stream.Latest);
}
public void TestCollapsed()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("BatchCommand");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
List <Task> tasks = new List <Task>();
for (int i = 0; i < 3; i++)
{
tasks.Add(Collapser.From(output, i).ExecuteAsync());
}
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 1;
expected[(int)HystrixEventType.COLLAPSED] = 3;
Assert.Equal(expected, stream.Latest);
}
public void Before()
{
context = HystrixRequestContext.InitializeContext();
HystrixCommandMetrics.Reset();
HystrixThreadPoolMetrics.Reset();
HystrixCollapserMetrics.Reset();
// clear collapsers
RequestCollapserFactory.Reset();
// clear circuit breakers
HystrixCircuitBreakerFactory.Reset();
HystrixPlugins.Reset();
HystrixOptionsFactory.Reset();
// clear up all streams
CumulativeCollapserEventCounterStream.Reset();
CumulativeCommandEventCounterStream.Reset();
CumulativeThreadPoolEventCounterStream.Reset();
RollingCollapserBatchSizeDistributionStream.Reset();
RollingCollapserEventCounterStream.Reset();
RollingCommandEventCounterStream.Reset();
RollingCommandLatencyDistributionStream.Reset();
RollingCommandMaxConcurrencyStream.Reset();
RollingCommandUserLatencyDistributionStream.Reset();
RollingThreadPoolEventCounterStream.Reset();
RollingThreadPoolMaxConcurrencyStream.Reset();
}
public void TestFallbackFailure()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-J");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(GroupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_FAILURE);
cmd.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.FAILURE] = 1;
expected[(int)HystrixEventType.FALLBACK_FAILURE] = 1;
expected[(int)HystrixEventType.EXCEPTION_THROWN] = 1;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(expected, stream.Latest);
}
public void TestShortCircuited()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-G");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(this.output, latch));
//3 failures in a row will trip circuit. let bucket roll once then submit 2 requests.
//should see 3 FAILUREs and 2 SHORT_CIRCUITs and then 5 FALLBACK_SUCCESSes
Command failure1 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0);
Command failure2 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0);
Command failure3 = Command.From(groupKey, key, HystrixEventType.FAILURE, 0);
Command shortCircuit1 = Command.From(groupKey, key, HystrixEventType.SUCCESS);
Command shortCircuit2 = Command.From(groupKey, key, HystrixEventType.SUCCESS);
failure1.Observe();
failure2.Observe();
failure3.Observe();
try
{
Time.Wait(125);
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
shortCircuit1.Observe();
shortCircuit2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(shortCircuit1.IsResponseShortCircuited);
Assert.True(shortCircuit2.IsResponseShortCircuited);
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.FAILURE] = 3;
expected[(int)HystrixEventType.SHORT_CIRCUITED] = 2;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 5;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal <long[]>(expected, stream.Latest);
}
public void TestFallbackMissing()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-K");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(this.output, latch));
CommandStreamTest.Command cmd = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_MISSING);
cmd.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.FAILURE] = 1;
expected[(int)HystrixEventType.FALLBACK_MISSING] = 1;
expected[(int)HystrixEventType.EXCEPTION_THROWN] = 1;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal <long[]>(expected, stream.Latest);
}
public override void Dispose()
{
latchSubscription?.Dispose();
stream?.Unsubscribe();
latchSubscription = null;
stream = null;
base.Dispose();
}
public void TestFallbackRejection()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-L");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(this.output, latch));
//fallback semaphore size is 5. So let 5 commands saturate that semaphore, then
//let 2 more commands go to fallback. they should get rejected by the fallback-semaphore
List <CommandStreamTest.Command> fallbackSaturators = new List <CommandStreamTest.Command>();
for (int i = 0; i < 5; i++)
{
fallbackSaturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 400));
}
CommandStreamTest.Command rejection1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0);
CommandStreamTest.Command rejection2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0);
foreach (CommandStreamTest.Command saturator in fallbackSaturators)
{
saturator.Observe();
}
try
{
Time.Wait(70);
}
catch (Exception ex)
{
Assert.False(true, ex.Message);
}
rejection1.Observe();
rejection2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.FAILURE] = 7;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 5;
expected[(int)HystrixEventType.FALLBACK_REJECTION] = 2;
expected[(int)HystrixEventType.EXCEPTION_THROWN] = 2;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal <long[]>(expected, stream.Latest);
}
public void TestThreadPoolRejected()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-I");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(this.output, latch));
//10 commands will saturate threadpools when called concurrently.
//submit 2 more requests and they should be THREADPOOL_REJECTED
//should see 10 SUCCESSes, 2 THREADPOOL_REJECTED and 2 FALLBACK_SUCCESSes
List <CommandStreamTest.Command> saturators = new List <CommandStreamTest.Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 200));
}
CommandStreamTest.Command rejected1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
CommandStreamTest.Command rejected2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 0);
foreach (CommandStreamTest.Command saturator in saturators)
{
saturator.Observe();
}
rejected1.Observe();
rejected2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(rejected1.IsResponseThreadPoolRejected);
Assert.True(rejected2.IsResponseThreadPoolRejected);
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 10;
expected[(int)HystrixEventType.THREAD_POOL_REJECTED] = 2;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 2;
Assert.Equal <long[]>(expected, stream.Latest);
}
public void TestSemaphoreRejected()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-H");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(this.output, latch));
// 10 commands will saturate semaphore when called from different threads.
// submit 2 more requests and they should be SEMAPHORE_REJECTED
// should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 400, ExecutionIsolationStrategy.SEMAPHORE));
}
Command rejected1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
Command rejected2 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
foreach (Command c in saturators)
{
new Thread(new ThreadStart(() => c.Observe())).Start();
}
try
{
Time.Wait(100);
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
rejected1.Observe();
rejected2.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(rejected1.IsResponseSemaphoreRejected, "Response not semaphore rejected as was expected (1)");
Assert.True(rejected2.IsResponseSemaphoreRejected, "Response not semaphore rejected as was expected (2)");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 10;
expected[(int)HystrixEventType.SEMAPHORE_REJECTED] = 2;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 2;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(expected, stream.Latest);
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-A");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
Assert.False(HasData(stream.Latest), "Stream has events when it should not");
}
public async void TestSemaphoreRejected()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-H");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 10 commands will saturate semaphore when called from different threads.
// submit 2 more requests and they should be SEMAPHORE_REJECTED
// should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes
var saturators = new List <Command>();
for (var i = 0; i < 10; i++)
{
saturators.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 500, ExecutionIsolationStrategy.SEMAPHORE));
}
var rejected1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
var rejected2 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
var tasks = new List <Task>();
foreach (var saturator in saturators)
{
tasks.Add(Task.Run(() => saturator.Execute()));
}
await Task.Delay(50);
await Task.Run(() => rejected1.Execute());
await Task.Run(() => rejected2.Execute());
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.True(rejected1.IsResponseSemaphoreRejected, "Response not semaphore rejected as was expected (1)");
Assert.True(rejected2.IsResponseSemaphoreRejected, "Response not semaphore rejected as was expected (2)");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
var expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 10;
expected[(int)HystrixEventType.SEMAPHORE_REJECTED] = 2;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 2;
Assert.Equal(expected, stream.Latest);
}
public async void TestFallbackRejection()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-L");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// fallback semaphore size is 5. So let 5 commands saturate that semaphore, then
// let 2 more commands go to fallback. they should get rejected by the fallback-semaphore
var fallbackSaturators = new List <Command>();
for (var i = 0; i < 5; i++)
{
fallbackSaturators.Add(Command.From(GroupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 500));
}
var rejection1 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 0);
var rejection2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_SUCCESS, 0);
var tasks = new List <Task>();
foreach (var saturator in fallbackSaturators)
{
tasks.Add(saturator.ExecuteAsync());
}
await Task.Delay(50);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await rejection1.Observe());
await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await rejection2.Observe());
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
var expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.FAILURE] = 7;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 5;
expected[(int)HystrixEventType.FALLBACK_REJECTION] = 2;
expected[(int)HystrixEventType.EXCEPTION_THROWN] = 2;
Assert.Equal(expected, stream.Latest);
}
public async Task TestThreadPoolRejected()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-I");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 10 commands will saturate threadpools when called concurrently.
// submit 2 more requests and they should be THREADPOOL_REJECTED
// should see 10 SUCCESSes, 2 THREADPOOL_REJECTED and 2 FALLBACK_SUCCESSes
var saturators = new List <Command>();
for (var i = 0; i < 10; i++)
{
saturators.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 500));
}
var rejected1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0);
var rejected2 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0);
var tasks = new List <Task>();
foreach (var saturator in saturators)
{
tasks.Add(saturator.ExecuteAsync());
}
await rejected1.Observe();
await rejected2.Observe();
Task.WaitAll(tasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.True(rejected1.IsResponseThreadPoolRejected, "Not ThreadPoolRejected as was expected (1)");
Assert.True(rejected2.IsResponseThreadPoolRejected, "Not ThreadPoolRejected as was expected (2)");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
var expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 10;
expected[(int)HystrixEventType.THREAD_POOL_REJECTED] = 2;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 2;
Assert.Equal(expected, stream.Latest);
}
public static void ShutdownThreads()
{
CumulativeCommandEventCounterStream.Reset();
CumulativeThreadPoolEventCounterStream.Reset();
RollingCommandEventCounterStream.Reset();
RollingThreadPoolEventCounterStream.Reset();
RollingCollapserEventCounterStream.Reset();
RollingCollapserEventCounterStream.Reset();
HealthCountsStream.Reset();
RollingCollapserBatchSizeDistributionStream.Reset();
RollingCommandLatencyDistributionStream.Reset();
RollingCommandUserLatencyDistributionStream.Reset();
RollingCommandMaxConcurrencyStream.Reset();
RollingThreadPoolMaxConcurrencyStream.Reset();
}
HystrixCommandMetrics(IHystrixCommandKey key, IHystrixCommandGroupKey commandGroup, IHystrixThreadPoolKey threadPoolKey, IHystrixCommandOptions properties, HystrixEventNotifier eventNotifier) : base(null)
{
this.key = key;
this.group = commandGroup;
this.threadPoolKey = threadPoolKey;
this.properties = properties;
healthCountsStream = HealthCountsStream.GetInstance(key, properties);
rollingCommandEventCounterStream = RollingCommandEventCounterStream.GetInstance(key, properties);
cumulativeCommandEventCounterStream = CumulativeCommandEventCounterStream.GetInstance(key, properties);
rollingCommandLatencyDistributionStream = RollingCommandLatencyDistributionStream.GetInstance(key, properties);
rollingCommandUserLatencyDistributionStream = RollingCommandUserLatencyDistributionStream.GetInstance(key, properties);
rollingCommandMaxConcurrencyStream = RollingCommandMaxConcurrencyStream.GetInstance(key, properties);
}
public async void TestShortCircuited()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-G");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 3 failures in a row will trip circuit. let bucket roll once then submit 2 requests.
// should see 3 FAILUREs and 2 SHORT_CIRCUITs and then 5 FALLBACK_SUCCESSes
var failure1 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0);
var failure2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0);
var failure3 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0);
var shortCircuit1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
var shortCircuit2 = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
await failure1.Observe();
await failure2.Observe();
await failure3.Observe();
Assert.True(WaitForHealthCountToUpdate(key.Name, 500, output), "health count took to long to update");
output.WriteLine(Time.CurrentTimeMillis + " running failures");
await shortCircuit1.Observe();
await shortCircuit2.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.True(shortCircuit1.IsResponseShortCircuited, "Circuit 1 not shorted as was expected");
Assert.True(shortCircuit2.IsResponseShortCircuited, "Circuit 2 not shorted as was expected");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
var expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.FAILURE] = 3;
expected[(int)HystrixEventType.SHORT_CIRCUITED] = 2;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 5;
Assert.Equal(expected, stream.Latest);
}
internal static void Reset()
{
foreach (HystrixCommandMetrics metricsInstance in GetInstances())
{
metricsInstance.UnsubscribeAll();
}
RollingCommandEventCounterStream.Reset();
CumulativeCommandEventCounterStream.Reset();
RollingCommandLatencyDistributionStream.Reset();
RollingCommandUserLatencyDistributionStream.Reset();
RollingCommandMaxConcurrencyStream.Reset();
HystrixThreadEventStream.Reset();
HealthCountsStream.Reset();
Metrics.Clear();
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-A");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
_ = stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
// no writes
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
Assert.False(HasData(stream.Latest));
}
public async void TestSingleSuccess()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-B");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 20);
await cmd.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 1;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(expected, stream.Latest);
}
public async void TestSingleTimeout()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-D");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
var expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.TIMEOUT] = 1;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 1;
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd = Command.From(GroupKey, key, HystrixEventType.TIMEOUT);
await cmd.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
Assert.Equal(expected, stream.Latest);
}
public async void TestFallbackMissing()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-K");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd = Command.From(GroupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_MISSING);
await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await cmd.Observe());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.FAILURE] = 1;
expected[(int)HystrixEventType.FALLBACK_MISSING] = 1;
expected[(int)HystrixEventType.EXCEPTION_THROWN] = 1;
Assert.Equal(expected, stream.Latest);
}
public async void TestSingleBadRequest()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-E");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd = Command.From(GroupKey, key, HystrixEventType.BAD_REQUEST);
await Assert.ThrowsAsync <HystrixBadRequestException>(async() => await cmd.Observe());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
var expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.BAD_REQUEST] = 1;
expected[(int)HystrixEventType.EXCEPTION_THROWN] = 1;
Assert.Equal(expected, stream.Latest);
}
public async void TestMultipleEventsOverTimeGetStoredAndAgeOut()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-M");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Take(30 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 20);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 10);
await cmd1.Observe();
await cmd2.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
Assert.Equal(expected, stream.Latest);
}
public override void Dispose()
{
base.Dispose();
stream.Unsubscribe();
RollingCommandEventCounterStream.Reset();
}
