public async Task TestSingleTimeout()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-D");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
var expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.TIMEOUT] = 1;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 1;
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd = Command.From(GroupKey, key, HystrixEventType.TIMEOUT);
await cmd.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
Assert.Equal(expected, stream.Latest);
}
public async void TestSingleTimeout()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-Health-D");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = HealthCountsStream.GetInstance(key, 10, 100);
var cmd = Command.From(GroupKey, key, HystrixEventType.TIMEOUT); // Timeout 1000
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
await cmd.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(1L, stream.Latest.ErrorCount);
Assert.Equal(1L, stream.Latest.TotalRequests);
}
public void TestStartsAndEndsInSameBucketProduceValue()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-B");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-B");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-B");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
var cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
Task t1 = cmd1.ExecuteAsync();
Task t2 = cmd2.ExecuteAsync();
Task.WaitAll(t1, t2);
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(2, stream.LatestRollingMax);
}
public void TestStartsAndEndsInSameBucketProduceValue()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-B");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(new LatchedObserver(output, latch));
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 100);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 100);
cmd1.Observe();
Time.Wait(1);
cmd2.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(2, stream.LatestRollingMax);
}
public async Task TestSemaphoreRejectedCommandDoesNotGetLatencyTracked()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-Latency-F");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 10 commands with latency should occupy all semaphores. execute those, then wait for bucket to roll
// next command should be a semaphore rejection
var commands = new List <Command>();
for (var i = 0; i < 10; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 500, ExecutionIsolationStrategy.SEMAPHORE));
}
var semaphoreRejected = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
var satTasks = new List <Task>();
foreach (var saturator in commands)
{
satTasks.Add(Task.Run(() => saturator.Execute()));
}
await Task.Delay(50);
await Task.Run(() => semaphoreRejected.Execute());
Task.WaitAll(satTasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(10, stream.Latest.GetTotalCount());
AssertBetween(500, 750, stream.LatestMean);
Assert.True(semaphoreRejected.IsResponseSemaphoreRejected);
}
public void TestRequestFromCache()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-F");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-F");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingCounter-F");
stream = RollingThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(GetSubscriber(output, latch));
CommandStreamTest.Command cmd1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
CommandStreamTest.Command cmd2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
CommandStreamTest.Command cmd3 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
cmd1.Observe();
cmd2.Observe();
cmd3.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
//RESPONSE_FROM_CACHE should not show up at all in thread pool counters - just the success
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestEmptyStreamProducesEmptyDistributions()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-A");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + DateTime.Now.Ticks / 10000 + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + Thread.CurrentThread.ManagedThreadId);
Assert.Equal(0, distribution.GetTotalCount());
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
//no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(0, stream.Latest.GetTotalCount());
}
public void TestRequestFromCache()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-F");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(this.output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
cmd1.Observe();
cmd2.Observe();
cmd3.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 1;
expected[(int)HystrixEventType.RESPONSE_FROM_CACHE] = 2;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal <long[]>(expected, stream.Latest);
}
public async void TestFallbackMissing()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-CumulativeCounter-K");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeCommandEventCounterStream.GetInstance(key, 10, 500);
Command cmd = Command.From(GroupKey, key, HystrixEventType.FAILURE, 0, HystrixEventType.FALLBACK_MISSING);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 2000), "Stream failed to start");
await Assert.ThrowsAsync <HystrixRuntimeException>(async() => await cmd.Observe());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 2000, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.FAILURE] = 1;
expected[(int)HystrixEventType.FALLBACK_MISSING] = 1;
expected[(int)HystrixEventType.EXCEPTION_THROWN] = 1;
Assert.Equal(expected, stream.Latest);
}
public async void TestMultipleEventsOverTimeGetStoredAndAgeOut()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-M");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Take(30 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 20);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 10);
await cmd1.Observe();
await cmd2.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
Assert.Equal(expected, stream.Latest);
}
public async void TestSingleFailure()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-C");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd = Command.From(GroupKey, key, HystrixEventType.FAILURE, 20);
await cmd.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.FAILURE] = 1;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 1;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(expected, stream.Latest);
}
public void TestSingleTimeout()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-CumulativeCounter-D");
stream = CumulativeCommandEventCounterStream.GetInstance(key, 10, 600);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(GroupKey, key, HystrixEventType.TIMEOUT);
cmd.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.TIMEOUT] = 1;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 1;
Assert.Equal(expected, stream.Latest);
}
public void TestSingleFailure()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-C");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-C");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-C");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(groupKey, key, HystrixEventType.FAILURE, 20);
cmd.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
}
public async void TestSingleBadRequest()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-CumulativeCounter-E");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeCommandEventCounterStream.GetInstance(key, 10, 500);
var cmd = Command.From(GroupKey, key, HystrixEventType.BAD_REQUEST);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 2000), "Stream failed to start");
await Assert.ThrowsAsync <HystrixBadRequestException>(async() => await cmd.Observe());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 2000, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
var expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.BAD_REQUEST] = 1;
expected[(int)HystrixEventType.EXCEPTION_THROWN] = 1;
Assert.Equal(expected, stream.Latest);
}
public void TestSingleBadRequest()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-E");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
Command cmd = Command.From(GroupKey, key, HystrixEventType.BAD_REQUEST);
cmd.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.BAD_REQUEST] = 1;
expected[(int)HystrixEventType.EXCEPTION_THROWN] = 1;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(expected, stream.Latest);
}
public void TestStartsAndEndsInSameBucketProduceValue()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-B");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-B");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-B");
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 50);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
cmd1.Observe();
Time.Wait(1);
cmd2.Observe();
Assert.True(latch.Wait(10000));
Assert.Equal(2, stream.LatestRollingMax);
}
public async void TestSingleSuccess()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-CumulativeCounter-B");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeCommandEventCounterStream.GetInstance(key, 10, 500); // Stream should start
var cmd = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 2000), "Stream failed to start");
await cmd.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 2000, output), "Latch took to long to update");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
var expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)(int)HystrixEventType.SUCCESS] = 1;
Assert.Equal(expected, stream.Latest);
}
public void TestMultipleEventsOverTimeGetStoredAndDoNotAgeOut()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-M");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-M");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-M");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(30).Subscribe(new LatchedObserver(output, latch));
CommandStreamTest.Command cmd1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
CommandStreamTest.Command cmd2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 10);
cmd1.Observe();
cmd2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
//all commands should have aged out
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(2, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestMultipleCommandsCarryOverMultipleBucketsForMultipleThreadPools()
{
var groupKeyX = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-X");
var groupKeyY = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-Y");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-X");
var keyX = HystrixCommandKeyDefault.AsKey("RollingConcurrency-X");
var keyY = HystrixCommandKeyDefault.AsKey("RollingConcurrency-Y");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Take(10 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(groupKeyX, keyX, HystrixEventType.SUCCESS, 300);
var cmd2 = Command.From(groupKeyY, keyY, HystrixEventType.SUCCESS, 300);
var cmd3 = Command.From(groupKeyX, keyY, HystrixEventType.SUCCESS, 10);
var cmd4 = Command.From(groupKeyX, keyY, HystrixEventType.SUCCESS, 10);
Task t1 = cmd1.ExecuteAsync();
// Time.Wait(100); // bucket roll
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t2 = cmd2.ExecuteAsync();
// Time.Wait(100); // bucket roll
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t3 = cmd3.ExecuteAsync();
// Time.Wait(100); // bucket roll
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Task t4 = cmd4.ExecuteAsync();
Task.WaitAll(t1, t2, t3, t4);
WaitForLatchedObserverToUpdate(observer, 1, 100, 125, output);
Assert.Equal(2, stream.LatestRollingMax);
}
public void TestResponseFromCacheDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-G");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
// should get 1 SUCCESS and 1 RESPONSE_FROM_CACHE
List <Command> commands = Command.GetCommandsWithResponseFromCache(GroupKey, key);
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
Assert.True(distribution.GetTotalCount() <= 1);
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
foreach (Command cmd in commands)
{
cmd.Observe();
}
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(1, stream.Latest.GetTotalCount());
AssertBetween(0, 30, stream.LatestMean);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
}
public async void TestThreadPoolRejectedCommandDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-E");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 10 commands with latency should occupy the entire threadpool. execute those, then wait for bucket to roll
// next command should be a thread-pool rejection
List <Command> commands = new List <Command>();
for (int i = 0; i < 10; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 500));
}
Command threadPoolRejected = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
List <Task> satTasks = new List <Task>();
foreach (Command cmd in commands)
{
satTasks.Add(cmd.ExecuteAsync());
}
await threadPoolRejected.Observe();
Task.WaitAll(satTasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(10, stream.Latest.GetTotalCount());
AssertBetween(500, 750, stream.LatestMean);
Assert.True(threadPoolRejected.IsResponseThreadPoolRejected);
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-A");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
var disposable = stream.Observe().Take(10).Subscribe(new LatchedObserver(this.output, latch));
//no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
Assert.False(HasData(stream.Latest));
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-A");
stream = HealthCountsStream.GetInstance(key, 10, 100);
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
// no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(0L, stream.Latest.ErrorCount);
Assert.Equal(0L, stream.Latest.TotalRequests);
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-CumulativeCounter-A");
stream = CumulativeCommandEventCounterStream.GetInstance(key, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(new LatchedObserver(output, latch));
//no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
Assert.False(HasData(stream.Latest));
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-A");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(
GetSubscriber(output, latch));
//no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(0, stream.LatestRollingMax);
}
public void TtestOneCommandCarriesOverToNextBucket()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-C");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 160);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 15);
cmd1.Observe();
Time.Wait(100); //bucket roll
cmd2.Observe();
Time.Wait(1);
cmd3.Observe();
Assert.True(latch.Wait(10000));
Assert.Equal(3, stream.LatestRollingMax);
}
public void TestStartsAndEndsInSameBucketSemaphoreIsolated()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("ThreadPool-Concurrency-C");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("ThreadPool-Concurrency-C");
var key = HystrixCommandKeyDefault.AsKey("RollingConcurrency-C");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingThreadPoolMaxConcurrencyStream.GetInstance(threadPoolKey, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10, ExecutionIsolationStrategy.SEMAPHORE);
var cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 14, ExecutionIsolationStrategy.SEMAPHORE);
Task t1 = cmd1.ExecuteAsync();
Task t2 = cmd2.ExecuteAsync();
Task.WaitAll(t1, t2);
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
// since commands run in semaphore isolation, they are not tracked by threadpool metrics
Assert.Equal(0, stream.LatestRollingMax);
}
public async Task TestSingleFailure()
{
var groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-C");
var threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-C");
var key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-C");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
var cmd = Command.From(groupKey, key, HystrixEventType.FAILURE, 0);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
await cmd.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(1, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestMultipleEventsOverTimeGetStoredAndAgeOut()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-M");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
// by doing a take(30), we ensure that all rolling counts go back to 0
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(30).Subscribe(new LatchedObserver(output, latch));
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 20);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 10);
cmd1.Observe();
cmd2.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(expected, stream.Latest);
}
public void TestCollapsed()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("BatchCommand");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
for (int i = 0; i < 3; i++)
{
Collapser.From(output, i).Observe();
}
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 1;
expected[(int)HystrixEventType.COLLAPSED] = 3;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(expected, stream.Latest);
}