public void TestConcurrencyStreamProperlyFiltersOutResponseFromCache()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-F");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 40);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
Command cmd4 = Command.From(groupKey, key, HystrixEventType.RESPONSE_FROM_CACHE);
cmd1.Observe();
Time.Wait(5);
cmd2.Observe();
cmd3.Observe();
cmd4.Observe();
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(1, stream.LatestRollingMax);
}
public void TestMultipleCommandsCarryOverMultipleBucketsAndThenAgeOut()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-E");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(30).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 300);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd4 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
cmd1.Observe();
Time.Wait(100); //bucket roll
cmd2.Observe();
Time.Wait(100);
cmd3.Observe();
Time.Wait(100);
cmd4.Observe();
Assert.True(latch.Wait(10000));
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestConcurrencyStreamProperlyFiltersOutSemaphoreRejections()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-H");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
// 10 commands executed concurrently on different caller threads should saturate semaphore
// once these are in-flight, execute 10 more concurrently on new caller threads.
// since these are semaphore-rejected, the max concurrency should be 10
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 400, ExecutionIsolationStrategy.SEMAPHORE));
}
List <Command> rejected = new List <Command>();
for (int i = 0; i < 10; i++)
{
rejected.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 100, ExecutionIsolationStrategy.SEMAPHORE));
}
// List<long> startTimes = new List<long>();
foreach (Command saturatingCmd in saturators)
{
Task t = new Task(
() =>
{
saturatingCmd.Observe();
}, CancellationToken.None,
TaskCreationOptions.LongRunning);
t.Start();
}
Time.Wait(30);
foreach (Command rejectedCmd in rejected)
{
Task t = new Task(
() =>
{
rejectedCmd.Observe();
}, CancellationToken.None,
TaskCreationOptions.LongRunning);
t.Start();
}
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(10, stream.LatestRollingMax);
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-A");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(
new LatchedObserver(output, latch));
// no writes
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(0, stream.LatestRollingMax);
}
public void TestConcurrencyStreamProperlyFiltersOutThreadPoolRejections()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-I");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
//10 commands executed concurrently should saturate the Hystrix threadpool
//once these are in-flight, execute 10 more concurrently
//since these are threadpool-rejected, the max concurrency should be 10
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 400));
}
List <Command> rejected = new List <Command>();
for (int i = 0; i < 10; i++)
{
rejected.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 100));
}
foreach (Command saturatingCmd in saturators)
{
saturatingCmd.Observe();
}
Time.Wait(30);
foreach (Command rejectedCmd in rejected)
{
rejectedCmd.Observe();
}
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(10, stream.LatestRollingMax);
}
public void TestConcurrencyStreamProperlyFiltersOutShortCircuits()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-G");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
//after 3 failures, next command should short-circuit.
//to prove short-circuited commands don't contribute to concurrency, execute 3 FAILURES in the first bucket sequentially
//then when circuit is open, execute 20 concurrent commands. they should all get short-circuited, and max concurrency should be 1
Command failure1 = Command.From(groupKey, key, HystrixEventType.FAILURE);
Command failure2 = Command.From(groupKey, key, HystrixEventType.FAILURE);
Command failure3 = Command.From(groupKey, key, HystrixEventType.FAILURE);
List <Command> shortCircuited = new List <Command>();
for (int i = 0; i < 20; i++)
{
shortCircuited.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 100));
}
failure1.Execute();
failure2.Execute();
failure3.Execute();
Time.Wait(150);
foreach (Command cmd in shortCircuited)
{
cmd.Observe();
}
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(1, stream.LatestRollingMax);
}
public void TestStartsAndEndsInSameBucketProduceValue()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-B");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 100);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 100);
cmd1.Observe();
Time.Wait(1);
cmd2.Observe();
Assert.True(latch.Wait(10000));
Assert.Equal(2, stream.LatestRollingMax);
}
public void TestEmptyStreamProducesZeros()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-A");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 500);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(5).Subscribe(
GetSubscriber(output, latch));
//no writes
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(0, stream.LatestRollingMax);
}
public void TtestOneCommandCarriesOverToNextBucket()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Concurrency-C");
stream = RollingCommandMaxConcurrencyStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(GetSubscriber(output, latch));
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 160);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd3 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 15);
cmd1.Observe();
Time.Wait(100); //bucket roll
cmd2.Observe();
Time.Wait(1);
cmd3.Observe();
Assert.True(latch.Wait(10000));
Assert.Equal(3, stream.LatestRollingMax);
}