public async Task TestMultipleBucketsBothGetStored()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-Latency-H");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 10);
var cmd2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 100);
var cmd3 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 60);
var cmd4 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 60);
var cmd5 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 70);
await cmd1.Observe();
await cmd2.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
await cmd3.Observe();
await cmd4.Observe();
await cmd5.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
AssertBetween(50, 150, stream.LatestMean);
AssertBetween(10, 150, stream.GetLatestPercentile(0.0));
AssertBetween(100, 150, stream.GetLatestPercentile(100.0));
}
public RollingCommandLatencyDistributionStreamTest(ITestOutputHelper output)
: base()
{
this.output = output;
RollingCommandLatencyDistributionStream.Reset();
HystrixCommandCompletionStream.Reset();
}
public async Task TestMultipleBucketsBothGetStoredAndThenAgeOut()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-Latency-I");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Take(20 + LatchedObserver.STABLE_TICK_COUNT).Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 10);
var cmd2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 100);
var cmd3 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 60);
var cmd4 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 60);
var cmd5 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 70);
await cmd1.Observe();
await cmd2.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
await cmd3.Observe();
await cmd4.Observe();
await cmd5.Observe();
WaitForLatchedObserverToUpdate(observer, 1, 500, output);
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(0, stream.Latest.GetTotalCount());
}
public void TestThreadPoolRejectedCommandDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-E");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
// 10 commands with latency should occupy the entire threadpool. execute those, then wait for bucket to roll
// next command should be a thread-pool rejection
List <Command> commands = new List <Command>();
for (int i = 0; i < 10; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 200));
}
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
if (distribution.GetTotalCount() > 0)
{
AssertBetween(200, 250, distribution.GetMean());
}
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
foreach (Command cmd in commands)
{
cmd.Observe();
}
Command threadPoolRejected = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
try
{
Time.Wait(40);
threadPoolRejected.Observe();
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal(10, stream.Latest.GetTotalCount());
AssertBetween(200, 250, stream.LatestMean);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(threadPoolRejected.IsResponseThreadPoolRejected, "Response NOT ThreadPoolRejected as expected");
}
public void TestEmptyStreamProducesEmptyDistributions()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-A");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + Thread.CurrentThread.ManagedThreadId);
Assert.Equal(0, distribution.GetTotalCount());
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
// no writes
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(0, stream.Latest.GetTotalCount());
}
public async Task TestSingleBucketWithMultipleEventTypes()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-Latency-C");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
var cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 10);
var cmd2 = Command.From(GroupKey, key, HystrixEventType.TIMEOUT); // latency = 600
var cmd3 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 30);
var cmd4 = Command.From(GroupKey, key, HystrixEventType.BAD_REQUEST, 40);
await cmd1.Observe();
await cmd3.Observe();
await Assert.ThrowsAsync <HystrixBadRequestException>(async() => await cmd4.Observe());
await cmd2.Observe(); // Timeout should run last
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
AssertBetween(100, 400, stream.LatestMean); // now timeout latency of 600ms is there
AssertBetween(10, 100, stream.GetLatestPercentile(0.0));
AssertBetween(300, 800, stream.GetLatestPercentile(100.0));
}
public void Before()
{
context = HystrixRequestContext.InitializeContext();
HystrixCommandMetrics.Reset();
HystrixThreadPoolMetrics.Reset();
HystrixCollapserMetrics.Reset();
// clear collapsers
RequestCollapserFactory.Reset();
// clear circuit breakers
HystrixCircuitBreakerFactory.Reset();
HystrixPlugins.Reset();
HystrixOptionsFactory.Reset();
// clear up all streams
CumulativeCollapserEventCounterStream.Reset();
CumulativeCommandEventCounterStream.Reset();
CumulativeThreadPoolEventCounterStream.Reset();
RollingCollapserBatchSizeDistributionStream.Reset();
RollingCollapserEventCounterStream.Reset();
RollingCommandEventCounterStream.Reset();
RollingCommandLatencyDistributionStream.Reset();
RollingCommandMaxConcurrencyStream.Reset();
RollingCommandUserLatencyDistributionStream.Reset();
RollingThreadPoolEventCounterStream.Reset();
RollingThreadPoolMaxConcurrencyStream.Reset();
}
public void TestMultipleBucketsBothGetStored()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-H");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
if (distribution.GetTotalCount() == 2)
{
AssertBetween(55, 90, distribution.GetMean());
}
if (distribution.GetTotalCount() == 5)
{
AssertBetween(60, 90, distribution.GetMean());
}
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 100);
cmd1.Observe();
cmd2.Observe();
try
{
Time.Wait(500);
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Command cmd3 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 60);
Command cmd4 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 60);
Command cmd5 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 70);
cmd3.Observe();
cmd4.Observe();
cmd5.Observe();
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
AssertBetween(55, 90, stream.LatestMean);
AssertBetween(10, 50, stream.GetLatestPercentile(0.0));
AssertBetween(100, 150, stream.GetLatestPercentile(100.0));
}
public override void Dispose()
{
latchSubscription?.Dispose();
stream?.Unsubscribe();
latchSubscription = null;
stream = null;
base.Dispose();
}
public void TestShortCircuitedCommandDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-D");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
// 3 failures is enough to trigger short-circuit. execute those, then wait for bucket to roll
// next command should be a short-circuit
List <Command> commands = new List <Command>();
for (int i = 0; i < 3; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.FAILURE, 0));
}
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
AssertBetween(0, 30, distribution.GetMean());
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
foreach (Command cmd in commands)
{
cmd.Observe();
}
Command shortCircuit = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
try
{
Time.Wait(200);
shortCircuit.Observe();
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
Assert.True(latch.Wait(10000), "CountdownEvent was not set!");
Assert.Equal(3, stream.Latest.GetTotalCount());
AssertBetween(0, 30, stream.LatestMean);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(shortCircuit.IsResponseShortCircuited);
}
public void TestSingleBucketWithMultipleEventTypes()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-C");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + DateTime.Now.Ticks / 10000 + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
if (distribution.GetTotalCount() < 4 && distribution.GetTotalCount() > 0)
{ //buckets before timeout latency registers
AssertBetween(10, 50, (int)distribution.GetMean());
}
else if (distribution.GetTotalCount() == 4)
{
AssertBetween(150, 250, (int)distribution.GetMean()); //now timeout latency of 600ms is there
}
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
Command cmd1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd2 = Command.From(groupKey, key, HystrixEventType.TIMEOUT); //latency = 600
Command cmd3 = Command.From(groupKey, key, HystrixEventType.FAILURE, 30);
Command cmd4 = Command.From(groupKey, key, HystrixEventType.BAD_REQUEST, 40);
cmd1.Observe();
cmd2.Observe();
cmd3.Observe();
cmd4.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
AssertBetween(150, 350, stream.LatestMean); //now timeout latency of 600ms is there
AssertBetween(10, 40, stream.GetLatestPercentile(0.0));
AssertBetween(600, 800, stream.GetLatestPercentile(100.0));
}
public void TestSingleBucketGetsStored()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-B");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
if (distribution.GetTotalCount() == 1)
{
AssertBetween(10, 50, (int)distribution.GetMean());
}
else if (distribution.GetTotalCount() == 2)
{
AssertBetween(300, 400, (int)distribution.GetMean());
}
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.TIMEOUT); // latency = 600
cmd1.Observe();
cmd2.Observe();
try
{
Assert.True(latch.Wait(10000));
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
}
catch (Exception)
{
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(false, "Interrupted ex");
}
AssertBetween(150, 400, stream.LatestMean);
AssertBetween(10, 50, stream.GetLatestPercentile(0.0));
AssertBetween(300, 800, stream.GetLatestPercentile(100.0));
}
public void TestEmptyStreamProducesEmptyDistributions()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-Latency-A");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(0, stream.Latest.GetTotalCount());
}
public static void ShutdownThreads()
{
CumulativeCommandEventCounterStream.Reset();
CumulativeThreadPoolEventCounterStream.Reset();
RollingCommandEventCounterStream.Reset();
RollingThreadPoolEventCounterStream.Reset();
RollingCollapserEventCounterStream.Reset();
RollingCollapserEventCounterStream.Reset();
HealthCountsStream.Reset();
RollingCollapserBatchSizeDistributionStream.Reset();
RollingCommandLatencyDistributionStream.Reset();
RollingCommandUserLatencyDistributionStream.Reset();
RollingCommandMaxConcurrencyStream.Reset();
RollingThreadPoolMaxConcurrencyStream.Reset();
}
HystrixCommandMetrics(IHystrixCommandKey key, IHystrixCommandGroupKey commandGroup, IHystrixThreadPoolKey threadPoolKey, IHystrixCommandOptions properties, HystrixEventNotifier eventNotifier) : base(null)
{
this.key = key;
this.group = commandGroup;
this.threadPoolKey = threadPoolKey;
this.properties = properties;
healthCountsStream = HealthCountsStream.GetInstance(key, properties);
rollingCommandEventCounterStream = RollingCommandEventCounterStream.GetInstance(key, properties);
cumulativeCommandEventCounterStream = CumulativeCommandEventCounterStream.GetInstance(key, properties);
rollingCommandLatencyDistributionStream = RollingCommandLatencyDistributionStream.GetInstance(key, properties);
rollingCommandUserLatencyDistributionStream = RollingCommandUserLatencyDistributionStream.GetInstance(key, properties);
rollingCommandMaxConcurrencyStream = RollingCommandMaxConcurrencyStream.GetInstance(key, properties);
}
internal static void Reset()
{
foreach (HystrixCommandMetrics metricsInstance in GetInstances())
{
metricsInstance.UnsubscribeAll();
}
RollingCommandEventCounterStream.Reset();
CumulativeCommandEventCounterStream.Reset();
RollingCommandLatencyDistributionStream.Reset();
RollingCommandUserLatencyDistributionStream.Reset();
RollingCommandMaxConcurrencyStream.Reset();
HystrixThreadEventStream.Reset();
HealthCountsStream.Reset();
Metrics.Clear();
}
public void TestResponseFromCacheDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-G");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
// should get 1 SUCCESS and 1 RESPONSE_FROM_CACHE
List <Command> commands = Command.GetCommandsWithResponseFromCache(GroupKey, key);
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
Assert.True(distribution.GetTotalCount() <= 1);
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
foreach (Command cmd in commands)
{
cmd.Observe();
}
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.False(true, "Interrupted ex");
}
Assert.Equal(1, stream.Latest.GetTotalCount());
AssertBetween(0, 30, stream.LatestMean);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
}
public async Task TestShortCircuitedCommandDoesNotGetLatencyTracked()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-Latency-D");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 3 failures is enough to trigger short-circuit. execute those, then wait for bucket to roll
// next command should be a short-circuit
var commands = new List <Command>();
for (var i = 0; i < 3; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.FAILURE, 0));
}
var shortCircuit = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
foreach (var cmd in commands)
{
await cmd.Observe();
}
Assert.True(WaitForHealthCountToUpdate(key.Name, 500, output), "health count took to long to update");
try
{
await shortCircuit.Observe();
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(3, stream.Latest.GetTotalCount());
AssertBetween(0, 75, stream.LatestMean);
Assert.True(shortCircuit.IsResponseShortCircuited);
}
public async void TestSingleBucketGetsStored()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-B");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.TIMEOUT); // latency = 600
await cmd1.Observe();
await cmd2.Observe();
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
AssertBetween(100, 400, stream.LatestMean);
AssertBetween(10, 100, stream.GetLatestPercentile(0.0));
AssertBetween(300, 800, stream.GetLatestPercentile(100.0));
}
public void TestResponseFromCacheDoesNotGetLatencyTracked()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-Latency-G");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// should get 1 SUCCESS and 1 RESPONSE_FROM_CACHE
var commands = Command.GetCommandsWithResponseFromCache(GroupKey, key);
foreach (var cmd in commands)
{
_ = cmd.ExecuteAsync();
}
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(1, stream.Latest.GetTotalCount());
AssertBetween(0, 75, stream.LatestMean);
}
public async Task TestSemaphoreRejectedCommandDoesNotGetLatencyTracked()
{
var key = HystrixCommandKeyDefault.AsKey("CMD-Latency-F");
var latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 10 commands with latency should occupy all semaphores. execute those, then wait for bucket to roll
// next command should be a semaphore rejection
var commands = new List <Command>();
for (var i = 0; i < 10; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 500, ExecutionIsolationStrategy.SEMAPHORE));
}
var semaphoreRejected = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
var satTasks = new List <Task>();
foreach (var saturator in commands)
{
satTasks.Add(Task.Run(() => saturator.Execute()));
}
await Task.Delay(50);
await Task.Run(() => semaphoreRejected.Execute());
Task.WaitAll(satTasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(10, stream.Latest.GetTotalCount());
AssertBetween(500, 750, stream.LatestMean);
Assert.True(semaphoreRejected.IsResponseSemaphoreRejected);
}
public async void TestThreadPoolRejectedCommandDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-E");
CountdownEvent latch = new CountdownEvent(1);
var observer = new LatchedObserver(output, latch);
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
latchSubscription = stream.Observe().Subscribe(observer);
Assert.True(Time.WaitUntil(() => observer.StreamRunning, 1000), "Stream failed to start");
// 10 commands with latency should occupy the entire threadpool. execute those, then wait for bucket to roll
// next command should be a thread-pool rejection
List <Command> commands = new List <Command>();
for (int i = 0; i < 10; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 500));
}
Command threadPoolRejected = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
List <Task> satTasks = new List <Task>();
foreach (Command cmd in commands)
{
satTasks.Add(cmd.ExecuteAsync());
}
await threadPoolRejected.Observe();
Task.WaitAll(satTasks.ToArray());
Assert.True(WaitForLatchedObserverToUpdate(observer, 1, 500, output), "Latch took to long to update");
Assert.Equal(10, stream.Latest.GetTotalCount());
AssertBetween(500, 750, stream.LatestMean);
Assert.True(threadPoolRejected.IsResponseThreadPoolRejected);
}
public void TestSemaphoreRejectedCommandDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-F");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
// 10 commands with latency should occupy all semaphores. execute those, then wait for bucket to roll
// next command should be a semaphore rejection
List <Command> commands = new List <Command>();
for (int i = 0; i < 10; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 200, ExecutionIsolationStrategy.SEMAPHORE));
}
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
if (distribution.GetTotalCount() > 0)
{
AssertBetween(200, 250, (int)distribution.GetMean());
}
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
foreach (Command cmd in commands)
{
Task t = new Task(
() =>
{
cmd.Observe();
}, CancellationToken.None,
TaskCreationOptions.LongRunning);
t.Start();
}
Command semaphoreRejected = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
try
{
Time.Wait(40);
semaphoreRejected.Observe();
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(10, stream.Latest.GetTotalCount());
AssertBetween(200, 250, stream.LatestMean);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(semaphoreRejected.IsResponseSemaphoreRejected);
}
