public void TestShortCircuitedCommandDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-D");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
//3 failures is enough to trigger short-circuit. execute those, then wait for bucket to roll
//next command should be a short-circuit
List <Command> commands = new List <Command>();
for (int i = 0; i < 3; i++)
{
commands.Add(Command.From(groupKey, key, HystrixEventType.FAILURE, 0));
}
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + DateTime.Now.Ticks / 10000 + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
AssertBetween(0, 30, (int)distribution.GetMean());
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
foreach (Command cmd in commands)
{
cmd.Observe();
}
Command shortCircuit = Command.From(groupKey, key, HystrixEventType.SUCCESS);
try
{
Time.Wait(200);
shortCircuit.Observe();
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(3, stream.Latest.GetTotalCount());
AssertBetween(0, 30, stream.LatestMean);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(shortCircuit.IsResponseShortCircuited);
}
public void TestTwoSubscribersOneUnsubscribes()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Health-O");
stream = HealthCountsStream.GetInstance(key, 10, 100);
CountdownEvent latch1 = new CountdownEvent(1);
CountdownEvent latch2 = new CountdownEvent(1);
AtomicInteger healthCounts1 = new AtomicInteger(0);
AtomicInteger healthCounts2 = new AtomicInteger(0);
IDisposable s1 = stream
.Observe()
.Take(10)
.ObserveOn(TaskPoolScheduler.Default)
.Finally(() =>
{
latch1.SignalEx();
})
.Subscribe(
(healthCounts) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Health 1 OnNext : " + healthCounts);
healthCounts1.IncrementAndGet();
},
(e) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Health 1 OnError : " + e);
latch1.SignalEx();
},
() =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Health 1 OnCompleted");
latch1.SignalEx();
});
IDisposable s2 = stream
.Observe()
.Take(10)
.ObserveOn(TaskPoolScheduler.Default)
.Finally(() =>
{
latch2.SignalEx();
})
.Subscribe(
(healthCounts) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Health 2 OnNext : " + healthCounts + " : " + healthCounts2.Value);
healthCounts2.IncrementAndGet();
},
(e) =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Health 2 OnError : " + e);
latch2.SignalEx();
},
() =>
{
output.WriteLine(Time.CurrentTimeMillis + " : " + Thread.CurrentThread.ManagedThreadId + " : Health 2 OnCompleted");
latch2.SignalEx();
});
// execute 5 commands, then unsubscribe from first stream. then execute the rest
for (int i = 0; i < 10; i++)
{
HystrixCommand <int> cmd = Command.From(groupKey, key, HystrixEventType.SUCCESS, 20);
cmd.Execute();
if (i == 5)
{
s1.Dispose();
}
}
Assert.True(stream.IsSourceCurrentlySubscribed); // only 1/2 subscriptions has been cancelled
Assert.True(latch1.Wait(10000));
Assert.True(latch2.Wait(10000));
output.WriteLine("s1 got : " + healthCounts1.Value + ", s2 got : " + healthCounts2.Value);
Assert.True(healthCounts1.Value >= 0);
Assert.True(healthCounts2.Value > 0);
Assert.True(healthCounts2.Value > healthCounts1.Value);
s2.Dispose();
}
public void TestBooleanCodeDefault()
{
HystrixCommandOptions properties = new HystrixCommandOptions(HystrixCommandKeyDefault.AsKey("TEST"), new HystrixCommandOptions());
Assert.Equal(HystrixCommandOptions.Default_CircuitBreakerForceClosed, properties.CircuitBreakerForceClosed);
}
public void TestCacheWithoutContext()
{
this.context.Dispose();
Assert.Throws <InvalidOperationException>(() => { HystrixRequestCache.GetInstance(HystrixCommandKeyDefault.AsKey("command1")).Get <Task <string> >("any"); });
}
public ExecutionReport PlaceOrder(ExecutionReport order)
{
var options = new HystrixCommandOptions(HystrixCommandGroupKeyDefault.AsKey("OMS"), HystrixCommandKeyDefault.AsKey("OMS.NewOrder"));
var cmd = new HystrixCommand <ExecutionReport>(options,
run: () => PlaceOrderRun(order),
fallback: () => PlaceOrderFallback(order));
// Thread.Sleep(1000);
var result = cmd.Execute();
return(result);
}
public void RegisterHystrixCommand_AddsToContainer()
{
ContainerBuilder services = new ContainerBuilder();
IConfiguration config = new ConfigurationBuilder().Build();
HystrixContainerBuilderExtensions.RegisterHystrixCommand <DummyCommand>(services, groupKey, config);
var provider = services.Build();
var command = provider.Resolve <DummyCommand>();
Assert.NotNull(command);
Assert.Equal(groupKey, command.CommandGroup);
var expectedCommandKey = HystrixCommandKeyDefault.AsKey(typeof(DummyCommand).Name);
Assert.Equal(expectedCommandKey, command.CommandKey);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
services = new ContainerBuilder();
config = new ConfigurationBuilder().Build();
HystrixContainerBuilderExtensions.RegisterHystrixCommand <IDummyCommand, DummyCommand>(services, groupKey, config);
provider = services.Build();
var icommand = provider.Resolve <IDummyCommand>();
Assert.NotNull(icommand);
command = icommand as DummyCommand;
Assert.NotNull(command);
Assert.Equal(groupKey, command.CommandGroup);
Assert.Equal(expectedCommandKey, command.CommandKey);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
services = new ContainerBuilder();
HystrixContainerBuilderExtensions.RegisterHystrixCommand <DummyCommand>(services, "GroupKey", config);
provider = services.Build();
command = provider.Resolve <DummyCommand>();
Assert.NotNull(command);
Assert.Equal("GroupKey", command.CommandGroup.Name);
expectedCommandKey = HystrixCommandKeyDefault.AsKey(typeof(DummyCommand).Name);
Assert.Equal(expectedCommandKey, command.CommandKey);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
services = new ContainerBuilder();
config = new ConfigurationBuilder().Build();
HystrixContainerBuilderExtensions.RegisterHystrixCommand <IDummyCommand, DummyCommand>(services, "GroupKey", config);
provider = services.Build();
icommand = provider.Resolve <IDummyCommand>();
Assert.NotNull(icommand);
command = icommand as DummyCommand;
Assert.NotNull(command);
Assert.Equal("GroupKey", command.CommandGroup.Name);
Assert.Equal(expectedCommandKey, command.CommandKey);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
services = new ContainerBuilder();
config = new ConfigurationBuilder().Build();
HystrixContainerBuilderExtensions.RegisterHystrixCommand <DummyCommand>(services, groupKey, commandKey, config);
provider = services.Build();
command = provider.Resolve <DummyCommand>();
Assert.NotNull(command);
Assert.Equal(groupKey, command.CommandGroup);
Assert.Equal(commandKey, command.CommandKey);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
services = new ContainerBuilder();
config = new ConfigurationBuilder().Build();
HystrixContainerBuilderExtensions.RegisterHystrixCommand <IDummyCommand, DummyCommand>(services, groupKey, commandKey, config);
provider = services.Build();
icommand = provider.Resolve <IDummyCommand>();
Assert.NotNull(icommand);
command = icommand as DummyCommand;
Assert.NotNull(command);
Assert.Equal(groupKey, command.CommandGroup);
Assert.Equal(commandKey, command.CommandKey);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
services = new ContainerBuilder();
HystrixContainerBuilderExtensions.RegisterHystrixCommand <DummyCommand>(services, "GroupKey", "CommandKey", config);
provider = services.Build();
command = provider.Resolve <DummyCommand>();
Assert.NotNull(command);
Assert.Equal("GroupKey", command.CommandGroup.Name);
Assert.Equal("CommandKey", command.CommandKey.Name);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
services = new ContainerBuilder();
config = new ConfigurationBuilder().Build();
HystrixContainerBuilderExtensions.RegisterHystrixCommand <IDummyCommand, DummyCommand>(services, "GroupKey", "CommandKey", config);
provider = services.Build();
icommand = provider.Resolve <IDummyCommand>();
Assert.NotNull(icommand);
command = icommand as DummyCommand;
Assert.NotNull(command);
Assert.Equal("GroupKey", command.CommandGroup.Name);
Assert.Equal("CommandKey", command.CommandKey.Name);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
}
public void TestSemaphoreRejected()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-RollingCounter-H");
stream = RollingCommandEventCounterStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(this.output, latch));
// 10 commands will saturate semaphore when called from different threads.
// submit 2 more requests and they should be SEMAPHORE_REJECTED
// should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes
List <CommandStreamTest.Command> saturators = new List <CommandStreamTest.Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(CommandStreamTest.Command.From(GroupKey, key, HystrixEventType.SUCCESS, 400, ExecutionIsolationStrategy.SEMAPHORE));
}
CommandStreamTest.Command rejected1 = CommandStreamTest.Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
CommandStreamTest.Command rejected2 = CommandStreamTest.Command.From(GroupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
foreach (Command c in saturators)
{
new Thread(new ThreadStart(() => c.Observe())).Start();
}
try
{
Time.Wait(100);
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
rejected1.Observe();
rejected2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(rejected1.IsResponseSemaphoreRejected);
Assert.True(rejected2.IsResponseSemaphoreRejected);
Assert.Equal(HystrixEventTypeHelper.Values.Count, stream.Latest.Length);
long[] expected = new long[HystrixEventTypeHelper.Values.Count];
expected[(int)HystrixEventType.SUCCESS] = 10;
expected[(int)HystrixEventType.SEMAPHORE_REJECTED] = 2;
expected[(int)HystrixEventType.FALLBACK_SUCCESS] = 2;
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.Equal <long[]>(expected, stream.Latest);
}
public void AddHystrixCommand_AddsToContainer()
{
IServiceCollection services = new ServiceCollection();
IConfiguration config = new ConfigurationBuilder().Build();
HystrixServiceCollectionExtensions.AddHystrixCommand <DummyCommand>(services, groupKey, config);
var provider = services.BuildServiceProvider();
var command = provider.GetService <DummyCommand>();
Assert.NotNull(command);
Assert.Equal(groupKey, command.CommandGroup);
var expectedCommandKey = HystrixCommandKeyDefault.AsKey(typeof(DummyCommand).Name);
Assert.Equal(expectedCommandKey, command.CommandKey);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
var expectedThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey(groupKey.Name);
Assert.Equal(expectedThreadPoolKey, command.Options.ThreadPoolKey);
var threadOptions = command.Options.ThreadPoolOptions as HystrixThreadPoolOptions;
Assert.NotNull(threadOptions);
Assert.NotNull(threadOptions._dynamic);
Assert.Equal(expectedThreadPoolKey, threadOptions.ThreadPoolKey);
services = new ServiceCollection();
config = new ConfigurationBuilder().Build();
HystrixServiceCollectionExtensions.AddHystrixCommand <IDummyCommand, DummyCommand>(services, groupKey, config);
provider = services.BuildServiceProvider();
var icommand = provider.GetService <IDummyCommand>();
Assert.NotNull(icommand);
command = icommand as DummyCommand;
Assert.NotNull(command);
Assert.Equal(groupKey, command.CommandGroup);
Assert.Equal(expectedCommandKey, command.CommandKey);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
expectedThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey(groupKey.Name);
Assert.Equal(expectedThreadPoolKey, command.Options.ThreadPoolKey);
threadOptions = command.Options.ThreadPoolOptions as HystrixThreadPoolOptions;
Assert.NotNull(threadOptions);
Assert.NotNull(threadOptions._dynamic);
Assert.Equal(expectedThreadPoolKey, threadOptions.ThreadPoolKey);
services = new ServiceCollection();
HystrixServiceCollectionExtensions.AddHystrixCommand <DummyCommand>(services, "GroupKey", config);
provider = services.BuildServiceProvider();
command = provider.GetService <DummyCommand>();
Assert.NotNull(command);
Assert.Equal("GroupKey", command.CommandGroup.Name);
expectedCommandKey = HystrixCommandKeyDefault.AsKey(typeof(DummyCommand).Name);
Assert.Equal(expectedCommandKey, command.CommandKey);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
expectedThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey(command.CommandGroup.Name);
Assert.Equal(expectedThreadPoolKey, command.Options.ThreadPoolKey);
threadOptions = command.Options.ThreadPoolOptions as HystrixThreadPoolOptions;
Assert.NotNull(threadOptions);
Assert.NotNull(threadOptions._dynamic);
Assert.Equal(expectedThreadPoolKey, threadOptions.ThreadPoolKey);
services = new ServiceCollection();
config = new ConfigurationBuilder().Build();
HystrixServiceCollectionExtensions.AddHystrixCommand <IDummyCommand, DummyCommand>(services, "GroupKey", config);
provider = services.BuildServiceProvider();
icommand = provider.GetService <IDummyCommand>();
Assert.NotNull(icommand);
command = icommand as DummyCommand;
Assert.NotNull(command);
Assert.Equal("GroupKey", command.CommandGroup.Name);
Assert.Equal(expectedCommandKey, command.CommandKey);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
expectedThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey(command.CommandGroup.Name);
Assert.Equal(expectedThreadPoolKey, command.Options.ThreadPoolKey);
threadOptions = command.Options.ThreadPoolOptions as HystrixThreadPoolOptions;
Assert.NotNull(threadOptions);
Assert.NotNull(threadOptions._dynamic);
Assert.Equal(expectedThreadPoolKey, threadOptions.ThreadPoolKey);
services = new ServiceCollection();
config = new ConfigurationBuilder().Build();
HystrixServiceCollectionExtensions.AddHystrixCommand <DummyCommand>(services, groupKey, commandKey, config);
provider = services.BuildServiceProvider();
command = provider.GetService <DummyCommand>();
Assert.NotNull(command);
Assert.Equal(groupKey, command.CommandGroup);
Assert.Equal(commandKey, command.CommandKey);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
expectedThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey(groupKey.Name);
Assert.Equal(expectedThreadPoolKey, command.Options.ThreadPoolKey);
threadOptions = command.Options.ThreadPoolOptions as HystrixThreadPoolOptions;
Assert.NotNull(threadOptions);
Assert.NotNull(threadOptions._dynamic);
Assert.Equal(expectedThreadPoolKey, threadOptions.ThreadPoolKey);
services = new ServiceCollection();
config = new ConfigurationBuilder().Build();
HystrixServiceCollectionExtensions.AddHystrixCommand <IDummyCommand, DummyCommand>(services, groupKey, commandKey, config);
provider = services.BuildServiceProvider();
icommand = provider.GetService <IDummyCommand>();
Assert.NotNull(icommand);
command = icommand as DummyCommand;
Assert.NotNull(command);
Assert.Equal(groupKey, command.CommandGroup);
Assert.Equal(commandKey, command.CommandKey);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
expectedThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey(groupKey.Name);
Assert.Equal(expectedThreadPoolKey, command.Options.ThreadPoolKey);
threadOptions = command.Options.ThreadPoolOptions as HystrixThreadPoolOptions;
Assert.NotNull(threadOptions);
Assert.NotNull(threadOptions._dynamic);
Assert.Equal(expectedThreadPoolKey, threadOptions.ThreadPoolKey);
services = new ServiceCollection();
HystrixServiceCollectionExtensions.AddHystrixCommand <DummyCommand>(services, "GroupKey", "CommandKey", config);
provider = services.BuildServiceProvider();
command = provider.GetService <DummyCommand>();
Assert.NotNull(command);
Assert.Equal("GroupKey", command.CommandGroup.Name);
Assert.Equal("CommandKey", command.CommandKey.Name);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
expectedThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey(command.CommandGroup.Name);
Assert.Equal(expectedThreadPoolKey, command.Options.ThreadPoolKey);
threadOptions = command.Options.ThreadPoolOptions as HystrixThreadPoolOptions;
Assert.NotNull(threadOptions);
Assert.NotNull(threadOptions._dynamic);
Assert.Equal(expectedThreadPoolKey, threadOptions.ThreadPoolKey);
services = new ServiceCollection();
config = new ConfigurationBuilder().Build();
HystrixServiceCollectionExtensions.AddHystrixCommand <IDummyCommand, DummyCommand>(services, "GroupKey", "CommandKey", config);
provider = services.BuildServiceProvider();
icommand = provider.GetService <IDummyCommand>();
Assert.NotNull(icommand);
command = icommand as DummyCommand;
Assert.NotNull(command);
Assert.Equal("GroupKey", command.CommandGroup.Name);
Assert.Equal("CommandKey", command.CommandKey.Name);
Assert.NotNull(command.Options);
Assert.NotNull(command.Options._dynamic);
expectedThreadPoolKey = HystrixThreadPoolKeyDefault.AsKey(command.CommandGroup.Name);
Assert.Equal(expectedThreadPoolKey, command.Options.ThreadPoolKey);
threadOptions = command.Options.ThreadPoolOptions as HystrixThreadPoolOptions;
Assert.NotNull(threadOptions);
Assert.NotNull(threadOptions._dynamic);
Assert.Equal(expectedThreadPoolKey, threadOptions.ThreadPoolKey);
}
public void TestFallbackRejection()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-L");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-L");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-L");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
//fallback semaphore size is 5. So let 5 commands saturate that semaphore, then
//let 2 more commands go to fallback. they should get rejected by the fallback-semaphore
List <CommandStreamTest.Command> fallbackSaturators = new List <CommandStreamTest.Command>();
for (int i = 0; i < 5; i++)
{
fallbackSaturators.Add(CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 400));
}
CommandStreamTest.Command rejection1 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0);
CommandStreamTest.Command rejection2 = CommandStreamTest.Command.From(groupKey, key, HystrixEventType.FAILURE, 20, HystrixEventType.FALLBACK_SUCCESS, 0);
foreach (CommandStreamTest.Command saturator in fallbackSaturators)
{
saturator.Observe();
}
try
{
Time.Wait(70);
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
rejection1.Observe();
rejection2.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
//all 7 commands executed on-thread, so should be executed according to thread-pool metrics
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(7, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestMultipleBucketsBothGetStoredAndThenAgeOut()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-I");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(30).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
if (distribution.GetTotalCount() == 2)
{
AssertBetween(55, 90, distribution.GetMean());
}
if (distribution.GetTotalCount() == 5)
{
AssertBetween(60, 90, distribution.GetMean());
}
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
Command cmd1 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 10);
Command cmd2 = Command.From(GroupKey, key, HystrixEventType.FAILURE, 100);
cmd1.Observe();
cmd2.Observe();
try
{
Time.Wait(500);
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Command cmd3 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 60);
Command cmd4 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 60);
Command cmd5 = Command.From(GroupKey, key, HystrixEventType.SUCCESS, 70);
cmd3.Observe();
cmd4.Observe();
cmd5.Observe();
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(0, stream.Latest.GetTotalCount());
}
public void TestSemaphoreRejected()
{
IHystrixCommandGroupKey groupKey = HystrixCommandGroupKeyDefault.AsKey("Cumulative-ThreadPool-H");
IHystrixThreadPoolKey threadPoolKey = HystrixThreadPoolKeyDefault.AsKey("Cumulative-ThreadPool-H");
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("Cumulative-Counter-H");
stream = CumulativeThreadPoolEventCounterStream.GetInstance(threadPoolKey, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(new LatchedObserver(output, latch));
// 10 commands will saturate semaphore when called from different threads.
// submit 2 more requests and they should be SEMAPHORE_REJECTED
// should see 10 SUCCESSes, 2 SEMAPHORE_REJECTED and 2 FALLBACK_SUCCESSes
List <Command> saturators = new List <Command>();
for (int i = 0; i < 10; i++)
{
saturators.Add(Command.From(groupKey, key, HystrixEventType.SUCCESS, 300, ExecutionIsolationStrategy.SEMAPHORE));
}
Command rejected1 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
Command rejected2 = Command.From(groupKey, key, HystrixEventType.SUCCESS, 0, ExecutionIsolationStrategy.SEMAPHORE);
foreach (Command c in saturators)
{
Task t = new Task(
() =>
{
c.Observe();
}, CancellationToken.None,
TaskCreationOptions.LongRunning);
t.Start();
Task.Run(() => c.Observe());
}
Time.Wait(10);
Task.Run(() => rejected1.Observe());
Task.Run(() => rejected2.Observe());
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(rejected1.IsResponseSemaphoreRejected);
Assert.True(rejected2.IsResponseSemaphoreRejected);
Assert.Equal(2, stream.Latest.Length);
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.EXECUTED));
Assert.Equal(0, stream.GetLatestCount(ThreadPoolEventType.REJECTED));
}
public void TestSemaphoreRejectedCommandDoesNotGetLatencyTracked()
{
IHystrixCommandKey key = HystrixCommandKeyDefault.AsKey("CMD-Latency-F");
stream = RollingCommandLatencyDistributionStream.GetInstance(key, 10, 100);
stream.StartCachingStreamValuesIfUnstarted();
// 10 commands with latency should occupy all semaphores. execute those, then wait for bucket to roll
// next command should be a semaphore rejection
List <Command> commands = new List <Command>();
for (int i = 0; i < 10; i++)
{
commands.Add(Command.From(GroupKey, key, HystrixEventType.SUCCESS, 200, ExecutionIsolationStrategy.SEMAPHORE));
}
CountdownEvent latch = new CountdownEvent(1);
stream.Observe().Take(10).Subscribe(
(distribution) =>
{
output.WriteLine("OnNext @ " + (DateTime.Now.Ticks / 10000) + " : " + distribution.GetMean() + "/" + distribution.GetTotalCount() + " " + Thread.CurrentThread.ManagedThreadId);
if (distribution.GetTotalCount() > 0)
{
AssertBetween(200, 250, (int)distribution.GetMean());
}
},
(e) =>
{
Assert.True(false, e.Message);
},
() =>
{
latch.SignalEx();
});
foreach (Command cmd in commands)
{
Task t = new Task(
() =>
{
cmd.Observe();
}, CancellationToken.None,
TaskCreationOptions.LongRunning);
t.Start();
}
Command semaphoreRejected = Command.From(GroupKey, key, HystrixEventType.SUCCESS);
try
{
Time.Wait(40);
semaphoreRejected.Observe();
}
catch (Exception ie)
{
Assert.True(false, ie.Message);
}
try
{
Assert.True(latch.Wait(10000));
}
catch (Exception)
{
Assert.True(false, "Interrupted ex");
}
Assert.Equal(10, stream.Latest.GetTotalCount());
AssertBetween(200, 250, stream.LatestMean);
output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
Assert.True(semaphoreRejected.IsResponseSemaphoreRejected);
}
public async Task TestMultipleTimeWindowRetriesBeforeClosingCircuit()
{
string key = "cmd-H";
int sleepWindow = 400;
HystrixCommand <bool> cmd1 = new FailureCommand(key, 0);
IHystrixCircuitBreaker cb = cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed when expected!");
Assert.False(cb.IsOpen, Time.CurrentTimeMillis + " Circuit breaker is open when it should be closed!");
_ = await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new FailureCommand(key, 0);
_ = await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 0);
_ = await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new TimeoutCommand(key);
_ = await cmd4.ExecuteAsync();
// everything has failed in the test window so we should return false now
// Allow window to pass,
// Time.Wait(200);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine(Time.CurrentTimeMillis + " !!!! 1 4 failures, circuit will open on recalc");
// Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, Time.CurrentTimeMillis + " Circuit is closed when it should be open!");
// wait for sleepWindow to pass
output.WriteLine(Time.CurrentTimeMillis + " !!!! 2 Sleep window starting where all commands fail-fast");
Time.Wait(sleepWindow + 50);
output.WriteLine(Time.CurrentTimeMillis + " !!!! 3 Sleep window over, should allow singleTest()");
// but the circuit should still be open
Assert.True(cb.IsOpen, Time.CurrentTimeMillis + " Circuit is closed when it should be open!");
// we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open
HystrixCommand <bool> cmd5 = new FailureCommand(key, 50);
IObservable <bool> asyncResult5 = cmd5.Observe();
output.WriteLine(Time.CurrentTimeMillis + " !!!! Kicked off the single-test");
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed when NOT expected!");
output.WriteLine(Time.CurrentTimeMillis + " !!!! Confirmed that no other requests go out during single-test");
await asyncResult5.SingleAsync();
output.WriteLine(Time.CurrentTimeMillis + " !!!! SingleTest just completed");
// all requests should still be blocked, because the singleTest failed
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (1) when NOT expected!");
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (2) when NOT expected!");
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (3) when NOT expected!");
// wait for sleepWindow to pass
output.WriteLine(Time.CurrentTimeMillis + " !!!! 2nd sleep window START");
Time.Wait(sleepWindow + 50);
output.WriteLine(Time.CurrentTimeMillis + " !!!! 2nd sleep window over");
// we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open
HystrixCommand <bool> cmd6 = new FailureCommand(key, 50);
IObservable <bool> asyncResult6 = cmd6.Observe();
output.WriteLine(Time.CurrentTimeMillis + " 2nd singleTest just kicked off");
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed when NOT expected!");
Assert.False(await asyncResult6.SingleAsync());
output.WriteLine(Time.CurrentTimeMillis + " 2nd singleTest now over");
// all requests should still be blocked, because the singleTest failed
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (1) when NOT expected!");
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (2) when NOT expected!");
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed (3) when NOT expected!");
// wait for sleepWindow to pass
Time.Wait(sleepWindow);
// but the circuit should still be open
Assert.True(cb.IsOpen, Time.CurrentTimeMillis + " Circuit is closed when it should be open!");
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand <bool> cmd7 = new SuccessCommand(key, 50);
IObservable <bool> asyncResult7 = cmd7.Observe();
// and further requests are still blocked while the singleTest command is in flight
Assert.False(cb.AllowRequest, Time.CurrentTimeMillis + " Request allowed when NOT expected!");
await asyncResult7.SingleAsync();
// all requests should be open again
Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed (1) when expected!");
Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed (2) when expected!");
Assert.True(cb.AllowRequest, Time.CurrentTimeMillis + " Request NOT allowed (3) when expected!");
// and the circuit should be closed again
Assert.False(cb.IsOpen, Time.CurrentTimeMillis + " Circuit breaker is open when it should be closed!");
// and the circuit should be closed again
Assert.False(cb.IsOpen, Time.CurrentTimeMillis + " Circuit breaker is open when it should be closed!");
}
public async Task TestCircuitClosedAfterSuccess()
{
string key = "cmd-G";
int sleepWindow = 400;
HystrixCommand <bool> cmd1 = new FailureCommand(key, 0, sleepWindow);
HystrixCircuitBreakerImpl cb = (HystrixCircuitBreakerImpl)cmd1._circuitBreaker;
var stream = HealthCountsStream.GetInstance(HystrixCommandKeyDefault.AsKey(key), cmd1.CommandOptions);
Assert.True(WaitForHealthCountToUpdate(key, 1000, output), "Health count stream failed to start");
// this should start as allowing requests
Assert.True(cb.AllowRequest, "Request NOT allowed when expected!");
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
_ = await cmd1.ExecuteAsync();
HystrixCommand <bool> cmd2 = new FailureCommand(key, 0, sleepWindow);
_ = await cmd2.ExecuteAsync();
HystrixCommand <bool> cmd3 = new FailureCommand(key, 0, sleepWindow);
_ = await cmd3.ExecuteAsync();
HystrixCommand <bool> cmd4 = new TimeoutCommand(key, sleepWindow);
_ = await cmd4.ExecuteAsync();
// Allow window to pass, everything has failed in the test window so we should return false now
// Time.Wait(200);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine("ReqLog : " + Time.CurrentTimeMillis + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString());
output.WriteLine("CircuitBreaker state 1 : " + Time.CurrentTimeMillis + cmd1.Metrics.Healthcounts);
Assert.False(cb.AllowRequest, "Request allowed when NOT expected!");
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
// wait for sleepWindow to pass
Time.Wait(sleepWindow + 100);
// but the circuit should still be open
Assert.True(cb.IsOpen, "Circuit is closed when it should be open!");
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand <bool> cmd5 = new SuccessCommand(key, 10, sleepWindow);
output.WriteLine("Starting test cmd : " + Time.CurrentTimeMillis + cmd1.Metrics.Healthcounts);
_ = await cmd5.Observe();
// Allow window to pass, all requests should be open again
// Time.Wait(200);
Assert.True(WaitForHealthCountToUpdate(key, 250, output), "Health count stream failed to update");
output.WriteLine("CircuitBreaker state 2 : " + Time.CurrentTimeMillis + cmd1.Metrics.Healthcounts);
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (1)!");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (2)!");
Assert.True(cb.AllowRequest, "Request NOT allowed when expected (3)!");
// and the circuit should be closed again
Assert.False(cb.IsOpen, "Circuit breaker is open when it should be closed!");
}
public string GetQuote()
{
var options = new HystrixCommandOptions(HystrixCommandGroupKeyDefault.AsKey("Legacy"), HystrixCommandKeyDefault.AsKey("Cookie.Asmx"));
var cmd = new HystrixCommand <string>(options,
run: GetCookieRun,
fallback: GetCookieFallback);
return(cmd.Execute());
}
public async Task <string> GetQuoteAsync()
{
var options = new HystrixCommandOptions(HystrixCommandGroupKeyDefault.AsKey("Legacy"), HystrixCommandKeyDefault.AsKey("Legacy.Wcf"));
var cmd = new HystrixCommand <string>(options,
run: GetCookieRun,
fallback: GetCookieFallback);
return(await cmd.ExecuteAsync());
}
