public void Command_ExecutionHookFailureWithSemaphoreIsolation()
{
/* test with Execute() */
TryableSemaphore semaphore =
new TryableSemaphore(HystrixPropertyFactory.AsProperty(0));
TestSemaphoreCommand command = new TestSemaphoreCommand(new TestCircuitBreaker(), semaphore, TimeSpan.FromMilliseconds(200));
try
{
command.Execute();
Assert.Fail("we expect a failure");
}
catch (Exception)
{
// expected
}
Assert.IsFalse(command.IsExecutedInThread);
Assert.IsTrue(command.IsResponseRejected);
// the run() method should not run as we are rejected
Assert.AreEqual(0, command.Builder.ExecutionHook.StartRun);
// null as run() does not get invoked
Assert.IsNull(command.Builder.ExecutionHook.RunSuccessResponse);
// null as run() does not get invoked
Assert.IsNull(command.Builder.ExecutionHook.RunFailureException);
// the fallback() method should run because of rejection
Assert.AreEqual(1, command.Builder.ExecutionHook.StartFallback);
// null since there is no fallback
Assert.IsNull(command.Builder.ExecutionHook.FallbackSuccessResponse);
// not null since the fallback is not implemented
Assert.IsNotNull(command.Builder.ExecutionHook.FallbackFailureException);
// the Execute() method was used
Assert.AreEqual(1, command.Builder.ExecutionHook.StartExecute);
// we should not have a response since fallback has nothing
Assert.IsNull(command.Builder.ExecutionHook.EndExecuteSuccessResponse);
// we won't have an exception because rejection doesn't have one
Assert.IsNull(command.Builder.ExecutionHook.EndExecuteFailureException);
// but we do expect to receive a onError call with FailureType.Shortcircuit
Assert.AreEqual(FailureType.RejectedSemaphoreExecution, command.Builder.ExecutionHook.EndExecuteFailureType);
// thread execution
Assert.AreEqual(0, command.Builder.ExecutionHook.ThreadStart);
Assert.AreEqual(0, command.Builder.ExecutionHook.ThreadComplete);
Hystrix.Reset();
}
public void Command_ExecutionSemaphoreWithExecution()
{
TestCircuitBreaker circuitBreaker = new TestCircuitBreaker();
// single thread should work
try
{
TestSemaphoreCommand command = new TestSemaphoreCommand(circuitBreaker, 1, TimeSpan.FromMilliseconds(200));
bool result = command.Execute();
Assert.IsFalse(command.IsExecutedInThread);
Assert.IsTrue(result);
}
catch (Exception e)
{
// we shouldn't fail on this one
throw new Exception("Unexpected exception.", e);
}
ArrayBlockingQueue<bool> results = new ArrayBlockingQueue<bool>(2);
AtomicBoolean exceptionReceived = new AtomicBoolean();
TryableSemaphore semaphore =
new TryableSemaphore(HystrixPropertyFactory.AsProperty(1));
IRunnable r = new HystrixContextRunnable(new Runnable(() =>
{
try
{
results.Add(new TestSemaphoreCommand(circuitBreaker, semaphore, TimeSpan.FromMilliseconds(200)).Execute());
}
catch (Exception)
{
exceptionReceived.Value = true;
}
}));
// 2 threads, the second should be rejected by the semaphore
Thread t1 = new Thread(r.Run);
Thread t2 = new Thread(r.Run);
t1.Start();
t2.Start();
try
{
t1.Join();
t2.Join();
}
catch (Exception)
{
Assert.Fail("failed waiting on threads");
}
if (!exceptionReceived.Value)
{
Assert.Fail("We expected an exception on the 2nd get");
}
// only 1 value is expected as the other should have thrown an exception
Assert.AreEqual(1, results.Count);
// should contain only a true result
Assert.IsTrue(results.Contains(true));
Assert.IsFalse(results.Contains(false));
Assert.AreEqual(2, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Success));
Assert.AreEqual(1, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown));
// no failure ... we throw an exception because of rejection but the command does not fail execution
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Failure));
// there is no fallback implemented so no failure can occur on it
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess));
// we rejected via semaphore
Assert.AreEqual(1, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected));
// the rest should not be involved in this test
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Timeout));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache));
Assert.AreEqual(3, HystrixRequestLog.GetCurrentRequest().ExecutedCommands.Count());
Hystrix.Reset();
}
public void Command_SemaphorePermitsInUse()
{
TestCircuitBreaker circuitBreaker = new TestCircuitBreaker();
// this semaphore will be shared across multiple command instances
TryableSemaphore sharedSemaphore =
new TryableSemaphore(HystrixPropertyFactory.AsProperty(3));
// used to wait until all commands have started
CountdownEvent startLatch = new CountdownEvent(sharedSemaphore.NumberOfPermits.Get() + 1);
// used to signal that all command can finish
CountdownEvent sharedLatch = new CountdownEvent(1);
IRunnable sharedSemaphoreRunnable = new HystrixContextRunnable(new Runnable(() =>
{
try
{
new LatchedSemaphoreCommand(circuitBreaker, sharedSemaphore, startLatch, sharedLatch).Execute();
}
catch (Exception)
{
}
}));
// creates group of threads each using command sharing a single semaphore
// I create extra threads and commands so that I can verify that some of them fail to obtain a semaphore
int sharedThreadCount = sharedSemaphore.NumberOfPermits.Get() * 2;
Thread[] sharedSemaphoreThreads = new Thread[sharedThreadCount];
for (int i = 0; i < sharedThreadCount; i++)
{
sharedSemaphoreThreads[i] = new Thread(sharedSemaphoreRunnable.Run);
}
// creates thread using isolated semaphore
TryableSemaphore isolatedSemaphore =
new TryableSemaphore(HystrixPropertyFactory.AsProperty(1));
CountdownEvent isolatedLatch = new CountdownEvent(1);
// tracks failures to obtain semaphores
AtomicInteger failureCount = new AtomicInteger();
Thread isolatedThread = new Thread(new HystrixContextRunnable(new Runnable(() =>
{
try
{
new LatchedSemaphoreCommand(circuitBreaker, isolatedSemaphore, startLatch, isolatedLatch).Execute();
}
catch (Exception)
{
failureCount.IncrementAndGet();
}
})).Run);
// verifies no permits in use before starting threads
Assert.AreEqual(0, sharedSemaphore.GetNumberOfPermitsUsed(), "wrong number of permits for shared semaphore");
Assert.AreEqual(0, isolatedSemaphore.GetNumberOfPermitsUsed(), "wrong number of permits for isolated semaphore");
for (int i = 0; i < sharedThreadCount; i++)
{
sharedSemaphoreThreads[i].Start();
}
isolatedThread.Start();
// waits until all commands have started
try
{
startLatch.Wait(TimeSpan.FromSeconds(1.0));
}
catch (ThreadInterruptedException e)
{
throw new Exception("Unexpected exception.", e);
}
// verifies that all semaphores are in use
Assert.AreEqual(sharedSemaphore.NumberOfPermits.Get(), sharedSemaphore.GetNumberOfPermitsUsed(), "wrong number of permits for shared semaphore");
Assert.AreEqual(isolatedSemaphore.NumberOfPermits.Get(), isolatedSemaphore.GetNumberOfPermitsUsed(), "wrong number of permits for isolated semaphore");
// signals commands to finish
sharedLatch.Signal();
isolatedLatch.Signal();
try
{
for (int i = 0; i < sharedThreadCount; i++)
{
sharedSemaphoreThreads[i].Join();
}
isolatedThread.Join();
}
catch (Exception)
{
Assert.Fail("failed waiting on threads");
}
// verifies no permits in use after finishing threads
Assert.AreEqual(0, sharedSemaphore.GetNumberOfPermitsUsed(), "wrong number of permits for shared semaphore");
Assert.AreEqual(0, isolatedSemaphore.GetNumberOfPermitsUsed(), "wrong number of permits for isolated semaphore");
// verifies that some executions failed
int expectedFailures = sharedSemaphore.GetNumberOfPermitsUsed();
Assert.AreEqual(expectedFailures, failureCount.Value, "failures expected but did not happen");
Hystrix.Reset();
}
public void Command_ExecutionSemaphoreWithQueue()
{
TestCircuitBreaker circuitBreaker = new TestCircuitBreaker();
// single thread should work
try
{
bool result = new TestSemaphoreCommand(circuitBreaker, 1, TimeSpan.FromMilliseconds(200)).Queue().Get();
Assert.IsTrue(result);
}
catch (Exception e)
{
// we shouldn't fail on this one
throw new Exception("Unexpected exception.", e);
}
AtomicBoolean exceptionReceived = new AtomicBoolean();
TryableSemaphore semaphore =
new TryableSemaphore(HystrixPropertyFactory.AsProperty(1));
IRunnable r = new HystrixContextRunnable(new Runnable(() =>
{
try
{
new TestSemaphoreCommand(circuitBreaker, semaphore, TimeSpan.FromMilliseconds(200)).Queue().Get();
}
catch (Exception)
{
exceptionReceived.Value = true;
}
}));
// 2 threads, the second should be rejected by the semaphore
Thread t1 = new Thread(r.Run);
Thread t2 = new Thread(r.Run);
t1.Start();
t2.Start();
try
{
t1.Join();
t2.Join();
}
catch (Exception)
{
Assert.Fail("failed waiting on threads");
}
if (!exceptionReceived.Value)
{
Assert.Fail("We expected an exception on the 2nd get");
}
Assert.AreEqual(2, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Success));
// we don't have a fallback so threw an exception when rejected
Assert.AreEqual(1, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown));
// not a failure as the command never executed so can't fail
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Failure));
// no fallback failure as there isn't a fallback implemented
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess));
// we should have rejected via semaphore
Assert.AreEqual(1, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected));
// the rest should not be involved in this test
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.Timeout));
Assert.AreEqual(0, circuitBreaker.Metrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache));
Assert.AreEqual(3, HystrixRequestLog.GetCurrentRequest().ExecutedCommands.Count());
Hystrix.Reset();
}
