public async Task WhenFailed_FiresBreakerFailureCountMetricEvent()
{
// Arrange
var key = AnyString;
var mockMetricEvents = new Mock <IMetricEvents>();
var mockMetrics = new Mock <ICommandMetrics>();
var mockBreaker = new Mock <ICircuitBreaker>();
mockBreaker.SetupGet(m => m.Name).Returns(key);
mockBreaker.Setup(m => m.IsAllowing()).Returns(true); // We want to get past the initial rejection check.
mockBreaker.SetupGet(m => m.Metrics).Returns(mockMetrics.Object);
var mockCircuitBreakerFactory = new Mock <ICircuitBreakerFactory>(MockBehavior.Strict);
mockCircuitBreakerFactory.Setup(m => m.GetCircuitBreaker(It.IsAny <GroupKey>())).Returns(mockBreaker.Object);
var mockBreakerExceptionHandler = new Mock <IBreakerExceptionHandler>(MockBehavior.Strict);
mockBreakerExceptionHandler.Setup(m => m.IsExceptionIgnored(It.IsAny <Type>())).Returns(false);
var command = new NoOpAsyncCommand(); // Should be successful.
var invoker = new BreakerInvoker(mockCircuitBreakerFactory.Object, mockMetricEvents.Object, mockBreakerExceptionHandler.Object);
// Act
await invoker.ExecuteWithBreakerAsync(command, CancellationToken.None);
// Assert
mockMetricEvents.Verify(m => m.BreakerSuccessCount(key, command.Name));
}
public async Task WhenRejected_FiresBreakerRejectedMetricEvent()
{
// Arrange
var key = AnyString;
var mockMetricEvents = new Mock <IMetricEvents>();
var mockMetrics = new Mock <ICommandMetrics>();
var mockBreaker = new Mock <ICircuitBreaker>();
mockBreaker.SetupGet(m => m.Name).Returns(key);
mockBreaker.Setup(m => m.IsAllowing()).Returns(false); // We want to reject.
mockBreaker.SetupGet(m => m.Metrics).Returns(mockMetrics.Object);
var mockCircuitBreakerFactory = new Mock <ICircuitBreakerFactory>(MockBehavior.Strict);
mockCircuitBreakerFactory.Setup(m => m.GetCircuitBreaker(It.IsAny <GroupKey>())).Returns(mockBreaker.Object);
var mockBreakerExceptionHandler = new Mock <IBreakerExceptionHandler>(MockBehavior.Strict);
mockBreakerExceptionHandler.Setup(m => m.IsExceptionIgnored(It.IsAny <Type>())).Returns(false);
var command = new NoOpAsyncCommand();
var invoker = new BreakerInvoker(mockCircuitBreakerFactory.Object, mockMetricEvents.Object, mockBreakerExceptionHandler.Object);
// Act + Assert
await Assert.ThrowsAsync <CircuitBreakerRejectedException>(() => invoker.ExecuteWithBreakerAsync(command, CancellationToken.None));
mockMetricEvents.Verify(m => m.RejectedByBreaker(key, command.Name));
}
public void CallsTryEnterAndReleaseOnTheSameBulkheadDuringConfigChange()
{
// The assumption tested here is important. If the bulkhead max concurrent
// configuration value changes, the bulkhead holder will build a new semaphore and
// swap it out with the old one. However, any bulkheads that acquired a lock on
// the original semaphore need to release that semaphore instead of the new one.
// Otherwise, they'll release on the new one. If that happens, the new semaphore's
// counter won't be accurate respective to the number of commands concurrently
// executing with it.
// This test is complicated, but it's one of the most important unit tests in the
// project. If you need to change it, take care that it gets re-written properly.
// The test is performed by having the command itself change the config
// value, which will happen between the TryEnter and Release calls.
// Arrange
const int initialMaxConcurrent = 10;
const int newMaxConcurrent = 15;
var key = AnyString;
var groupKey = GroupKey.Named(key);
var mockConfig = new MjolnirConfiguration
{
UseCircuitBreakers = true,
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key,
new BulkheadConfiguration
{
MaxConcurrent = initialMaxConcurrent
}
}
}
};
var mockBreakerExceptionHandler = new Mock <IBreakerExceptionHandler>(MockBehavior.Strict);
mockBreakerExceptionHandler.Setup(m => m.IsExceptionIgnored(It.IsAny <Type>())).Returns(false);
var mockCircuitBreaker = new Mock <ICircuitBreaker>(MockBehavior.Strict);
mockCircuitBreaker.Setup(m => m.IsAllowing()).Returns(true);
mockCircuitBreaker.Setup(m => m.Name).Returns(AnyString);
mockCircuitBreaker.Setup(m => m.Metrics).Returns(new Mock <ICommandMetrics>().Object);
mockCircuitBreaker.Setup(m => m.MarkSuccess(It.IsAny <long>()));
var mockCircuitBreakerFactory = new Mock <ICircuitBreakerFactory>(MockBehavior.Strict);
mockCircuitBreakerFactory.Setup(m => m.GetCircuitBreaker(groupKey)).Returns(mockCircuitBreaker.Object);
var mockMetricEvents = new Mock <IMetricEvents>(); // Non-Strict: we aren't testing metric events here, let's keep the test simpler.
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new Mock <IMjolnirLog <BulkheadFactory> >().Object);
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new Mock <IMjolnirLog <SemaphoreBulkheadHolder> >().Object);
// Use a real BulkheadFactory, which will give us access to its BulkheadHolder.
var bulkheadFactory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
var holder = bulkheadFactory.GetBulkheadHolder(groupKey);
var initialBulkhead = bulkheadFactory.GetBulkhead(groupKey);
// Use a real BreakerInvoker instead of a mocked one so that we actually
// invoke the command that changes the config value.
var breakerInvoker = new BreakerInvoker(mockCircuitBreakerFactory.Object, mockMetricEvents.Object, mockBreakerExceptionHandler.Object);
var command = new ChangeBulkheadLimitSyncCommand(key, holder, newMaxConcurrent);
var invoker = new BulkheadInvoker(breakerInvoker, bulkheadFactory, mockMetricEvents.Object, mockConfig);
var unusedCancellationToken = CancellationToken.None;
// Make sure the BulkheadFactory has the expected Bulkhead initialized for the key.
Assert.Equal(initialMaxConcurrent, bulkheadFactory.GetBulkhead(groupKey).CountAvailable);
// Act
var result = invoker.ExecuteWithBulkhead(command, unusedCancellationToken);
// Assert
// The assertions here are a bit indirect and, if we were mocking, could be more
// deterministic. We check to see if the CountAvailable values change correctly.
// Mocking would let us make Verify calls on TryEnter() and Release(), but mocking
// is challenging because of how the BulkheadFactory internally keeps hold of the
// Bulkheads it's managing within SemaphoreBulkheadHolders. The tests here should
// be okay enough, though.
// Since the config changed, the factory should have a new bulkhead for the key.
var newBulkhead = bulkheadFactory.GetBulkhead(groupKey);
Assert.True(initialBulkhead != newBulkhead);
// The bulkhead we used should have its original value. We're making sure that
// we didn't TryEnter() and then skip the Release() because a different bulkhead
// was used.
Assert.Equal(initialMaxConcurrent, initialBulkhead.CountAvailable);
// For the sake of completeness, make sure the config change actually got
// applied (otherwise we might not be testing an actual config change up
// above).
Assert.Equal(newMaxConcurrent, newBulkhead.CountAvailable);
}
