public void GetBulkhead_WhenInitializingBulkheadAndMaxConcurrentConfigIsInvalid_AndThenConfigChangedToValidValue_CreatesBulkhead()
{
// Arrange
var key = AnyGroupKey;
const int invalidMaxConcurrent = -1;
const int validMaxConcurrent = 1;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key.Name,
new BulkheadConfiguration
{
MaxConcurrent = invalidMaxConcurrent
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new DefaultMjolnirLog <BulkheadFactory>());
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
var factory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
try
{
factory.GetBulkhead(key);
}
catch (ArgumentOutOfRangeException)
{
// Expected, config is invalid for the first attempt.
}
mockConfig.BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key.Name,
new BulkheadConfiguration
{
MaxConcurrent = validMaxConcurrent
}
}
};
mockConfig.NotifyAfterConfigUpdate();
// Act
var bulkhead = factory.GetBulkhead(key); // Should not throw.
// Assert
Assert.Equal(validMaxConcurrent, bulkhead.CountAvailable);
}
public void Construct_InitializesConfigGauge_GaugeFiresForOneBulkhead()
{
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
var expectedMaxConcurrent = AnyPositiveInt;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
mockMetricEvents.Setup(m => m.BulkheadGauge(groupKey.Name, "semaphore", expectedMaxConcurrent, It.IsAny <int>()));
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
groupKey.Name,
new BulkheadConfiguration
{
MaxConcurrent = expectedMaxConcurrent
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new DefaultMjolnirLog <BulkheadFactory>());
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
// Act
var factory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
// Add a bulkhead
factory.GetBulkhead(groupKey);
// The timer will fire after 1 second.
Thread.Sleep(TimeSpan.FromMilliseconds(1500));
// Assert
// Gauges should fire every second, so wait one second and then verify.
mockMetricEvents.Verify(m => m.BulkheadGauge(key, "semaphore", expectedMaxConcurrent, It.IsAny <int>()), Times.AtLeastOnce);
}
public void GetBulkhead_ReturnsSameBulkheadForKey()
{
// Bulkheads are long-lived objects and used for many requests. In the absence
// of any configuration changes, we should be using the same one through the
// lifetime of the app.
// Arrange
var key = AnyGroupKey;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key.Name,
new BulkheadConfiguration
{
MaxConcurrent = AnyPositiveInt
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new DefaultMjolnirLog <BulkheadFactory>());
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
var factory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
// Act
var bulkhead = factory.GetBulkhead(key);
// Assert
Assert.Equal(bulkhead, factory.GetBulkhead(key));
}
public void GetBulkhead_WhenInitializingBulkheadAndMaxConcurrentConfigIsInvalid_Throws()
{
// Arrange
var key = AnyGroupKey;
const int invalidMaxConcurrent = -1;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key.Name,
new BulkheadConfiguration
{
MaxConcurrent = invalidMaxConcurrent
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new DefaultMjolnirLog <BulkheadFactory>());
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
var factory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
// Act + Assert
var exception = Assert.Throws <ArgumentOutOfRangeException>(() => factory.GetBulkhead(key));
Assert.Equal("maxConcurrent", exception.ParamName);
Assert.Equal(invalidMaxConcurrent, exception.ActualValue);
}
public void Construct_InitializesConfigGauge_GaugeFiresForMultipleBulkheads()
{
// Arrange
var key1 = AnyString;
var groupKey1 = GroupKey.Named(key1);
var key2 = AnyString;
var groupKey2 = GroupKey.Named(key2);
var expectedMaxConcurrent1 = AnyPositiveInt;
var expectedMaxConcurrent2 = AnyPositiveInt;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
mockMetricEvents.Setup(m => m.BulkheadGauge(groupKey1.Name, "semaphore", expectedMaxConcurrent1, It.IsAny <int>()));
mockMetricEvents.Setup(m => m.BulkheadGauge(groupKey2.Name, "semaphore", expectedMaxConcurrent2, It.IsAny <int>()));
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
groupKey1.Name,
new BulkheadConfiguration
{
MaxConcurrent = expectedMaxConcurrent1
}
},
{
groupKey2.Name,
new BulkheadConfiguration
{
MaxConcurrent = expectedMaxConcurrent2
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new DefaultMjolnirLog <BulkheadFactory>());
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
// Act + Assert
var factory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
// Wait 2s - since we haven't yet created any bulkheads, we shouldn't have any events.
Thread.Sleep(TimeSpan.FromMilliseconds(1500));
mockMetricEvents.Verify(
m => m.BulkheadGauge(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <int>(), It.IsAny <int>()),
Times.Never);
// Add two bulkheads
factory.GetBulkhead(groupKey1);
factory.GetBulkhead(groupKey2);
Thread.Sleep(TimeSpan.FromMilliseconds(1500));
mockMetricEvents.Verify(m => m.BulkheadGauge(key1, "semaphore", expectedMaxConcurrent1, It.IsAny <int>()), Times.AtLeastOnce);
mockMetricEvents.Verify(m => m.BulkheadGauge(key2, "semaphore", expectedMaxConcurrent2, It.IsAny <int>()), Times.AtLeastOnce);
}
public void GetBulkhead_ReturnsNewBulkheadWhenConfigChanges()
{
// Config can be used to resize the bulkhead at runtime, which results in a new
// bulkhead being created.
// To ensure consistency, callers who retrieve a bulkhead and call TryEnter()
// on it should keep a local reference to the same bulkhead to later call
// Release() on (rather than re-retrieving the bulkhead from the context).
// That behavior is tested elsewhere (with the BulkheadInvoker tests).
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
const int initialExpectedCount = 5;
const int newExpectedCount = 6;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
groupKey.Name,
new BulkheadConfiguration
{
MaxConcurrent = initialExpectedCount
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new DefaultMjolnirLog <BulkheadFactory>());
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
var factory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
// Act
var firstBulkhead = factory.GetBulkhead(groupKey);
mockConfig.BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
groupKey.Name,
new BulkheadConfiguration
{
MaxConcurrent = newExpectedCount
}
}
};
mockConfig.NotifyAfterConfigUpdate();
// Give the change handler callback enough time to create and reassign the bulkhead.
Thread.Sleep(500);
var secondBulkhead = factory.GetBulkhead(groupKey);
// Assert
// Shouldn't change any existing referenced bulkheads...
Assert.Equal(initialExpectedCount, firstBulkhead.CountAvailable);
// ...but newly-retrieved bulkheads should get a new instance
// with the updated count.
Assert.Equal(newExpectedCount, secondBulkhead.CountAvailable);
// And they shouldn't be the same bulkhead (which should be obvious by this point).
Assert.False(firstBulkhead == secondBulkhead);
}
public void CallsTryEnterAndReleaseOnTheSameBulkheadDuringConfigChange()
{
// The assumption tested here is important. If the bulkhead max concurrent
// configuration value changes, the bulkhead holder will build a new semaphore and
// swap it out with the old one. However, any bulkheads that acquired a lock on
// the original semaphore need to release that semaphore instead of the new one.
// Otherwise, they'll release on the new one. If that happens, the new semaphore's
// counter won't be accurate respective to the number of commands concurrently
// executing with it.
// This test is complicated, but it's one of the most important unit tests in the
// project. If you need to change it, take care that it gets re-written properly.
// The test is performed by having the command itself change the config
// value, which will happen between the TryEnter and Release calls.
// Arrange
const int initialMaxConcurrent = 10;
const int newMaxConcurrent = 15;
var key = AnyString;
var groupKey = GroupKey.Named(key);
var mockConfig = new MjolnirConfiguration
{
UseCircuitBreakers = true,
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key,
new BulkheadConfiguration
{
MaxConcurrent = initialMaxConcurrent
}
}
}
};
var mockBreakerExceptionHandler = new Mock <IBreakerExceptionHandler>(MockBehavior.Strict);
mockBreakerExceptionHandler.Setup(m => m.IsExceptionIgnored(It.IsAny <Type>())).Returns(false);
var mockCircuitBreaker = new Mock <ICircuitBreaker>(MockBehavior.Strict);
mockCircuitBreaker.Setup(m => m.IsAllowing()).Returns(true);
mockCircuitBreaker.Setup(m => m.Name).Returns(AnyString);
mockCircuitBreaker.Setup(m => m.Metrics).Returns(new Mock <ICommandMetrics>().Object);
mockCircuitBreaker.Setup(m => m.MarkSuccess(It.IsAny <long>()));
var mockCircuitBreakerFactory = new Mock <ICircuitBreakerFactory>(MockBehavior.Strict);
mockCircuitBreakerFactory.Setup(m => m.GetCircuitBreaker(groupKey)).Returns(mockCircuitBreaker.Object);
var mockMetricEvents = new Mock <IMetricEvents>(); // Non-Strict: we aren't testing metric events here, let's keep the test simpler.
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new Mock <IMjolnirLog <BulkheadFactory> >().Object);
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new Mock <IMjolnirLog <SemaphoreBulkheadHolder> >().Object);
// Use a real BulkheadFactory, which will give us access to its BulkheadHolder.
var bulkheadFactory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
var holder = bulkheadFactory.GetBulkheadHolder(groupKey);
var initialBulkhead = bulkheadFactory.GetBulkhead(groupKey);
// Use a real BreakerInvoker instead of a mocked one so that we actually
// invoke the command that changes the config value.
var breakerInvoker = new BreakerInvoker(mockCircuitBreakerFactory.Object, mockMetricEvents.Object, mockBreakerExceptionHandler.Object);
var command = new ChangeBulkheadLimitSyncCommand(key, holder, newMaxConcurrent);
var invoker = new BulkheadInvoker(breakerInvoker, bulkheadFactory, mockMetricEvents.Object, mockConfig);
var unusedCancellationToken = CancellationToken.None;
// Make sure the BulkheadFactory has the expected Bulkhead initialized for the key.
Assert.Equal(initialMaxConcurrent, bulkheadFactory.GetBulkhead(groupKey).CountAvailable);
// Act
var result = invoker.ExecuteWithBulkhead(command, unusedCancellationToken);
// Assert
// The assertions here are a bit indirect and, if we were mocking, could be more
// deterministic. We check to see if the CountAvailable values change correctly.
// Mocking would let us make Verify calls on TryEnter() and Release(), but mocking
// is challenging because of how the BulkheadFactory internally keeps hold of the
// Bulkheads it's managing within SemaphoreBulkheadHolders. The tests here should
// be okay enough, though.
// Since the config changed, the factory should have a new bulkhead for the key.
var newBulkhead = bulkheadFactory.GetBulkhead(groupKey);
Assert.True(initialBulkhead != newBulkhead);
// The bulkhead we used should have its original value. We're making sure that
// we didn't TryEnter() and then skip the Release() because a different bulkhead
// was used.
Assert.Equal(initialMaxConcurrent, initialBulkhead.CountAvailable);
// For the sake of completeness, make sure the config change actually got
// applied (otherwise we might not be testing an actual config change up
// above).
Assert.Equal(newMaxConcurrent, newBulkhead.CountAvailable);
}
