public void FiresMetricEventWhenEnteringAndLeavingBulkheadAndCommandFails()
{
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
var mockMetricEvents = new Mock <IMetricEvents>();
var mockBreakerInvoker = new Mock <IBreakerInvoker>();
var mockBulkhead = new Mock <ISemaphoreBulkhead>();
mockBulkhead.Setup(m => m.TryEnter()).Returns(true);
mockBulkhead.SetupGet(m => m.Name).Returns(key);
var mockBulkheadFactory = new Mock <IBulkheadFactory>(MockBehavior.Strict);
mockBulkheadFactory.Setup(m => m.GetBulkhead(groupKey)).Returns(mockBulkhead.Object);
var mockConfig = new MjolnirConfiguration {
UseCircuitBreakers = false
};
// The breaker invoker behavior doesn't matter here, we shouldn't get to the point
// where we try to use it. Pass a "false" value for useCircuitBreakers to help
// ensure that.
var invoker = new BulkheadInvoker(mockBreakerInvoker.Object, mockBulkheadFactory.Object, mockMetricEvents.Object, mockConfig);
var command = new ConfigurableKeyThrowingCommand(key);
// Act + Assert
Assert.Throws <ExpectedTestException>(() => invoker.ExecuteWithBulkhead(command, CancellationToken.None));
mockMetricEvents.Verify(m => m.EnterBulkhead(key, command.Name));
mockMetricEvents.Verify(m => m.LeaveBulkhead(key, command.Name));
}
public void GetForceFixed_UsesDefaultValueIfNoSpecificValueOrDefaultValueConfigured_DefaultIsFalse()
{
// Arrange
const bool expectedDefaultForceFixed = false;
var groupKey = AnyGroupKey;
var mockConfig = new MjolnirConfiguration
{
DefaultBreakerConfiguration = new BreakerConfiguration
{
ForceFixed = expectedDefaultForceFixed
}
};
var config = new FailurePercentageCircuitBreakerConfig(mockConfig);
// Act
var value = config.GetForceFixed(groupKey);
// Assert
Assert.Equal(expectedDefaultForceFixed, value);
}
// Default Timeout: The system default timeout (2 seconds). Used if nothing else is set.
// Constructor Timeout: Value defined in the Command constructor.
// Configured Timeout: Value provided by config.
// Invocation Timeout: Value passed into the Invoke() / InvokeAsync() call.
internal TimeSpan DetermineTimeout(MjolnirConfiguration config, long?invocationTimeoutMillis = null)
{
// Thoughts on invocation timeout vs. configured timeout:
//
// Generally, I'd recommend using one or the other, but not both. Configurable timeouts
// are useful for changing the value at runtime, but are (at least in our environment)
// shared across many services, so they don't permit fine-grained control over
// individual calls. Invocation timeouts allow that fine-grained control, but require a
// code deploy to change them.
//
// Basically, consider the balance between runtime control and fine-grained per-call
// control when setting timeouts.
// Prefer the invocation timeout first. It's more specific than the Constructor
// Timeout (which is defined by the command author and is treated as a "catch-all"),
// It's also more local than the Configured Timeout, which is a way to tune
// the Constructor Timeout more specifically (i.e. still "catch-all" behavior).
if (invocationTimeoutMillis.HasValue && invocationTimeoutMillis.Value >= 0)
{
return(TimeSpan.FromMilliseconds(invocationTimeoutMillis.Value));
}
var configured = config.GetCommandConfiguration(_name).Timeout;
// We don't want to include 0 here. Since this comes from a potentially non-nullable
// ConfigurableValue, it's possible (and probably likely) that an unconfigured
// timeout will return a default(long), which will be 0.
if (configured > 0)
{
return(TimeSpan.FromMilliseconds(configured));
}
return(_constructorTimeout);
}
public void GetMaxConcurrent_UsesSpecificValueIfConfigured()
{
// Arrange
var groupKey = AnyGroupKey;
var expectedConfigValue = AnyPositiveInt;
var config = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
groupKey.Name,
new BulkheadConfiguration
{
MaxConcurrent = expectedConfigValue
}
}
}
};
// Act
var value = config.GetBulkheadConfiguration(groupKey.Name).MaxConcurrent;
// Assert
Assert.Equal(expectedConfigValue, value);
}
public void GetMinimumOperations_UsesDefaultValueIfNoSpecificValueOrDefaultValueConfigured_DefaultIs10()
{
// Arrange
const long expectedDefaultMinimumOperations = 10;
var groupKey = AnyGroupKey;
const int expectedConfigValue = 10;
var mockConfig = new MjolnirConfiguration
{
DefaultBreakerConfiguration = new BreakerConfiguration
{
MinimumOperations = expectedConfigValue
}
};
var config = new FailurePercentageCircuitBreakerConfig(mockConfig);
// Act
var value = config.GetMinimumOperations(groupKey);
// Assert
Assert.Equal(expectedDefaultMinimumOperations, value);
}
public BulkheadInvoker(IBreakerInvoker breakerInvoker, IBulkheadFactory bulkheadFactory, IMetricEvents metricEvents, MjolnirConfiguration config)
{
_breakerInvoker = breakerInvoker ?? throw new ArgumentNullException(nameof(breakerInvoker));
_bulkheadFactory = bulkheadFactory ?? throw new ArgumentNullException(nameof(bulkheadFactory));
_metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
_config = config ?? throw new ArgumentNullException(nameof(config));
}
public CommandInvoker(
MjolnirConfiguration config = null,
IMjolnirLogFactory logFactory = null,
IMetricEvents metricEvents = null,
IBreakerExceptionHandler breakerExceptionHandler = null)
: this(config, logFactory, metricEvents, breakerExceptionHandler, null)
{ }
public SemaphoreBulkheadHolder(GroupKey key, IMetricEvents metricEvents, MjolnirConfiguration config, IMjolnirLogFactory logFactory)
{
_key = key;
_metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
_config = config ?? throw new ArgumentNullException(nameof(config));
if (logFactory == null)
{
throw new ArgumentNullException(nameof(logFactory));
}
_log = logFactory.CreateLog <SemaphoreBulkheadHolder>();
if (_log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(SemaphoreBulkheadHolder)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
// The order of things here is very intentional.
// We get the MaxConcurrent value first and then initialize the semaphore bulkhead.
// The change handler is registered after that. The order ought to help avoid a
// situation where we might fire a config change handler before we add the
// semaphore to the dictionary, potentially trying to add two entries with
// different values in rapid succession.
var value = _config.GetBulkheadConfiguration(key.Name).MaxConcurrent;
_bulkhead = new SemaphoreBulkhead(_key, value);
// On change, we'll replace the bulkhead. The assumption here is that a caller
// using the bulkhead will have kept a local reference to the bulkhead that they
// acquired a lock on, and will release the lock on that bulkhead and not one that
// has been replaced after a config change.
_config.OnConfigurationChanged(c => c.GetBulkheadConfiguration(key.Name).MaxConcurrent, UpdateMaxConcurrent);
}
public async Task FiresMetricEventWhenRejected()
{
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
var mockMetricEvents = new Mock <IMetricEvents>();
var mockBreakerInvoker = new Mock <IBreakerInvoker>();
var mockBulkhead = new Mock <ISemaphoreBulkhead>();
mockBulkhead.Setup(m => m.TryEnter()).Returns(false);
mockBulkhead.SetupGet(m => m.Name).Returns(key);
var mockBulkheadFactory = new Mock <IBulkheadFactory>(MockBehavior.Strict);
mockBulkheadFactory.Setup(m => m.GetBulkhead(groupKey)).Returns(mockBulkhead.Object);
var mockConfig = new MjolnirConfiguration {
UseCircuitBreakers = true
};
// The breaker invoker behavior doesn't matter here, we shouldn't get to the point
// where we try to use it.
var invoker = new BulkheadInvoker(mockBreakerInvoker.Object, mockBulkheadFactory.Object, mockMetricEvents.Object, mockConfig);
var command = new ConfigurableKeyAsyncCommand(key);
// Act + Assert
await Assert.ThrowsAsync <BulkheadRejectedException>(() => invoker.ExecuteWithBulkheadAsync(command, CancellationToken.None));
mockMetricEvents.Verify(m => m.RejectedByBulkhead(key, command.Name));
}
public void SetsExecutionTimeOnCommandWhenInvokedWithoutBreakerAndCommandFails()
{
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
var mockMetricEvents = new Mock <IMetricEvents>();
var mockBreakerInvoker = new Mock <IBreakerInvoker>();
var mockBulkhead = new Mock <ISemaphoreBulkhead>();
mockBulkhead.Setup(m => m.TryEnter()).Returns(true);
mockBulkhead.SetupGet(m => m.Name).Returns(key);
var mockBulkheadFactory = new Mock <IBulkheadFactory>(MockBehavior.Strict);
mockBulkheadFactory.Setup(m => m.GetBulkhead(groupKey)).Returns(mockBulkhead.Object);
var mockConfig = new MjolnirConfiguration {
UseCircuitBreakers = false
};
// Pass false for useCircuitBreakers to bypass the breaker; we're testing that here.
var invoker = new BulkheadInvoker(mockBreakerInvoker.Object, mockBulkheadFactory.Object, mockMetricEvents.Object, mockConfig);
var command = new ConfigurableKeyThrowingCommand(key);
// Act + Assert
Assert.Throws <ExpectedTestException>(() => invoker.ExecuteWithBulkhead(command, CancellationToken.None));
Assert.True(command.ExecutionTimeMillis > 0);
}
public void Construct_WhenMaxConcurrentConfigIsInvalid_DoesSomething()
{
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
const int invalidMaxConcurrent = -1;
var mockMetricEvents = new Mock <IMetricEvents>(); // Not Strict: we're not testing the events here.
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
groupKey.Name,
new BulkheadConfiguration
{
MaxConcurrent = invalidMaxConcurrent
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
// Act + Assert
var exception = Assert.Throws <ArgumentOutOfRangeException>(() => new SemaphoreBulkheadHolder(groupKey, mockMetricEvents.Object, mockConfig, mockLogFactory.Object));
Assert.Equal("maxConcurrent", exception.ParamName);
Assert.Equal(invalidMaxConcurrent, exception.ActualValue);
}
public void GetForceFixed_UsesSpecificValueIfConfigured()
{
// Arrange
var groupKey = AnyGroupKey;
var expectedConfigValue = AnyBool;
var mockConfig = new MjolnirConfiguration
{
BreakerConfigurations = new Dictionary <string, BreakerConfiguration>
{
{
groupKey.Name,
new BreakerConfiguration
{
ForceFixed = expectedConfigValue
}
}
}
};
var config = new FailurePercentageCircuitBreakerConfig(mockConfig);
// Act
var value = config.GetForceFixed(groupKey);
// Assert
Assert.Equal(expectedConfigValue, value);
}
public void GetThresholdPercentage_UsesDefaultValueIfNoSpecificValueOrDefaultValueConfigured_DefaultIs50()
{
// Arrange
const int expectedDefaultThresholdPercentage = 50;
var groupKey = AnyGroupKey;
var mockConfig = new MjolnirConfiguration
{
DefaultBreakerConfiguration = new BreakerConfiguration
{
ThresholdPercentage = expectedDefaultThresholdPercentage
}
};
var config = new FailurePercentageCircuitBreakerConfig(mockConfig);
// Act
var value = config.GetThresholdPercentage(groupKey);
// Assert
Assert.Equal(expectedDefaultThresholdPercentage, value);
}
public void GetForceTripped_UsesDefaultValueIfNoSpecificValueConfigured()
{
// Arrange
var groupKey = AnyGroupKey;
var expectedConfigValue = AnyBool;
var mockConfig = new MjolnirConfiguration
{
DefaultBreakerConfiguration = new BreakerConfiguration
{
ForceTripped = expectedConfigValue
}
};
var config = new FailurePercentageCircuitBreakerConfig(mockConfig);
// Act
var value = config.GetForceTripped(groupKey);
// Assert
Assert.Equal(expectedConfigValue, value);
}
public void GetSnapshotTtlMillis_UsesDefaultValueIfNoSpecificValueOrDefaultValueConfigured_DefaultIs1000()
{
// Arrange
const long expectedDefaultSnapshotTtlMillis = 1000;
var groupKey = AnyGroupKey;
var mockConfig = new MjolnirConfiguration
{
DefaultBreakerConfiguration = new BreakerConfiguration
{
SnapshotTtlMillis = expectedDefaultSnapshotTtlMillis
}
};
var config = new FailurePercentageCircuitBreakerConfig(mockConfig);
// Act
var value = config.GetSnapshotTtlMillis(groupKey);
// Assert
Assert.Equal(expectedDefaultSnapshotTtlMillis, value);
}
private void TestTimeouts(TimeSpan expected, long?invocationMs, long configuredMs, long?constructorMs)
{
// The 'configured' value can't be nullable because the injected config
// implementation may pass along a default(long) (i.e. 0) if the config value
// isn't set, and we need to be prepared for that and not treat it as an actual
// timeout.
var constructorTs = (constructorMs == null
? (TimeSpan?)null
: TimeSpan.FromMilliseconds(constructorMs.Value));
var command = new TestCommand(AnyString, AnyString, AnyString, constructorTs);
var mockConfig = new MjolnirConfiguration
{
IsEnabled = true,
IgnoreTimeouts = false,
CommandConfigurations = new Dictionary <string, CommandConfiguration>
{
{
command.Name,
new CommandConfiguration
{
Timeout = configuredMs
}
}
}
};
var determined = command.DetermineTimeout(mockConfig, invocationMs);
Assert.Equal(expected, determined);
}
public void GetMinimumOperations_UsesDefaultValueIfNoSpecificValueConfigured()
{
// Arrange
var groupKey = AnyGroupKey;
var expectedConfigValue = AnyPositiveInt;
var mockConfig = new MjolnirConfiguration
{
DefaultBreakerConfiguration = new BreakerConfiguration
{
MinimumOperations = expectedConfigValue
}
};
var config = new FailurePercentageCircuitBreakerConfig(mockConfig);
// Act
var value = config.GetMinimumOperations(groupKey);
// Assert
Assert.Equal(expectedConfigValue, value);
}
public void GetSnapshotTtlMillis_UsesSpecificValueIfConfigured()
{
// Arrange
var groupKey = AnyGroupKey;
var expectedConfigValue = AnyPositiveInt;
var mockConfig = new MjolnirConfiguration
{
BreakerConfigurations = new Dictionary <string, BreakerConfiguration>
{
{
groupKey.Name,
new BreakerConfiguration
{
SnapshotTtlMillis = expectedConfigValue
}
}
}
};
var config = new FailurePercentageCircuitBreakerConfig(mockConfig);
// Act
var value = config.GetSnapshotTtlMillis(groupKey);
// Assert
Assert.Equal(expectedConfigValue, value);
}
public void GetBulkhead_WhenInitializingBulkheadAndMaxConcurrentConfigIsInvalid_AndThenConfigChangedToValidValue_CreatesBulkhead()
{
// Arrange
var key = AnyGroupKey;
const int invalidMaxConcurrent = -1;
const int validMaxConcurrent = 1;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key.Name,
new BulkheadConfiguration
{
MaxConcurrent = invalidMaxConcurrent
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new DefaultMjolnirLog <BulkheadFactory>());
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
var factory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
try
{
factory.GetBulkhead(key);
}
catch (ArgumentOutOfRangeException)
{
// Expected, config is invalid for the first attempt.
}
mockConfig.BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key.Name,
new BulkheadConfiguration
{
MaxConcurrent = validMaxConcurrent
}
}
};
mockConfig.NotifyAfterConfigUpdate();
// Act
var bulkhead = factory.GetBulkhead(key); // Should not throw.
// Assert
Assert.Equal(validMaxConcurrent, bulkhead.CountAvailable);
}
public void UpdateMaxConcurrent_ReplacesBulkhead()
{
// Arrange
var key = AnyGroupKey;
const int initialExpectedCount = 5;
const int newExpectedCount = 6;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
mockMetricEvents.Setup(m => m.BulkheadGauge(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <int>(), It.IsAny <int>()));
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key.Name,
new BulkheadConfiguration
{
MaxConcurrent = initialExpectedCount
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
var holder = new SemaphoreBulkheadHolder(key, mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
// Act
var firstBulkhead = holder.Bulkhead;
holder.UpdateMaxConcurrent(newExpectedCount);
var secondBulkhead = holder.Bulkhead;
// Assert
// Shouldn't change any existing referenced bulkheads...
Assert.Equal(initialExpectedCount, firstBulkhead.CountAvailable);
// ...but newly-retrieved bulkheads should get a new instance
// with the updated count.
Assert.Equal(newExpectedCount, secondBulkhead.CountAvailable);
// And they shouldn't be the same bulkhead (which should be obvious by this point).
Assert.False(firstBulkhead == secondBulkhead);
}
public void Construct_ThrowsIfNullLogFactory()
{
// Arrange
var key = AnyGroupKey;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
var mockConfig = new MjolnirConfiguration();
// Act + Assert
var exception = Assert.Throws <ArgumentNullException>(() => new SemaphoreBulkheadHolder(key, mockMetricEvents.Object, mockConfig, null));
Assert.Equal("logFactory", exception.ParamName);
}
public void UpdateMaxConcurrent_IgnoresInvalidValues()
{
// Arrange
var key = AnyGroupKey;
const int invalidMaxConcurrent = -1;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
mockMetricEvents.Setup(m => m.BulkheadGauge(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <int>(), It.IsAny <int>()));
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key.Name,
new BulkheadConfiguration
{
MaxConcurrent = AnyPositiveInt
}
}
}
};
var mockLog = new Mock <IMjolnirLog <SemaphoreBulkheadHolder> >(MockBehavior.Strict);
mockLog.Setup(m => m.Error(It.IsAny <string>()));
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(mockLog.Object);
var holder = new SemaphoreBulkheadHolder(key, mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
// Act
var initialBulkhead = holder.Bulkhead;
holder.UpdateMaxConcurrent(invalidMaxConcurrent);
// Assert
// Bulkhead should be unchanged.
Assert.True(initialBulkhead == holder.Bulkhead);
mockLog.Verify(m => m.Error($"Semaphore bulkhead config for key {key.Name} changed to an invalid limit of {invalidMaxConcurrent}, the bulkhead will not be changed"), Times.Once);
}
public void Construct_InitializesConfigGauge_GaugeFiresForOneBulkhead()
{
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
var expectedMaxConcurrent = AnyPositiveInt;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
mockMetricEvents.Setup(m => m.BulkheadGauge(groupKey.Name, "semaphore", expectedMaxConcurrent, It.IsAny <int>()));
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
groupKey.Name,
new BulkheadConfiguration
{
MaxConcurrent = expectedMaxConcurrent
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new DefaultMjolnirLog <BulkheadFactory>());
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
// Act
var factory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
// Add a bulkhead
factory.GetBulkhead(groupKey);
// The timer will fire after 1 second.
Thread.Sleep(TimeSpan.FromMilliseconds(1500));
// Assert
// Gauges should fire every second, so wait one second and then verify.
mockMetricEvents.Verify(m => m.BulkheadGauge(key, "semaphore", expectedMaxConcurrent, It.IsAny <int>()), Times.AtLeastOnce);
}
public async Task DoesntSetExecutionTimeOnCommandWhenInvokedWithBreakerAndCommandSucceeds()
{
// If we execute on the breaker, the breaker should set the execution time instead
// of the bulkhead invoker.
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
var mockMetricEvents = new Mock <IMetricEvents>();
var mockBreakerInvoker = new Mock <IBreakerInvoker>();
var mockBulkhead = new Mock <ISemaphoreBulkhead>();
mockBulkhead.Setup(m => m.TryEnter()).Returns(true);
mockBulkhead.SetupGet(m => m.Name).Returns(key);
var mockBulkheadFactory = new Mock <IBulkheadFactory>(MockBehavior.Strict);
mockBulkheadFactory.Setup(m => m.GetBulkhead(groupKey)).Returns(mockBulkhead.Object);
var command = new ConfigurableKeyThrowingAsyncCommand(key);
mockBreakerInvoker.Setup(m => m.ExecuteWithBreakerAsync(command, It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(true));
var mockConfig = new MjolnirConfiguration {
UseCircuitBreakers = true
};
// Pass true for useCircuitBreakers, we need to test that behavior here.
var invoker = new BulkheadInvoker(mockBreakerInvoker.Object, mockBulkheadFactory.Object, mockMetricEvents.Object, mockConfig);
// Act
await invoker.ExecuteWithBulkheadAsync(command, CancellationToken.None);
// Assert
Assert.Equal(0, command.ExecutionTimeMillis);
}
public void GetBulkhead_ReturnsSameBulkheadForKey()
{
// Bulkheads are long-lived objects and used for many requests. In the absence
// of any configuration changes, we should be using the same one through the
// lifetime of the app.
// Arrange
var key = AnyGroupKey;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key.Name,
new BulkheadConfiguration
{
MaxConcurrent = AnyPositiveInt
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new DefaultMjolnirLog <BulkheadFactory>());
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
var factory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
// Act
var bulkhead = factory.GetBulkhead(key);
// Assert
Assert.Equal(bulkhead, factory.GetBulkhead(key));
}
public MjolnirConfiguration GetConfig()
{
if (_currentConfig != null)
{
return(_currentConfig);
}
const string rootKey = "MjolnirConfiguration";
var section = _root.GetSection(rootKey);
if (section == null || section.Value == null && !section.GetChildren().Any())
{
_currentConfig = new MjolnirConfiguration();
return(default(MjolnirConfiguration));
}
var config = new MjolnirConfiguration();
section.Bind(config);
_currentConfig = config;
return(config);
}
public void Construct_SetsInitialBulkhead()
{
// Arrange
var key = AnyGroupKey;
var expectedMaxConcurrent = AnyPositiveInt;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
mockMetricEvents.Setup(m => m.BulkheadGauge(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <int>(), It.IsAny <int>()));
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key.Name,
new BulkheadConfiguration
{
MaxConcurrent = expectedMaxConcurrent
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
// Act
var holder = new SemaphoreBulkheadHolder(key, mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
// Assert
Assert.Equal(key.Name, holder.Bulkhead.Name);
Assert.Equal(expectedMaxConcurrent, holder.Bulkhead.CountAvailable);
}
public void GetMaxConcurrent_UsesDefaultValueIfNoSpecificValueConfigured()
{
// Arrange
var groupKey = AnyGroupKey;
var expectedConfigValue = AnyPositiveInt;
var config = new MjolnirConfiguration
{
DefaultBulkheadConfiguration = new BulkheadConfiguration
{
MaxConcurrent = expectedConfigValue
}
};
// Act
var value = config.GetBulkheadConfiguration(groupKey.Name).MaxConcurrent;
// Assert
Assert.Equal(expectedConfigValue, value);
}
public void GetBulkhead_WhenInitializingBulkheadAndMaxConcurrentConfigIsInvalid_Throws()
{
// Arrange
var key = AnyGroupKey;
const int invalidMaxConcurrent = -1;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
key.Name,
new BulkheadConfiguration
{
MaxConcurrent = invalidMaxConcurrent
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new DefaultMjolnirLog <BulkheadFactory>());
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
var factory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
// Act + Assert
var exception = Assert.Throws <ArgumentOutOfRangeException>(() => factory.GetBulkhead(key));
Assert.Equal("maxConcurrent", exception.ParamName);
Assert.Equal(invalidMaxConcurrent, exception.ActualValue);
}
// ReSharper restore PrivateFieldCanBeConvertedToLocalVariable
public BulkheadFactory(IMetricEvents metricEvents, MjolnirConfiguration config, IMjolnirLogFactory logFactory)
{
// No null checks on parameters; we don't use them, we're just passing them through to
// the objects we're creating.
_metricEvents = metricEvents;
_config = config;
_logFactory = logFactory ?? throw new ArgumentNullException(nameof(logFactory));
var log = logFactory.CreateLog <BulkheadFactory>();
if (log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(BulkheadFactory)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_timer = new GaugeTimer(state =>
{
try
{
var keys = _bulkheads.Keys;
foreach (var key in keys)
{
if (_bulkheads.TryGetValue(key, out Lazy <SemaphoreBulkheadHolder> holder) && holder.IsValueCreated)
{
var bulkhead = holder.Value.Bulkhead;
_metricEvents.BulkheadGauge(bulkhead.Name, "semaphore", _config.GetBulkheadConfiguration(key.Name).MaxConcurrent, bulkhead.CountAvailable);
}
}
}
catch (Exception e)
{
log.Error($"Error sending {nameof(IMetricEvents.BulkheadGauge)} metric event", e);
}
});
}
