public void Construct_WithIsolationKey_UsesIsolationKeyAsAllKeys()
{
var command = new KeyTestCommand("test", "foo");
Assert.Equal(GroupKey.Named("foo"), command.BreakerKey);
Assert.Equal(GroupKey.Named("foo"), command.PoolKey);
}
private static void AssertInvalidName(string name)
{
Assert.Throws <ArgumentException>(() =>
{
GroupKey.Named(name);
});
}
public async Task FiresMetricEventWhenRejected()
{
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
var mockMetricEvents = new Mock <IMetricEvents>();
var mockBreakerInvoker = new Mock <IBreakerInvoker>();
var mockBulkhead = new Mock <ISemaphoreBulkhead>();
mockBulkhead.Setup(m => m.TryEnter()).Returns(false);
mockBulkhead.SetupGet(m => m.Name).Returns(key);
var mockBulkheadFactory = new Mock <IBulkheadFactory>(MockBehavior.Strict);
mockBulkheadFactory.Setup(m => m.GetBulkhead(groupKey)).Returns(mockBulkhead.Object);
var mockConfig = new MjolnirConfiguration {
UseCircuitBreakers = true
};
// The breaker invoker behavior doesn't matter here, we shouldn't get to the point
// where we try to use it.
var invoker = new BulkheadInvoker(mockBreakerInvoker.Object, mockBulkheadFactory.Object, mockMetricEvents.Object, mockConfig);
var command = new ConfigurableKeyAsyncCommand(key);
// Act + Assert
await Assert.ThrowsAsync <BulkheadRejectedException>(() => invoker.ExecuteWithBulkheadAsync(command, CancellationToken.None));
mockMetricEvents.Verify(m => m.RejectedByBulkhead(key, command.Name));
}
public void FiresMetricEventWhenEnteringAndLeavingBulkheadAndCommandFails()
{
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
var mockMetricEvents = new Mock <IMetricEvents>();
var mockBreakerInvoker = new Mock <IBreakerInvoker>();
var mockBulkhead = new Mock <ISemaphoreBulkhead>();
mockBulkhead.Setup(m => m.TryEnter()).Returns(true);
mockBulkhead.SetupGet(m => m.Name).Returns(key);
var mockBulkheadFactory = new Mock <IBulkheadFactory>(MockBehavior.Strict);
mockBulkheadFactory.Setup(m => m.GetBulkhead(groupKey)).Returns(mockBulkhead.Object);
var mockConfig = new MjolnirConfiguration {
UseCircuitBreakers = false
};
// The breaker invoker behavior doesn't matter here, we shouldn't get to the point
// where we try to use it. Pass a "false" value for useCircuitBreakers to help
// ensure that.
var invoker = new BulkheadInvoker(mockBreakerInvoker.Object, mockBulkheadFactory.Object, mockMetricEvents.Object, mockConfig);
var command = new ConfigurableKeyThrowingCommand(key);
// Act + Assert
Assert.Throws <ExpectedTestException>(() => invoker.ExecuteWithBulkhead(command, CancellationToken.None));
mockMetricEvents.Verify(m => m.EnterBulkhead(key, command.Name));
mockMetricEvents.Verify(m => m.LeaveBulkhead(key, command.Name));
}
public void SetsExecutionTimeOnCommandWhenInvokedWithoutBreakerAndCommandFails()
{
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
var mockMetricEvents = new Mock <IMetricEvents>();
var mockBreakerInvoker = new Mock <IBreakerInvoker>();
var mockBulkhead = new Mock <ISemaphoreBulkhead>();
mockBulkhead.Setup(m => m.TryEnter()).Returns(true);
mockBulkhead.SetupGet(m => m.Name).Returns(key);
var mockBulkheadFactory = new Mock <IBulkheadFactory>(MockBehavior.Strict);
mockBulkheadFactory.Setup(m => m.GetBulkhead(groupKey)).Returns(mockBulkhead.Object);
var mockConfig = new MjolnirConfiguration {
UseCircuitBreakers = false
};
// Pass false for useCircuitBreakers to bypass the breaker; we're testing that here.
var invoker = new BulkheadInvoker(mockBreakerInvoker.Object, mockBulkheadFactory.Object, mockMetricEvents.Object, mockConfig);
var command = new ConfigurableKeyThrowingCommand(key);
// Act + Assert
Assert.Throws <ExpectedTestException>(() => invoker.ExecuteWithBulkhead(command, CancellationToken.None));
Assert.True(command.ExecutionTimeMillis > 0);
}
public void Construct_WhenMaxConcurrentConfigIsInvalid_DoesSomething()
{
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
const int invalidMaxConcurrent = -1;
var mockMetricEvents = new Mock <IMetricEvents>(); // Not Strict: we're not testing the events here.
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
groupKey.Name,
new BulkheadConfiguration
{
MaxConcurrent = invalidMaxConcurrent
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
// Act + Assert
var exception = Assert.Throws <ArgumentOutOfRangeException>(() => new SemaphoreBulkheadHolder(groupKey, mockMetricEvents.Object, mockConfig, mockLogFactory.Object));
Assert.Equal("maxConcurrent", exception.ParamName);
Assert.Equal(invalidMaxConcurrent, exception.ActualValue);
}
public void TryEnterAndReleaseALot()
{
var semaphore = new SemaphoreSlimIsolationSemaphore(GroupKey.Named("Test"), new TransientConfigurableValue <int>(5), new IgnoringStats());
Assert.True(semaphore.TryEnter());
Assert.True(semaphore.TryEnter());
Assert.True(semaphore.TryEnter());
Assert.True(semaphore.TryEnter());
Assert.True(semaphore.TryEnter());
Assert.False(semaphore.TryEnter());
Assert.False(semaphore.TryEnter());
Assert.False(semaphore.TryEnter());
semaphore.Release();
Assert.True(semaphore.TryEnter());
Assert.False(semaphore.TryEnter());
semaphore.Release();
semaphore.Release();
semaphore.Release();
semaphore.Release();
semaphore.Release();
Assert.True(semaphore.TryEnter());
Assert.True(semaphore.TryEnter());
Assert.True(semaphore.TryEnter());
semaphore.Release();
Assert.True(semaphore.TryEnter());
Assert.True(semaphore.TryEnter());
Assert.True(semaphore.TryEnter());
Assert.False(semaphore.TryEnter());
Assert.False(semaphore.TryEnter());
semaphore.Release();
semaphore.Release();
semaphore.Release();
semaphore.Release();
semaphore.Release();
}
public void Run()
{
var mockMetrics = CreateMockMetricsWithSnapshot(_metricsTotal, _metricsPercent);
var config = CreateMockBreakerConfig(_breakerTotal, _breakerPercent, 30000);
var breaker = new FailurePercentageCircuitBreaker(GroupKey.Named("Test"), mockMetrics.Object, new IgnoringMetricEvents(), config.Object, new DefaultMjolnirLogFactory());
Assert.NotEqual(_shouldTrip, breaker.IsAllowing());
}
public void Run()
{
var mockMetrics = CreateMockMetricsWithSnapshot(_metricsTotal, _metricsPercent);
var properties = CreateBreakerProperties(_breakerTotal, _breakerPercent, 30000);
var breaker = new FailurePercentageCircuitBreaker(GroupKey.Named("Test"), mockMetrics.Object, new IgnoringStats(), properties);
Assert.NotEqual(_shouldTrip, breaker.IsAllowing());
}
public void TryEnter_WhenSemaphoreIsAvailable_ReturnsTrueImmediately()
{
var semaphore = new SemaphoreSlimIsolationSemaphore(GroupKey.Named("Test"), new TransientConfigurableValue <int>(1), new IgnoringStats());
var stopwatch = Stopwatch.StartNew();
Assert.True(semaphore.TryEnter());
Assert.True(stopwatch.ElapsedMilliseconds < 10);
}
public void SetsBreakerKey()
{
var key = AnyString;
var expected = GroupKey.Named(key);
var command = new TestCommand(AnyString, key, AnyString, ValidTimeout);
Assert.Equal(expected, command.BreakerKey);
}
private static StandardCommandMetrics CreateMetrics(string key, IMock <IStats> mockStats, IClock clock = null,
long?windowMillis = null, long?snapshotTtlMillis = null)
{
return(new StandardCommandMetrics(
GroupKey.Named(key),
new TransientConfigurableValue <long>(windowMillis ?? 30000),
new TransientConfigurableValue <long>(snapshotTtlMillis ?? 10000),
(clock ?? new ManualTestClock()),
mockStats.Object));
}
public void Release_WhenSemaphoreNotAvailable_MakesItAvailable()
{
var semaphore = new SemaphoreSlimIsolationSemaphore(GroupKey.Named("Test"), new TransientConfigurableValue <int>(1), new IgnoringStats());
semaphore.TryEnter();
Assert.False(semaphore.TryEnter());
semaphore.Release();
Assert.True(semaphore.TryEnter());
}
public async Task Construct_CreatesGauges()
{
const long gaugeIntervalMillis = 50;
var mockStats = new Mock <IStats>();
var semaphore = new SemaphoreSlimIsolationSemaphore(GroupKey.Named("Test"), new TransientConfigurableValue <int>(10), mockStats.Object, new TransientConfigurableValue <long>(gaugeIntervalMillis));
await Task.Delay(TimeSpan.FromMilliseconds(gaugeIntervalMillis + 50));
mockStats.Verify(m => m.Gauge("mjolnir fallback-semaphore Test available", "Available", 10), Times.AtLeastOnce);
}
private StpIsolationThreadPool CreateAndStartPool(int threadCount, int queueLength)
{
var pool = new StpIsolationThreadPool(
GroupKey.Named("Test"),
new TransientConfigurableValue <int>(threadCount),
new TransientConfigurableValue <int>(queueLength),
new IgnoringStats());
pool.Start();
return(pool);
}
public void Start_Elapsed()
{
var mockStats = new Mock <IStats>();
var pool = new StpIsolationThreadPool(
GroupKey.Named("Test"),
new TransientConfigurableValue <int>(10),
new TransientConfigurableValue <int>(20),
mockStats.Object);
pool.Start();
mockStats.Verify(m => m.Elapsed("mjolnir pool Test Start", null, It.IsAny <TimeSpan>()), Times.Once);
}
public void MarkCommandFailure_BeforeFirstSnapshot_GetsIncludedInSnapshot()
{
var metrics = new StandardCommandMetrics(
GroupKey.Named("Test"),
new TransientConfigurableValue <long>(30000),
new TransientConfigurableValue <long>(1000),
new IgnoringStats());
metrics.MarkCommandFailure();
var snapshot = metrics.GetSnapshot();
Assert.Equal(1, snapshot.Total);
Assert.Equal(100, snapshot.ErrorPercentage);
}
public async Task Construct_CreatesGauges()
{
const long gaugeIntervalMillis = 50;
var mockStats = new Mock <IStats>();
var pool = new StpIsolationThreadPool(
GroupKey.Named("Test"),
new TransientConfigurableValue <int>(10),
new TransientConfigurableValue <int>(20),
mockStats.Object,
new TransientConfigurableValue <long>(gaugeIntervalMillis));
await Task.Delay(TimeSpan.FromMilliseconds(gaugeIntervalMillis + 50));
mockStats.Verify(m => m.Gauge("mjolnir pool Test activeThreads", null, It.IsAny <long>()), Times.AtLeastOnce);
mockStats.Verify(m => m.Gauge("mjolnir pool Test inUseThreads", null, It.IsAny <long>()), Times.AtLeastOnce);
mockStats.Verify(m => m.Gauge("mjolnir pool Test pendingCompletion", null, It.IsAny <long>()), Times.AtLeastOnce);
}
public void Construct_InitializesConfigGauge_GaugeFiresForOneBulkhead()
{
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
var expectedMaxConcurrent = AnyPositiveInt;
var mockMetricEvents = new Mock <IMetricEvents>(MockBehavior.Strict);
mockMetricEvents.Setup(m => m.BulkheadGauge(groupKey.Name, "semaphore", expectedMaxConcurrent, It.IsAny <int>()));
var mockConfig = new MjolnirConfiguration
{
BulkheadConfigurations = new Dictionary <string, BulkheadConfiguration>
{
{
groupKey.Name,
new BulkheadConfiguration
{
MaxConcurrent = expectedMaxConcurrent
}
}
}
};
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <BulkheadFactory>()).Returns(new DefaultMjolnirLog <BulkheadFactory>());
mockLogFactory.Setup(m => m.CreateLog <SemaphoreBulkheadHolder>()).Returns(new DefaultMjolnirLog <SemaphoreBulkheadHolder>());
// Act
var factory = new BulkheadFactory(mockMetricEvents.Object, mockConfig, mockLogFactory.Object);
// Add a bulkhead
factory.GetBulkhead(groupKey);
// The timer will fire after 1 second.
Thread.Sleep(TimeSpan.FromMilliseconds(1500));
// Assert
// Gauges should fire every second, so wait one second and then verify.
mockMetricEvents.Verify(m => m.BulkheadGauge(key, "semaphore", expectedMaxConcurrent, It.IsAny <int>()), Times.AtLeastOnce);
}
public void Construct_InitializesConfigGauge_GaugeFiresForOneBulkhead()
{
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
var expectedMinimumOperations = AnyPositiveInt;
var expectedWindowMillis = AnyPositiveInt;
var expectedThresholdPercent = AnyPositiveInt;
var expectedTrippedDurationMillis = AnyPositiveInt;
var expectedForceTripped = false;
var expectedForceFixed = false;
var mockMetricEvents = new Mock <IMetricEvents>();
var mockBreakerConfig = new Mock <IFailurePercentageCircuitBreakerConfig>(MockBehavior.Strict);
mockBreakerConfig.Setup(m => m.GetMinimumOperations(groupKey)).Returns(expectedMinimumOperations);
mockBreakerConfig.Setup(m => m.GetWindowMillis(groupKey)).Returns(expectedWindowMillis);
mockBreakerConfig.Setup(m => m.GetThresholdPercentage(groupKey)).Returns(expectedThresholdPercent);
mockBreakerConfig.Setup(m => m.GetTrippedDurationMillis(groupKey)).Returns(expectedTrippedDurationMillis);
mockBreakerConfig.Setup(m => m.GetForceTripped(groupKey)).Returns(expectedForceTripped);
mockBreakerConfig.Setup(m => m.GetForceFixed(groupKey)).Returns(expectedForceFixed);
var mockLogFactory = new Mock <IMjolnirLogFactory>(MockBehavior.Strict);
mockLogFactory.Setup(m => m.CreateLog <FailurePercentageCircuitBreaker>()).Returns(new DefaultMjolnirLog <FailurePercentageCircuitBreaker>());
mockLogFactory.Setup(m => m.CreateLog <ResettingNumbersBucket>()).Returns(new DefaultMjolnirLog <ResettingNumbersBucket>());
// Act + Assert
var factory = new CircuitBreakerFactory(mockMetricEvents.Object, mockBreakerConfig.Object, mockLogFactory.Object);
// Gauge won't fire immediately, it'll start one second after construction
mockMetricEvents.Verify(m => m.BreakerGauge(It.IsAny <string>(), It.IsAny <long>(), It.IsAny <long>(), It.IsAny <int>(), It.IsAny <int>(), It.IsAny <bool>(), It.IsAny <bool>(), It.IsAny <bool>(), It.IsAny <long>(), It.IsAny <long>()), Times.Never);
// Add two breakers
var firstBreaker = factory.GetCircuitBreaker(groupKey);
Thread.Sleep(TimeSpan.FromMilliseconds(1500));
mockMetricEvents.Verify(m => m.BreakerGauge(key, expectedMinimumOperations, expectedWindowMillis, expectedThresholdPercent, expectedTrippedDurationMillis, expectedForceTripped, expectedForceFixed, false, 0, 0));
}
private MetricsSnapshot SnapshotFor(int success, int failure)
{
var metrics = new StandardCommandMetrics(
GroupKey.Named("Test"),
new TransientConfigurableValue <long>(10000),
new TransientConfigurableValue <long>(0),
new IgnoringStats()); // Don't cache snapshots.
for (var i = 0; i < success; i++)
{
metrics.MarkCommandSuccess();
}
for (var i = 0; i < failure; i++)
{
metrics.MarkCommandFailure();
}
return(metrics.GetSnapshot());
}
public async Task DoesntSetExecutionTimeOnCommandWhenInvokedWithBreakerAndCommandSucceeds()
{
// If we execute on the breaker, the breaker should set the execution time instead
// of the bulkhead invoker.
// Arrange
var key = AnyString;
var groupKey = GroupKey.Named(key);
var mockMetricEvents = new Mock <IMetricEvents>();
var mockBreakerInvoker = new Mock <IBreakerInvoker>();
var mockBulkhead = new Mock <ISemaphoreBulkhead>();
mockBulkhead.Setup(m => m.TryEnter()).Returns(true);
mockBulkhead.SetupGet(m => m.Name).Returns(key);
var mockBulkheadFactory = new Mock <IBulkheadFactory>(MockBehavior.Strict);
mockBulkheadFactory.Setup(m => m.GetBulkhead(groupKey)).Returns(mockBulkhead.Object);
var command = new ConfigurableKeyThrowingAsyncCommand(key);
mockBreakerInvoker.Setup(m => m.ExecuteWithBreakerAsync(command, It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(true));
var mockConfig = new MjolnirConfiguration {
UseCircuitBreakers = true
};
// Pass true for useCircuitBreakers, we need to test that behavior here.
var invoker = new BulkheadInvoker(mockBreakerInvoker.Object, mockBulkheadFactory.Object, mockMetricEvents.Object, mockConfig);
// Act
await invoker.ExecuteWithBulkheadAsync(command, CancellationToken.None);
// Assert
Assert.Equal(0, command.ExecutionTimeMillis);
}
public void GetSnapshot_WithinCachePeriod_ReturnsPreviousSnapshot()
{
var clock = new ManualTestClock();
// Within the metrics, the last snapshot timestamp will probably be zero.
// Let's start our clock with something far away from zero.
clock.AddMilliseconds(new SystemClock().GetMillisecondTimestamp());
var metrics = new StandardCommandMetrics(
GroupKey.Named("Test"),
new TransientConfigurableValue <long>(10000),
new TransientConfigurableValue <long>(1000),
clock,
new IgnoringStats());
metrics.MarkCommandSuccess();
metrics.GetSnapshot(); // Take the first snapshot to cache it.
metrics.MarkCommandSuccess();
clock.AddMilliseconds(500); // Still within the snapshot TTL (1000).
Assert.Equal(1, metrics.GetSnapshot().Total);
clock.AddMilliseconds(1000); // Push time past the TTL.
Assert.Equal(2, metrics.GetSnapshot().Total);
}
internal BaseCommand(string group, string name, string breakerKey, string bulkheadKey, TimeSpan?defaultTimeout = null)
{
if (string.IsNullOrWhiteSpace(group))
{
throw new ArgumentNullException(nameof(group));
}
if (string.IsNullOrWhiteSpace(breakerKey))
{
throw new ArgumentNullException(nameof(breakerKey));
}
if (string.IsNullOrWhiteSpace(bulkheadKey))
{
throw new ArgumentNullException(nameof(bulkheadKey));
}
if (breakerKey == "default")
{
throw new ArgumentException($"Cannot use 'default' as {nameof(breakerKey)}, it is a reserved name");
}
if (bulkheadKey == "default")
{
throw new ArgumentException($"Cannot use 'default' as {nameof(bulkheadKey)}, it is a reserved name");
}
if (defaultTimeout != null && defaultTimeout.Value.TotalMilliseconds <= 0)
{
throw new ArgumentException($"Positive default timeout is required if passed (received invalid timeout of {defaultTimeout.Value.TotalMilliseconds}ms)", nameof(defaultTimeout));
}
_group = GroupKey.Named(group);
_name = string.IsNullOrWhiteSpace(name) ? GenerateAndCacheName(Group) : CacheProvidedName(Group, name);
_breakerKey = GroupKey.Named(breakerKey);
_bulkheadKey = GroupKey.Named(bulkheadKey);
_constructorTimeout = defaultTimeout ?? DefaultTimeout;
}
public FailurePercentageCircuitBreaker Create()
{
var config = FailurePercentageCircuitBreakerTests.CreateMockBreakerConfig(_minimumOperations, _failurePercent, _waitMillis);
return(new FailurePercentageCircuitBreaker(GroupKey.Named(_key), _clock, _mockMetrics.Object, _metricEvents, config.Object, new DefaultMjolnirLogFactory()));
}
private static void AssertValidName(string name)
{
GroupKey.Named(name);
Assert.True(true); // Didn't throw exception.
}
public void ToString_ReturnsName()
{
Assert.Equal("Foo", GroupKey.Named("Foo").ToString());
}
public void GetHashCode_ReturnsNameHashCode()
{
Assert.Equal("Foo".GetHashCode(), GroupKey.Named("Foo").GetHashCode());
}
public void Equals_WhenNamesDiffer_ReturnsFalse()
{
Assert.NotEqual(GroupKey.Named("Foo"), GroupKey.Named("Bar"));
}
public void Equals_WhenNamesEqual_ReturnsTrue()
{
Assert.Equal(GroupKey.Named("Foo"), GroupKey.Named("Foo"));
}
