public void MarkSuccess_ImmediatelyAfterTrippingButStartedBeforeTripped_DoesntImmediatelyFix()
{
// 1. Breaker is near tripping.
// 2. Operation A and B are Allowed and begin work.
// 3. Before Operation A completes
// a. Operation B has an error and updates metrics.
// b. Operation C calls IsAllowing(), which trips breaker.
// 4. Operation A completes successfully and calls MarkSuccess().
// 5. Since breaker is tripped but we haven't hit our wait duration yet,
// MarkSuccess() should result in the the breaker remaining tripped.
// Arrange
var manualClock = new ManualTestClock();
var mockMetrics = new Mock <ICommandMetrics>();
mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(2, 100)); // 2 ops, 100% failing.
var mockEvents = new Mock <IMetricEvents>();
var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>();
mockConfig.SetupSequence(m => m.GetMinimumOperations(It.IsAny <GroupKey>()))
.Returns(5) // First access should be > 1 so that the breaker doesn't trip.
.Returns(1); // Second access should be 1, so that we trip the breaker because we have the minimum met.
mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(1);
mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(30000);
mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(false);
mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(false);
// 5 ops, 1% failure required to break.
var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());
// Act / Assert
// #2. Operation A is allowed and begins.
Assert.True(breaker.IsAllowing()); // Haven't hit the 1-operation threshold yet, should be allowed.
// #3a. Operation B errors. This is easier to simulate by changing the breaker's trip
// conditions. The sequenced Mock (above) has the second MinimumOperations returning
// 1, which would now mean we have enough operations to trip (where we didn't before).
// #3b. Breaker exceeds metrics thresholds, Operation C tries to IsAllowing() and trips breaker.
Assert.False(breaker.IsAllowing());
// #4. Operation A completes successfully.
// Breaker's internal _lastTrippedTimestamp should be equal to zero (current clock time).
// Since we say the transaction took 100ms, that'll be before the breaker tripped, and should
// be ignored.
breaker.MarkSuccess(100);
// #5. Make sure we're still tripped and we didn't reset the metrics.
Assert.False(breaker.IsAllowing());
mockMetrics.Verify(m => m.Reset(), Times.Never);
}
public void MarkSuccess_ForLongRunningSingleTest_FixesBreaker()
{
// Since we use the elapsed time in MarkSuccess(), address the following situation:
// 1. Breaker trips
// 2. Window passes
// 3. Single test is allowed
// 4. Single test takes a long time (say, > 2 windows) to complete (even though it started after we tripped).
// Verify that:
// - No other requests are allowed while the single test is waiting.
// - The single test, if successful, fixes the breaker.
// This is somewhat unlikely since we probably wont have command timeouts
// that'd allow this. But worth verifying.
// Arrange
var manualClock = new ManualTestClock();
var mockMetrics = new Mock <ICommandMetrics>();
mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(2, 100)); // 2 ops, 100% failing.
var mockEvents = new Mock <IMetricEvents>();
const long trippedDurationMillis = 30000;
var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>();
mockConfig.Setup(m => m.GetMinimumOperations(It.IsAny <GroupKey>())).Returns(1);
mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(1);
mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(trippedDurationMillis);
mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(false);
mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(false);
// 1 ops, 1% failure required to break.
var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());
// Act / Assert
Assert.False(breaker.IsAllowing()); // Should immediately trip.
manualClock.AddMilliseconds(trippedDurationMillis + 10);
Assert.True(breaker.IsAllowing()); // Single test is allowed.
Assert.False(breaker.IsAllowing());
manualClock.AddMilliseconds(trippedDurationMillis * 2);
breaker.MarkSuccess(trippedDurationMillis * 2);
// Metrics will have been reset.
mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));
Assert.True(breaker.IsAllowing());
Assert.True(breaker.IsAllowing());
}
public void IsAllowing_WhenBothForcePropertiesSet_Rejects()
{
// Arrange
// Most property values don't matter, IsAllowing() should reject before it tries to use them.
var manualClock = new ManualTestClock();
var mockMetrics = new Mock <ICommandMetrics>();
mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));
var mockEvents = new Mock <IMetricEvents>();
var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>();
mockConfig.Setup(m => m.GetMinimumOperations(It.IsAny <GroupKey>())).Returns(1);
mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(1);
mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(30000);
mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(true); // The config we're testing here.
mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(true); // The config we're testing here.
var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());
// Act / Assert
Assert.False(breaker.IsAllowing());
}
public void Run()
{
var mockMetrics = CreateMockMetricsWithSnapshot(_metricsTotal, _metricsPercent);
var config = CreateMockBreakerConfig(_breakerTotal, _breakerPercent, 30000);
var breaker = new FailurePercentageCircuitBreaker(GroupKey.Named("Test"), mockMetrics.Object, new IgnoringMetricEvents(), config.Object, new DefaultMjolnirLogFactory());
Assert.NotEqual(_shouldTrip, breaker.IsAllowing());
}
public void Run()
{
var mockMetrics = CreateMockMetricsWithSnapshot(_metricsTotal, _metricsPercent);
var properties = CreateBreakerProperties(_breakerTotal, _breakerPercent, 30000);
var breaker = new FailurePercentageCircuitBreaker(GroupKey.Named("Test"), mockMetrics.Object, new IgnoringStats(), properties);
Assert.NotEqual(_shouldTrip, breaker.IsAllowing());
}
public void IsAllowing_WhenPropertiesForceFixedButBreakerWouldNormallyTrip_SilentlyTripsTheBreaker()
{
// Arrange
// Most property values don't matter, IsAllowing() should reject before it tries to use them.
var manualClock = new ManualTestClock();
var mockMetrics = new Mock <ICommandMetrics>();
mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(2, 50));
var mockEvents = new Mock <IMetricEvents>();
var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>();
mockConfig.Setup(m => m.GetMinimumOperations(It.IsAny <GroupKey>())).Returns(1);
mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(25);
mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(30000);
mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(false);
mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(true);
// We'll call IsAllowing() three times. The first two should force the breaker fixed,
// the third should un-force it. The breaker should have tripped by then, so the third
// call should show the breaker disallowing the call.
mockConfig.SetupSequence(m => m.GetForceFixed(It.IsAny <GroupKey>()))
.Returns(true)
.Returns(true)
.Returns(false);
var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());
// Act / Assert
Assert.True(breaker.IsAllowing()); // Will have tripped internally.
Assert.True(breaker.IsAllowing()); // Continues to allow even when tripped.
// Test that when the the forced fix property is no longer true, IsAllowing() then rejects.
// The Mock sequencing enables this.
Assert.False(breaker.IsAllowing());
}