public void MarkSuccess_ImmediatelyAfterTrippingButStartedBeforeTripped_DoesntImmediatelyFix()
{
// 1. Breaker is near tripping.
// 2. Operation A and B are Allowed and begin work.
// 3. Before Operation A completes
// a. Operation B has an error and updates metrics.
// b. Operation C calls IsAllowing(), which trips breaker.
// 4. Operation A completes successfully and calls MarkSuccess().
// 5. Since breaker is tripped but we haven't hit our wait duration yet,
// MarkSuccess() should result in the the breaker remaining tripped.
// Arrange
var manualClock = new ManualTestClock();
var mockMetrics = new Mock <ICommandMetrics>();
mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(2, 100)); // 2 ops, 100% failing.
var mockEvents = new Mock <IMetricEvents>();
var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>();
mockConfig.SetupSequence(m => m.GetMinimumOperations(It.IsAny <GroupKey>()))
.Returns(5) // First access should be > 1 so that the breaker doesn't trip.
.Returns(1); // Second access should be 1, so that we trip the breaker because we have the minimum met.
mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(1);
mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(30000);
mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(false);
mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(false);
// 5 ops, 1% failure required to break.
var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());
// Act / Assert
// #2. Operation A is allowed and begins.
Assert.True(breaker.IsAllowing()); // Haven't hit the 1-operation threshold yet, should be allowed.
// #3a. Operation B errors. This is easier to simulate by changing the breaker's trip
// conditions. The sequenced Mock (above) has the second MinimumOperations returning
// 1, which would now mean we have enough operations to trip (where we didn't before).
// #3b. Breaker exceeds metrics thresholds, Operation C tries to IsAllowing() and trips breaker.
Assert.False(breaker.IsAllowing());
// #4. Operation A completes successfully.
// Breaker's internal _lastTrippedTimestamp should be equal to zero (current clock time).
// Since we say the transaction took 100ms, that'll be before the breaker tripped, and should
// be ignored.
breaker.MarkSuccess(100);
// #5. Make sure we're still tripped and we didn't reset the metrics.
Assert.False(breaker.IsAllowing());
mockMetrics.Verify(m => m.Reset(), Times.Never);
}
public void MarkSuccess_ForLongRunningSingleTest_FixesBreaker()
{
// Since we use the elapsed time in MarkSuccess(), address the following situation:
// 1. Breaker trips
// 2. Window passes
// 3. Single test is allowed
// 4. Single test takes a long time (say, > 2 windows) to complete (even though it started after we tripped).
// Verify that:
// - No other requests are allowed while the single test is waiting.
// - The single test, if successful, fixes the breaker.
// This is somewhat unlikely since we probably wont have command timeouts
// that'd allow this. But worth verifying.
// Arrange
var manualClock = new ManualTestClock();
var mockMetrics = new Mock <ICommandMetrics>();
mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(2, 100)); // 2 ops, 100% failing.
var mockEvents = new Mock <IMetricEvents>();
const long trippedDurationMillis = 30000;
var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>();
mockConfig.Setup(m => m.GetMinimumOperations(It.IsAny <GroupKey>())).Returns(1);
mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(1);
mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(trippedDurationMillis);
mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(false);
mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(false);
// 1 ops, 1% failure required to break.
var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());
// Act / Assert
Assert.False(breaker.IsAllowing()); // Should immediately trip.
manualClock.AddMilliseconds(trippedDurationMillis + 10);
Assert.True(breaker.IsAllowing()); // Single test is allowed.
Assert.False(breaker.IsAllowing());
manualClock.AddMilliseconds(trippedDurationMillis * 2);
breaker.MarkSuccess(trippedDurationMillis * 2);
// Metrics will have been reset.
mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));
Assert.True(breaker.IsAllowing());
Assert.True(breaker.IsAllowing());
}