public void MarkSuccess_WhenNotTripped_DoesntResetMetrics()
{
var mockMetrics = CreateMockMetricsWithSnapshot(0, 0);
var breaker = new BreakerBuilder(10, 50).WithMockMetrics(mockMetrics).Create();
Assert.True(breaker.IsAllowing()); // Shouldn't be tripped.
breaker.MarkSuccess(0);
mockMetrics.Verify(m => m.Reset(), Times.Never);
}
public void IsAllowing_Allows()
{
var mockStats = new Mock<IStats>();
var breaker = new BreakerBuilder(10, 50, "Test")
.WithStats(mockStats.Object)
.Create();
breaker.IsAllowing();
mockStats.Verify(m => m.Elapsed("mjolnir breaker Test IsAllowing", "Allowed", It.IsAny<TimeSpan>()), Times.Once);
}
public void IsAllowing_ForceTripped()
{
var mockStats = new Mock<IStats>();
var breaker = new BreakerBuilder(10, 50, "Test")
.WithStats(mockStats.Object)
.Create();
breaker.Properties.ForceTripped.Value = true;
breaker.IsAllowing();
mockStats.Verify(m => m.Elapsed("mjolnir breaker Test IsAllowing", "Rejected", It.IsAny<TimeSpan>()), Times.Once);
}
public void IsAllowing_Tripped()
{
var mockStats = new Mock<IStats>();
var breaker = new BreakerBuilder(0, 0, "Test")
.WithStats(mockStats.Object)
.Create();
breaker.IsAllowing(); // Trip.
mockStats.Verify(m => m.Elapsed("mjolnir breaker Test IsAllowing", "Rejected", It.IsAny<TimeSpan>()), Times.Once);
breaker.IsAllowing();
mockStats.Verify(m => m.Elapsed("mjolnir breaker Test IsAllowing", "Rejected", It.IsAny<TimeSpan>()), Times.Exactly(2));
}
public async Task Construct_CreatesGauges()
{
const long gaugeIntervalMillis = 50;
var mockStats = new Mock<IStats>();
var breaker = new BreakerBuilder(10, 50, "Test")
.WithStats(mockStats.Object)
.WithGaugeIntervalOverride(gaugeIntervalMillis)
.Create();
await Task.Delay(TimeSpan.FromMilliseconds(gaugeIntervalMillis + 50));
mockStats.Verify(m => m.Gauge("mjolnir breaker Test total", It.IsIn("Above", "Below"), It.IsAny<long>()), Times.AtLeastOnce);
mockStats.Verify(m => m.Gauge("mjolnir breaker Test error", It.IsIn("Above", "Below"), It.IsAny<int>()), Times.AtLeastOnce);
}
public void MarkSuccess_WhenTrippedAndAfterWaitDuration_ResetsMetrics()
{
var clock = new ManualTestClock();
var mockMetrics = CreateMockMetricsWithSnapshot(1, 100);
var breaker = new BreakerBuilder(1, 50)
.WithMockMetrics(mockMetrics)
.WithWaitMillis(10000)
.WithClock(clock)
.Create();
Assert.False(breaker.IsAllowing()); // Trip the breaker first.
clock.AddMilliseconds(11000);
Assert.True(breaker.IsAllowing()); // Single test...
breaker.MarkSuccess(0); // ... that's successful.
mockMetrics.Verify(m => m.Reset(), Times.Once);
}
public void MarkSuccess_ImmediatelyAfterTrippingButStartedBeforeTripped_DoesntImmediatelyFix()
{
// 1. Breaker is near tripping.
// 2. Operation A and B are Allowed and begin work.
// 3. Before Operation A completes
// a. Operation B has an error and updates metrics.
// b. Operation C calls IsAllowing(), which trips breaker.
// 4. Operation A completes successfully and calls MarkSuccess().
// 5. Since breaker is tripped but we haven't hit our wait duration yet,
// MarkSuccess() should result in the the breaker remaining tripped.
var clock = new ManualTestClock();
var mockMetrics = CreateMockMetricsWithSnapshot(2, 100); // 2 ops, 100% failing.
var breaker = new BreakerBuilder(5, 1)
.WithMockMetrics(mockMetrics)
.WithClock(clock)
.Create(); // 5 ops, 1% failure required to break.
// #2. Operation A is allowed and begins.
Assert.True(breaker.IsAllowing()); // Haven't hit the 1-operation threshold yet, should be allowed.
// #3a. Operation B errors
breaker.Properties.MinimumOperations.Value = 1; // Easier to test by changing the breaker conditions.
// #3b. Breaker exceeds metrics thresholds, Operation C tries to IsAllowing() and trips breaker.
Assert.False(breaker.IsAllowing());
// #4. Operation A completes successfully.
// Breaker's internal _lastTrippedTimestamp should be equal to zero (current clock time).
// Since we say the transaction took 100ms, that'll be before the breaker tripped, and should
// be ignored.
breaker.MarkSuccess(100);
// #5. Make sure we're still tripped and we didn't reset the metrics.
Assert.False(breaker.IsAllowing());
mockMetrics.Verify(m => m.Reset(), Times.Never);
}
public void CheckAndSetTripped_JustTripped()
{
var mockStats = new Mock<IStats>();
var mockMetricEvents = new Mock<IMetricEvents>();
var breaker = new BreakerBuilder(0, 0, "Test")
.WithStats(mockStats.Object)
.WithMetricEvents(mockMetricEvents.Object)
.Create();
breaker.IsAllowing(); // Trip.
mockStats.Verify(m => m.Elapsed("mjolnir breaker Test CheckAndSetTripped", "JustTripped", It.IsAny<TimeSpan>()), Times.Once);
mockStats.Verify(m => m.Event("mjolnir breaker Test", "Tripped", null));
mockMetricEvents.Verify(m => m.BreakerTripped("Test"));
}
public void CheckAndSetTripped_AlreadyTripped()
{
var mockStats = new Mock<IStats>();
var breaker = new BreakerBuilder(0, 0, "Test")
.WithStats(mockStats.Object)
.Create();
breaker.IsAllowing(); // Trip.
breaker.IsAllowing();
mockStats.Verify(m => m.Elapsed("mjolnir breaker Test CheckAndSetTripped", "AlreadyTripped", It.IsAny<TimeSpan>()), Times.Once);
}
public void CheckAndSetTripped_ErrorBelowThreshold()
{
var mockStats = new Mock<IStats>();
var breaker = new BreakerBuilder(0, 100, "Test")
.WithStats(mockStats.Object)
.Create();
breaker.IsAllowing();
mockStats.Verify(m => m.Elapsed("mjolnir breaker Test CheckAndSetTripped", "CriteriaNotMet", It.IsAny<TimeSpan>()), Times.Once);
}
public void IsAllowing_WhenBothForcePropertiesSet_Rejects()
{
var breaker = new BreakerBuilder(0, 0).Create();
breaker.Properties.ForceTripped.Value = true;
breaker.Properties.ForceFixed.Value = true;
Assert.False(breaker.IsAllowing());
}
public void AllowSingleTest_NotTripped()
{
var mockStats = new Mock<IStats>();
var breaker = new BreakerBuilder(10, 100, "Test")
.WithStats(mockStats.Object)
.Create();
breaker.IsAllowing();
mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", It.IsAny<string>(), It.IsAny<TimeSpan>()), Times.Never);
}
public void Construct_BreakerIsntTripped()
{
var breaker = new BreakerBuilder(10, 50).Create();
Assert.True(breaker.IsAllowing());
}
public void IsAllowing_WhenPropertiesForceFixed_Allows()
{
var breaker = new BreakerBuilder(0, 0).Create();
breaker.Properties.ForceFixed.Value = true;
Assert.True(breaker.IsAllowing());
}
public void IsAllowing_WhenPropertiesForceTripped_Rejects()
{
// Most property values don't matter, IsAllowing() should reject before it tries to use them.
var breaker = new BreakerBuilder(0, 0).Create();
breaker.Properties.ForceTripped.Value = true;
Assert.False(breaker.IsAllowing());
}
public void IsAllowing_WhenPropertiesForceFixedButBreakerWouldNormallyTrip_SilentlyTripsTheBreaker()
{
var mockMetrics = CreateMockMetricsWithSnapshot(2, 50);
var breaker = new BreakerBuilder(1, 25).WithMockMetrics(mockMetrics).Create();
breaker.Properties.ForceFixed.Value = true;
Assert.True(breaker.IsAllowing()); // Will have tripped internally.
Assert.True(breaker.IsAllowing()); // Continues to allow even when tripped.
// Test that if we remove the forced fix property, IsAllowing() then rejects.
breaker.Properties.ForceFixed.Value = false;
Assert.False(breaker.IsAllowing());
}
public void IsAllowing_MultipleDurationsBetweenFailureAndNextFailure_KeepsBreakerTripped()
{
const long durationMillis = 10000;
var clock = new ManualTestClock();
var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing.
.WithMockMetrics(mockMetrics)
.WithWaitMillis(durationMillis)
.WithClock(clock)
.Create();
Assert.False(breaker.IsAllowing()); // Trip the breaker.
clock.AddMilliseconds(durationMillis * 5);
Assert.True(breaker.IsAllowing()); // Single test, but it won't MarkSuccess().
Assert.False(breaker.IsAllowing()); // Make sure the last one was the single test.
clock.AddMilliseconds(durationMillis * 5);
Assert.True(breaker.IsAllowing()); // Single test, but it won't MarkSuccess().
Assert.False(breaker.IsAllowing()); // Make sure the last one was the single test.
}
public void IsAllowing_AfterTrippedAndWithinWaitPeriod_Rejects()
{
var clock = new ManualTestClock();
var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing.
.WithMockMetrics(mockMetrics)
.WithWaitMillis(10000)
.WithClock(clock)
.Create();
Assert.False(breaker.IsAllowing()); // Should reject and trip.
clock.AddMilliseconds(5000); // Half the wait duration.
Assert.False(breaker.IsAllowing());
}
public void IsAllowing_MultipleDurationsBetweenFailureAndNextSuccess_FixesBreakerOnSuccess()
{
const long durationMillis = 10000;
var clock = new ManualTestClock();
var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing.
.WithMockMetrics(mockMetrics)
.WithWaitMillis(durationMillis)
.WithClock(clock)
.Create();
Assert.False(breaker.IsAllowing()); // Trip the breaker.
clock.AddMilliseconds(durationMillis * 5);
Assert.True(breaker.IsAllowing()); // Single test.
breaker.MarkSuccess(0);
mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));
Assert.True(breaker.IsAllowing());
Assert.True(breaker.IsAllowing());
Assert.True(breaker.IsAllowing());
}
public void IsAllowing_AfterFailedSingleTest_KeepsBreakerTrippedAndSendsAnotherTestAfterAnotherDuration()
{
var clock = new ManualTestClock();
var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing.
.WithMockMetrics(mockMetrics)
.WithWaitMillis(10000)
.WithClock(clock)
.Create();
Assert.False(breaker.IsAllowing()); // Should reject and trip.
clock.AddMilliseconds(11000);
Assert.True(breaker.IsAllowing()); // Single test.
// Advance to halfway through the duration. We should still be rejecting.
clock.AddMilliseconds(6000);
Assert.False(breaker.IsAllowing());
// No mark success call here. Metrics will probably have a failure added to it.
clock.AddMilliseconds(5000); // 6000 + 5000 = 11000 > 10000 (the duration).
Assert.True(breaker.IsAllowing()); // It's been another duration, we should allow another single test.
// Advance to halfway through the duration. We should still be rejecting.
clock.AddMilliseconds(6000);
Assert.False(breaker.IsAllowing());
// No mark success call here. Metrics will probably have a failure added to it.
clock.AddMilliseconds(5000); // 6000 + 5000 = 11000 > 10000 (the duration).
Assert.True(breaker.IsAllowing()); // It's been another duration, we should allow another single test.
// Advance a second or so, pretend the single test took some time.
clock.AddMilliseconds(500);
// Let's pretend this test succeeds. Mark it and reset the metrics.
breaker.MarkSuccess(0);
mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));
// We should immediately be opened.
Assert.True(breaker.IsAllowing());
Assert.True(breaker.IsAllowing());
Assert.True(breaker.IsAllowing());
}
public void IsAllowing_AfterSuccessfulSingleTest_FixesBreaker()
{
var clock = new ManualTestClock();
var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing.
.WithMockMetrics(mockMetrics)
.WithWaitMillis(10000)
.WithClock(clock)
.Create();
Assert.False(breaker.IsAllowing()); // Should reject and trip.
clock.AddMilliseconds(11000);
breaker.IsAllowing(); // Send the single test.
clock.AddMilliseconds(500); // Pretend the test operation took a bit (not really necessary here).
breaker.MarkSuccess(0); // The single test transaction marks a success.
// Metrics should be reset.
mockMetrics.Verify(m => m.Reset(), Times.Once);
mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));
// Should be fixed now.
// Test IsAllowing() twice to make sure it's not just the single test we're allowing.
Assert.True(breaker.IsAllowing());
Assert.True(breaker.IsAllowing());
}
public void MarkSuccess_NotTripped()
{
var mockStats = new Mock<IStats>();
var breaker = new BreakerBuilder(0, 0, "Test")
.WithStats(mockStats.Object)
.Create();
breaker.MarkSuccess(0);
mockStats.Verify(m => m.Event("mjolnir breaker Test MarkSuccess", "Ignored", null), Times.Once);
}
public void MarkSuccess_Fixed()
{
var mockStats = new Mock<IStats>();
var mockMetricEvents = new Mock<IMetricEvents>();
var breaker = new BreakerBuilder(0, 0, "Test")
.WithStats(mockStats.Object)
.WithMetricEvents(mockMetricEvents.Object)
.Create();
breaker.IsAllowing(); // Trip.
breaker.MarkSuccess(0);
mockStats.Verify(m => m.Event("mjolnir breaker Test MarkSuccess", "Fixed", null), Times.Once);
mockMetricEvents.Verify(m => m.BreakerFixed("Test"));
}
public void AllowSingleTest_TrippedAndPastWaitDuration()
{
var mockStats = new Mock<IStats>();
var clock = new ManualTestClock();
var breaker = new BreakerBuilder(0, 0, "Test")
.WithStats(mockStats.Object)
.WithClock(clock)
.WithWaitMillis(1000)
.Create();
breaker.IsAllowing(); // Trip.
mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "NotEligible", It.IsAny<TimeSpan>()), Times.Once);
clock.AddMilliseconds(2000); // Advance past wait duration.
breaker.IsAllowing();
mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "Allowed", It.IsAny<TimeSpan>()), Times.Once);
}
public void IsAllowing_WhenAlreadyTripped_DoesntReTripBreaker()
{
const long durationMillis = 10000;
var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
var stats = new InternallyCountingStats();
var metricEvents = new Mock<IMetricEvents>();
var breaker = new BreakerBuilder(1, 1, "Test") // Trip at 1 op, 1% failing.
.WithMockMetrics(mockMetrics)
.WithWaitMillis(durationMillis)
.WithStats(stats)
.WithMetricEvents(metricEvents.Object)
.Create();
breaker.IsAllowing(); // Trip the breaker.
Assert.Equal(1, stats.ServicesAndStates.Count(ss => ss.Service == "mjolnir breaker Test" && ss.State == "Tripped"));
metricEvents.Verify(m => m.BreakerTripped("Test"));
metricEvents.ResetCalls();
breaker.IsAllowing(); // Make another call, which should bail immediately (and not re-trip).
// Best way to test this right now is to make sure we don't fire a stat for the state change.
Assert.Equal(1, stats.ServicesAndStates.Count(ss => ss.Service == "mjolnir breaker Test" && ss.State == "Tripped"));
metricEvents.Verify(m => m.BreakerTripped(It.IsAny<string>()), Times.Never);
}
public void IsAllowing_AfterTrippedAndAfterWaitPeriod_SendsSingleTestAndRejectsOthers()
{
var clock = new ManualTestClock();
var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing.
.WithMockMetrics(mockMetrics)
.WithWaitMillis(10000)
.WithClock(clock)
.Create();
Assert.False(breaker.IsAllowing()); // Should reject and trip.
clock.AddMilliseconds(11000);
Assert.True(breaker.IsAllowing()); // Allow the first one, it's the single test.
Assert.False(breaker.IsAllowing()); // Reject the next one, the test was already allowed.
}
public void AllowSingleTest_TrippedAndNotPastWaitDuration()
{
var mockStats = new Mock<IStats>();
var clock = new ManualTestClock();
var breaker = new BreakerBuilder(0, 0, "Test")
.WithStats(mockStats.Object)
.WithClock(clock)
.WithWaitMillis(1000)
.Create();
breaker.IsAllowing(); // Trip.
mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "NotEligible", It.IsAny<TimeSpan>()), Times.Once);
// Don't advance the clock.
breaker.IsAllowing();
mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "NotEligible", It.IsAny<TimeSpan>()), Times.Exactly(2));
}
public void MarkSuccess_ForLongRunningSingleTest_FixesBreaker()
{
// Since we use the elapsed time in MarkSuccess(), address the following situation:
// 1. Breaker trips
// 2. Window passes
// 3. Single test is allowed
// 4. Single test takes a long time (say, > 2 windows) to complete (even though it started after we tripped).
// Verify that:
// - No other requests are allowed while the single test is waiting.
// - The single test, if successful, fixes the breaker.
// This is somewhat unlikely since we probably wont have command timeouts
// that'd allow this. But worth verifying.
var clock = new ManualTestClock();
var mockMetrics = CreateMockMetricsWithSnapshot(2, 100); // 2 ops, 100% failing.
var breaker = new BreakerBuilder(1, 1)
.WithMockMetrics(mockMetrics)
.WithClock(clock)
.Create(); // 1 ops, 1% failure required to break.
var duration = breaker.Properties.TrippedDurationMillis;
Assert.False(breaker.IsAllowing()); // Should immediately trip.
clock.AddMilliseconds(duration.Value + 10);
Assert.True(breaker.IsAllowing()); // Single test is allowed.
Assert.False(breaker.IsAllowing());
clock.AddMilliseconds(duration.Value * 2);
breaker.MarkSuccess(duration.Value * 2);
// Metrics will have been reset.
mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));
Assert.True(breaker.IsAllowing());
Assert.True(breaker.IsAllowing());
}
