public void IsAllowing_AfterSuccessfulSingleTest_FixesBreaker() { var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(10000) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Should reject and trip. clock.AddMilliseconds(11000); breaker.IsAllowing(); // Send the single test. clock.AddMilliseconds(500); // Pretend the test operation took a bit (not really necessary here). breaker.MarkSuccess(0); // The single test transaction marks a success. // Metrics should be reset. mockMetrics.Verify(m => m.Reset(), Times.Once); mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0)); // Should be fixed now. // Test IsAllowing() twice to make sure it's not just the single test we're allowing. Assert.True(breaker.IsAllowing()); Assert.True(breaker.IsAllowing()); }
public void IsAllowing_Tripped() { var mockStats = new Mock<IStats>(); var breaker = new BreakerBuilder(0, 0, "Test") .WithStats(mockStats.Object) .Create(); breaker.IsAllowing(); // Trip. mockStats.Verify(m => m.Elapsed("mjolnir breaker Test IsAllowing", "Rejected", It.IsAny<TimeSpan>()), Times.Once); breaker.IsAllowing(); mockStats.Verify(m => m.Elapsed("mjolnir breaker Test IsAllowing", "Rejected", It.IsAny<TimeSpan>()), Times.Exactly(2)); }
public void IsAllowing_WhenPropertiesForceFixedButBreakerWouldNormallyTrip_SilentlyTripsTheBreaker() { var mockMetrics = CreateMockMetricsWithSnapshot(2, 50); var breaker = new BreakerBuilder(1, 25).WithMockMetrics(mockMetrics).Create(); breaker.Properties.ForceFixed.Value = true; Assert.True(breaker.IsAllowing()); // Will have tripped internally. Assert.True(breaker.IsAllowing()); // Continues to allow even when tripped. // Test that if we remove the forced fix property, IsAllowing() then rejects. breaker.Properties.ForceFixed.Value = false; Assert.False(breaker.IsAllowing()); }
public void IsAllowing_AfterTrippedAndWithinWaitPeriod_Rejects() { var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(10000) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Should reject and trip. clock.AddMilliseconds(5000); // Half the wait duration. Assert.False(breaker.IsAllowing()); }
public void IsAllowing_WhenPropertiesForceFixed_Allows() { var breaker = new BreakerBuilder(0, 0).Create(); breaker.Properties.ForceFixed.Value = true; Assert.True(breaker.IsAllowing()); }
public void IsAllowing_AfterTrippedAndAfterWaitPeriod_SendsSingleTestAndRejectsOthers() { var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(10000) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Should reject and trip. clock.AddMilliseconds(11000); Assert.True(breaker.IsAllowing()); // Allow the first one, it's the single test. Assert.False(breaker.IsAllowing()); // Reject the next one, the test was already allowed. }
public void IsAllowing_WhenPropertiesForceTripped_Rejects() { // Most property values don't matter, IsAllowing() should reject before it tries to use them. var breaker = new BreakerBuilder(0, 0).Create(); breaker.Properties.ForceTripped.Value = true; Assert.False(breaker.IsAllowing()); }
public void IsAllowing_WhenBothForcePropertiesSet_Rejects() { var breaker = new BreakerBuilder(0, 0).Create(); breaker.Properties.ForceTripped.Value = true; breaker.Properties.ForceFixed.Value = true; Assert.False(breaker.IsAllowing()); }
public void MarkSuccess_WhenTrippedAndAfterWaitDuration_ResetsMetrics() { var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(1, 100); var breaker = new BreakerBuilder(1, 50) .WithMockMetrics(mockMetrics) .WithWaitMillis(10000) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Trip the breaker first. clock.AddMilliseconds(11000); Assert.True(breaker.IsAllowing()); // Single test... breaker.MarkSuccess(0); // ... that's successful. mockMetrics.Verify(m => m.Reset(), Times.Once); }
public void MarkSuccess_WhenTrippedAndAfterWaitDuration_ResetsMetrics() { var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(1, 100); var breaker = new BreakerBuilder(1, 50) .WithMockMetrics(mockMetrics) .WithWaitMillis(10000) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Trip the breaker first. clock.AddMilliseconds(11000); Assert.True(breaker.IsAllowing()); // Single test... breaker.MarkSuccess(0); // ... that's successful. mockMetrics.Verify(m => m.Reset(), Times.Once); }
public void MarkSuccess_WhenNotTripped_DoesntResetMetrics() { var mockMetrics = CreateMockMetricsWithSnapshot(0, 0); var breaker = new BreakerBuilder(10, 50).WithMockMetrics(mockMetrics).Create(); Assert.True(breaker.IsAllowing()); // Shouldn't be tripped. breaker.MarkSuccess(0); mockMetrics.Verify(m => m.Reset(), Times.Never); }
public void MarkSuccess_WhenNotTripped_DoesntResetMetrics() { var mockMetrics = CreateMockMetricsWithSnapshot(0, 0); var breaker = new BreakerBuilder(10, 50).WithMockMetrics(mockMetrics).Create(); Assert.True(breaker.IsAllowing()); // Shouldn't be tripped. breaker.MarkSuccess(0); mockMetrics.Verify(m => m.Reset(), Times.Never); }
public void IsAllowing_Allows() { var mockStats = new Mock<IStats>(); var breaker = new BreakerBuilder(10, 50, "Test") .WithStats(mockStats.Object) .Create(); breaker.IsAllowing(); mockStats.Verify(m => m.Elapsed("mjolnir breaker Test IsAllowing", "Allowed", It.IsAny<TimeSpan>()), Times.Once); }
public void IsAllowing_WhenAlreadyTripped_DoesntReTripBreaker() { const long durationMillis = 10000; var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var stats = new InternallyCountingStats(); var breaker = new BreakerBuilder(1, 1, "Test") // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(durationMillis) .WithStats(stats) .Create(); breaker.IsAllowing(); // Trip the breaker. Assert.Equal(1, stats.ServicesAndStates.Count(ss => ss.Service == "mjolnir breaker Test" && ss.State == "Tripped")); breaker.IsAllowing(); // Make another call, which should bail immediately (and not re-trip). // Best way to test this right now is to make sure we don't fire a stat for the state change. Assert.Equal(1, stats.ServicesAndStates.Count(ss => ss.Service == "mjolnir breaker Test" && ss.State == "Tripped")); }
public void IsAllowing_WhenAlreadyTripped_DoesntReTripBreaker() { const long durationMillis = 10000; var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var metricEvents = new Mock <IMetricEvents>(); var breaker = new BreakerBuilder(1, 1, "Test") // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(durationMillis) .WithMetricEvents(metricEvents.Object) .Create(); breaker.IsAllowing(); // Trip the breaker. metricEvents.Verify(m => m.BreakerTripped("Test")); metricEvents.ResetCalls(); breaker.IsAllowing(); // Make another call, which should bail immediately (and not re-trip). // Best way to test this right now is to make sure we don't fire a metric event for the state change. metricEvents.Verify(m => m.BreakerTripped(It.IsAny <string>()), Times.Never); }
public void IsAllowing_ForceTripped() { var mockStats = new Mock<IStats>(); var breaker = new BreakerBuilder(10, 50, "Test") .WithStats(mockStats.Object) .Create(); breaker.Properties.ForceTripped.Value = true; breaker.IsAllowing(); mockStats.Verify(m => m.Elapsed("mjolnir breaker Test IsAllowing", "Rejected", It.IsAny<TimeSpan>()), Times.Once); }
public void IsAllowing_AfterFailedSingleTest_KeepsBreakerTrippedAndSendsAnotherTestAfterAnotherDuration() { var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(10000) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Should reject and trip. clock.AddMilliseconds(11000); Assert.True(breaker.IsAllowing()); // Single test. // Advance to halfway through the duration. We should still be rejecting. clock.AddMilliseconds(6000); Assert.False(breaker.IsAllowing()); // No mark success call here. Metrics will probably have a failure added to it. clock.AddMilliseconds(5000); // 6000 + 5000 = 11000 > 10000 (the duration). Assert.True(breaker.IsAllowing()); // It's been another duration, we should allow another single test. // Advance to halfway through the duration. We should still be rejecting. clock.AddMilliseconds(6000); Assert.False(breaker.IsAllowing()); // No mark success call here. Metrics will probably have a failure added to it. clock.AddMilliseconds(5000); // 6000 + 5000 = 11000 > 10000 (the duration). Assert.True(breaker.IsAllowing()); // It's been another duration, we should allow another single test. // Advance a second or so, pretend the single test took some time. clock.AddMilliseconds(500); // Let's pretend this test succeeds. Mark it and reset the metrics. breaker.MarkSuccess(0); mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0)); // We should immediately be opened. Assert.True(breaker.IsAllowing()); Assert.True(breaker.IsAllowing()); Assert.True(breaker.IsAllowing()); }
public void MarkSuccess_ForLongRunningSingleTest_FixesBreaker() { // Since we use the elapsed time in MarkSuccess(), address the following situation: // 1. Breaker trips // 2. Window passes // 3. Single test is allowed // 4. Single test takes a long time (say, > 2 windows) to complete (even though it started after we tripped). // Verify that: // - No other requests are allowed while the single test is waiting. // - The single test, if successful, fixes the breaker. // This is somewhat unlikely since we probably wont have command timeouts // that'd allow this. But worth verifying. var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(2, 100); // 2 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) .WithMockMetrics(mockMetrics) .WithClock(clock) .Create(); // 1 ops, 1% failure required to break. var duration = breaker.Properties.TrippedDurationMillis; Assert.False(breaker.IsAllowing()); // Should immediately trip. clock.AddMilliseconds(duration.Value + 10); Assert.True(breaker.IsAllowing()); // Single test is allowed. Assert.False(breaker.IsAllowing()); clock.AddMilliseconds(duration.Value * 2); breaker.MarkSuccess(duration.Value * 2); // Metrics will have been reset. mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0)); Assert.True(breaker.IsAllowing()); Assert.True(breaker.IsAllowing()); }
public void MarkSuccess_ImmediatelyAfterTrippingButStartedBeforeTripped_DoesntImmediatelyFix() { // 1. Breaker is near tripping. // 2. Operation A and B are Allowed and begin work. // 3. Before Operation A completes // a. Operation B has an error and updates metrics. // b. Operation C calls IsAllowing(), which trips breaker. // 4. Operation A completes successfully and calls MarkSuccess(). // 5. Since breaker is tripped but we haven't hit our wait duration yet, // MarkSuccess() should result in the the breaker remaining tripped. var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(2, 100); // 2 ops, 100% failing. var breaker = new BreakerBuilder(5, 1) .WithMockMetrics(mockMetrics) .WithClock(clock) .Create(); // 5 ops, 1% failure required to break. // #2. Operation A is allowed and begins. Assert.True(breaker.IsAllowing()); // Haven't hit the 1-operation threshold yet, should be allowed. // #3a. Operation B errors breaker.Properties.MinimumOperations.Value = 1; // Easier to test by changing the breaker conditions. // #3b. Breaker exceeds metrics thresholds, Operation C tries to IsAllowing() and trips breaker. Assert.False(breaker.IsAllowing()); // #4. Operation A completes successfully. // Breaker's internal _lastTrippedTimestamp should be equal to zero (current clock time). // Since we say the transaction took 100ms, that'll be before the breaker tripped, and should // be ignored. breaker.MarkSuccess(100); // #5. Make sure we're still tripped and we didn't reset the metrics. Assert.False(breaker.IsAllowing()); mockMetrics.Verify(m => m.Reset(), Times.Never); }
public void IsAllowing_MultipleDurationsBetweenFailureAndNextSuccess_FixesBreakerOnSuccess() { const long durationMillis = 10000; var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(durationMillis) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Trip the breaker. clock.AddMilliseconds(durationMillis * 5); Assert.True(breaker.IsAllowing()); // Single test. breaker.MarkSuccess(0); mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0)); Assert.True(breaker.IsAllowing()); Assert.True(breaker.IsAllowing()); Assert.True(breaker.IsAllowing()); }
public void IsAllowing_MultipleDurationsBetweenFailureAndNextFailure_KeepsBreakerTripped() { const long durationMillis = 10000; var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(durationMillis) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Trip the breaker. clock.AddMilliseconds(durationMillis * 5); Assert.True(breaker.IsAllowing()); // Single test, but it won't MarkSuccess(). Assert.False(breaker.IsAllowing()); // Make sure the last one was the single test. clock.AddMilliseconds(durationMillis * 5); Assert.True(breaker.IsAllowing()); // Single test, but it won't MarkSuccess(). Assert.False(breaker.IsAllowing()); // Make sure the last one was the single test. }
public void MarkSuccess_ImmediatelyAfterTrippingButStartedBeforeTripped_DoesntImmediatelyFix() { // 1. Breaker is near tripping. // 2. Operation A and B are Allowed and begin work. // 3. Before Operation A completes // a. Operation B has an error and updates metrics. // b. Operation C calls IsAllowing(), which trips breaker. // 4. Operation A completes successfully and calls MarkSuccess(). // 5. Since breaker is tripped but we haven't hit our wait duration yet, // MarkSuccess() should result in the the breaker remaining tripped. var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(2, 100); // 2 ops, 100% failing. var breaker = new BreakerBuilder(5, 1) .WithMockMetrics(mockMetrics) .WithClock(clock) .Create(); // 5 ops, 1% failure required to break. // #2. Operation A is allowed and begins. Assert.True(breaker.IsAllowing()); // Haven't hit the 1-operation threshold yet, should be allowed. // #3a. Operation B errors breaker.Properties.MinimumOperations.Value = 1; // Easier to test by changing the breaker conditions. // #3b. Breaker exceeds metrics thresholds, Operation C tries to IsAllowing() and trips breaker. Assert.False(breaker.IsAllowing()); // #4. Operation A completes successfully. // Breaker's internal _lastTrippedTimestamp should be equal to zero (current clock time). // Since we say the transaction took 100ms, that'll be before the breaker tripped, and should // be ignored. breaker.MarkSuccess(100); // #5. Make sure we're still tripped and we didn't reset the metrics. Assert.False(breaker.IsAllowing()); mockMetrics.Verify(m => m.Reset(), Times.Never); }
public void IsAllowing_AfterTrippedAndWithinWaitPeriod_Rejects() { var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(10000) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Should reject and trip. clock.AddMilliseconds(5000); // Half the wait duration. Assert.False(breaker.IsAllowing()); }
public void IsAllowing_WhenPropertiesForceFixedButBreakerWouldNormallyTrip_SilentlyTripsTheBreaker() { var mockMetrics = CreateMockMetricsWithSnapshot(2, 50); var breaker = new BreakerBuilder(1, 25).WithMockMetrics(mockMetrics).Create(); breaker.Properties.ForceFixed.Value = true; Assert.True(breaker.IsAllowing()); // Will have tripped internally. Assert.True(breaker.IsAllowing()); // Continues to allow even when tripped. // Test that if we remove the forced fix property, IsAllowing() then rejects. breaker.Properties.ForceFixed.Value = false; Assert.False(breaker.IsAllowing()); }
public void Construct_BreakerIsntTripped() { var breaker = new BreakerBuilder(10, 50).Create(); Assert.True(breaker.IsAllowing()); }
public void IsAllowing_AfterTrippedAndAfterWaitPeriod_SendsSingleTestAndRejectsOthers() { var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(10000) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Should reject and trip. clock.AddMilliseconds(11000); Assert.True(breaker.IsAllowing()); // Allow the first one, it's the single test. Assert.False(breaker.IsAllowing()); // Reject the next one, the test was already allowed. }
public void Construct_BreakerIsntTripped() { var breaker = new BreakerBuilder(10, 50).Create(); Assert.True(breaker.IsAllowing()); }
public void AllowSingleTest_TrippedAndNotPastWaitDuration() { var mockStats = new Mock<IStats>(); var clock = new ManualTestClock(); var breaker = new BreakerBuilder(0, 0, "Test") .WithStats(mockStats.Object) .WithClock(clock) .WithWaitMillis(1000) .Create(); breaker.IsAllowing(); // Trip. mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "NotEligible", It.IsAny<TimeSpan>()), Times.Once); // Don't advance the clock. breaker.IsAllowing(); mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "NotEligible", It.IsAny<TimeSpan>()), Times.Exactly(2)); }
public void IsAllowing_MultipleDurationsBetweenFailureAndNextFailure_KeepsBreakerTripped() { const long durationMillis = 10000; var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(durationMillis) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Trip the breaker. clock.AddMilliseconds(durationMillis * 5); Assert.True(breaker.IsAllowing()); // Single test, but it won't MarkSuccess(). Assert.False(breaker.IsAllowing()); // Make sure the last one was the single test. clock.AddMilliseconds(durationMillis * 5); Assert.True(breaker.IsAllowing()); // Single test, but it won't MarkSuccess(). Assert.False(breaker.IsAllowing()); // Make sure the last one was the single test. }
public void AllowSingleTest_TrippedAndPastWaitDuration() { var mockStats = new Mock<IStats>(); var clock = new ManualTestClock(); var breaker = new BreakerBuilder(0, 0, "Test") .WithStats(mockStats.Object) .WithClock(clock) .WithWaitMillis(1000) .Create(); breaker.IsAllowing(); // Trip. mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "NotEligible", It.IsAny<TimeSpan>()), Times.Once); clock.AddMilliseconds(2000); // Advance past wait duration. breaker.IsAllowing(); mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "Allowed", It.IsAny<TimeSpan>()), Times.Once); }
public void IsAllowing_AfterFailedSingleTest_KeepsBreakerTrippedAndSendsAnotherTestAfterAnotherDuration() { var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(10000) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Should reject and trip. clock.AddMilliseconds(11000); Assert.True(breaker.IsAllowing()); // Single test. // Advance to halfway through the duration. We should still be rejecting. clock.AddMilliseconds(6000); Assert.False(breaker.IsAllowing()); // No mark success call here. Metrics will probably have a failure added to it. clock.AddMilliseconds(5000); // 6000 + 5000 = 11000 > 10000 (the duration). Assert.True(breaker.IsAllowing()); // It's been another duration, we should allow another single test. // Advance to halfway through the duration. We should still be rejecting. clock.AddMilliseconds(6000); Assert.False(breaker.IsAllowing()); // No mark success call here. Metrics will probably have a failure added to it. clock.AddMilliseconds(5000); // 6000 + 5000 = 11000 > 10000 (the duration). Assert.True(breaker.IsAllowing()); // It's been another duration, we should allow another single test. // Advance a second or so, pretend the single test took some time. clock.AddMilliseconds(500); // Let's pretend this test succeeds. Mark it and reset the metrics. breaker.MarkSuccess(0); mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0)); // We should immediately be opened. Assert.True(breaker.IsAllowing()); Assert.True(breaker.IsAllowing()); Assert.True(breaker.IsAllowing()); }
public void IsAllowing_MultipleDurationsBetweenFailureAndNextSuccess_FixesBreakerOnSuccess() { const long durationMillis = 10000; var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(durationMillis) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Trip the breaker. clock.AddMilliseconds(durationMillis * 5); Assert.True(breaker.IsAllowing()); // Single test. breaker.MarkSuccess(0); mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0)); Assert.True(breaker.IsAllowing()); Assert.True(breaker.IsAllowing()); Assert.True(breaker.IsAllowing()); }
public void CheckAndSetTripped_AlreadyTripped() { var mockStats = new Mock<IStats>(); var breaker = new BreakerBuilder(0, 0, "Test") .WithStats(mockStats.Object) .Create(); breaker.IsAllowing(); // Trip. breaker.IsAllowing(); mockStats.Verify(m => m.Elapsed("mjolnir breaker Test CheckAndSetTripped", "AlreadyTripped", It.IsAny<TimeSpan>()), Times.Once); }
public void CheckAndSetTripped_ErrorBelowThreshold() { var mockStats = new Mock<IStats>(); var breaker = new BreakerBuilder(0, 100, "Test") .WithStats(mockStats.Object) .Create(); breaker.IsAllowing(); mockStats.Verify(m => m.Elapsed("mjolnir breaker Test CheckAndSetTripped", "CriteriaNotMet", It.IsAny<TimeSpan>()), Times.Once); }
public void AllowSingleTest_NotTripped() { var mockStats = new Mock<IStats>(); var breaker = new BreakerBuilder(10, 100, "Test") .WithStats(mockStats.Object) .Create(); breaker.IsAllowing(); mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", It.IsAny<string>(), It.IsAny<TimeSpan>()), Times.Never); }
public void IsAllowing_WhenBothForcePropertiesSet_Rejects() { var breaker = new BreakerBuilder(0, 0).Create(); breaker.Properties.ForceTripped.Value = true; breaker.Properties.ForceFixed.Value = true; Assert.False(breaker.IsAllowing()); }
public void IsAllowing_WhenAlreadyTripped_DoesntReTripBreaker() { const long durationMillis = 10000; var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var stats = new InternallyCountingStats(); var metricEvents = new Mock<IMetricEvents>(); var breaker = new BreakerBuilder(1, 1, "Test") // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(durationMillis) .WithStats(stats) .WithMetricEvents(metricEvents.Object) .Create(); breaker.IsAllowing(); // Trip the breaker. Assert.Equal(1, stats.ServicesAndStates.Count(ss => ss.Service == "mjolnir breaker Test" && ss.State == "Tripped")); metricEvents.Verify(m => m.BreakerTripped("Test")); metricEvents.ResetCalls(); breaker.IsAllowing(); // Make another call, which should bail immediately (and not re-trip). // Best way to test this right now is to make sure we don't fire a stat for the state change. Assert.Equal(1, stats.ServicesAndStates.Count(ss => ss.Service == "mjolnir breaker Test" && ss.State == "Tripped")); metricEvents.Verify(m => m.BreakerTripped(It.IsAny<string>()), Times.Never); }
public void CheckAndSetTripped_JustTripped() { var mockStats = new Mock<IStats>(); var mockMetricEvents = new Mock<IMetricEvents>(); var breaker = new BreakerBuilder(0, 0, "Test") .WithStats(mockStats.Object) .WithMetricEvents(mockMetricEvents.Object) .Create(); breaker.IsAllowing(); // Trip. mockStats.Verify(m => m.Elapsed("mjolnir breaker Test CheckAndSetTripped", "JustTripped", It.IsAny<TimeSpan>()), Times.Once); mockStats.Verify(m => m.Event("mjolnir breaker Test", "Tripped", null)); mockMetricEvents.Verify(m => m.BreakerTripped("Test")); }
public void IsAllowing_WhenPropertiesForceTripped_Rejects() { // Most property values don't matter, IsAllowing() should reject before it tries to use them. var breaker = new BreakerBuilder(0, 0).Create(); breaker.Properties.ForceTripped.Value = true; Assert.False(breaker.IsAllowing()); }
public void IsAllowing_WhenPropertiesForceFixed_Allows() { var breaker = new BreakerBuilder(0, 0).Create(); breaker.Properties.ForceFixed.Value = true; Assert.True(breaker.IsAllowing()); }
public void MarkSuccess_Fixed() { var mockStats = new Mock<IStats>(); var mockMetricEvents = new Mock<IMetricEvents>(); var breaker = new BreakerBuilder(0, 0, "Test") .WithStats(mockStats.Object) .WithMetricEvents(mockMetricEvents.Object) .Create(); breaker.IsAllowing(); // Trip. breaker.MarkSuccess(0); mockStats.Verify(m => m.Event("mjolnir breaker Test MarkSuccess", "Fixed", null), Times.Once); mockMetricEvents.Verify(m => m.BreakerFixed("Test")); }
public void MarkSuccess_ForLongRunningSingleTest_FixesBreaker() { // Since we use the elapsed time in MarkSuccess(), address the following situation: // 1. Breaker trips // 2. Window passes // 3. Single test is allowed // 4. Single test takes a long time (say, > 2 windows) to complete (even though it started after we tripped). // Verify that: // - No other requests are allowed while the single test is waiting. // - The single test, if successful, fixes the breaker. // This is somewhat unlikely since we probably wont have command timeouts // that'd allow this. But worth verifying. var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(2, 100); // 2 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) .WithMockMetrics(mockMetrics) .WithClock(clock) .Create(); // 1 ops, 1% failure required to break. var duration = breaker.Properties.TrippedDurationMillis; Assert.False(breaker.IsAllowing()); // Should immediately trip. clock.AddMilliseconds(duration.Value + 10); Assert.True(breaker.IsAllowing()); // Single test is allowed. Assert.False(breaker.IsAllowing()); clock.AddMilliseconds(duration.Value * 2); breaker.MarkSuccess(duration.Value * 2); // Metrics will have been reset. mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0)); Assert.True(breaker.IsAllowing()); Assert.True(breaker.IsAllowing()); }
public void IsAllowing_AfterSuccessfulSingleTest_FixesBreaker() { var clock = new ManualTestClock(); var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing. var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing. .WithMockMetrics(mockMetrics) .WithWaitMillis(10000) .WithClock(clock) .Create(); Assert.False(breaker.IsAllowing()); // Should reject and trip. clock.AddMilliseconds(11000); breaker.IsAllowing(); // Send the single test. clock.AddMilliseconds(500); // Pretend the test operation took a bit (not really necessary here). breaker.MarkSuccess(0); // The single test transaction marks a success. // Metrics should be reset. mockMetrics.Verify(m => m.Reset(), Times.Once); mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0)); // Should be fixed now. // Test IsAllowing() twice to make sure it's not just the single test we're allowing. Assert.True(breaker.IsAllowing()); Assert.True(breaker.IsAllowing()); }