public void IsAllowing_WhenBothForcePropertiesSet_Rejects()
        {
            // Arrange

            // Most property values don't matter, IsAllowing() should reject before it tries to use them.
            var manualClock = new ManualTestClock();

            var mockMetrics = new Mock <ICommandMetrics>();

            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));

            var mockEvents = new Mock <IMetricEvents>();

            var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>();

            mockConfig.Setup(m => m.GetMinimumOperations(It.IsAny <GroupKey>())).Returns(1);
            mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(1);
            mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(30000);
            mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(true); // The config we're testing here.
            mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(true);   // The config we're testing here.

            var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());


            // Act / Assert

            Assert.False(breaker.IsAllowing());
        }
예제 #2
0
        public void ManualTestClock_GetMillisecondTimestamp_UpdatesAfterAddingMillis()
        {
            var clock = new ManualTestClock();

            clock.AddMilliseconds(20);
            Assert.Equal(20, clock.GetMillisecondTimestamp());
        }
        public void IsAllowing_AfterSuccessfulSingleTest_FixesBreaker()
        {
            var clock       = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
            var breaker     = new BreakerBuilder(1, 1)                // Trip at 1 op, 1% failing.
                              .WithMockMetrics(mockMetrics)
                              .WithWaitMillis(10000)
                              .WithClock(clock)
                              .Create();

            Assert.False(breaker.IsAllowing()); // Should reject and trip.

            clock.AddMilliseconds(11000);
            breaker.IsAllowing();       // Send the single test.

            clock.AddMilliseconds(500); // Pretend the test operation took a bit (not really necessary here).
            breaker.MarkSuccess(0);     // The single test transaction marks a success.

            // Metrics should be reset.
            mockMetrics.Verify(m => m.Reset(), Times.Once);
            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));

            // Should be fixed now.
            // Test IsAllowing() twice to make sure it's not just the single test we're allowing.
            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
        }
예제 #4
0
        public void ManualTestClock_GetMillisecondTimestamp_DoesntAdvanceAutomatically()
        {
            var clock = new ManualTestClock();

            Thread.Sleep(10);
            Assert.Equal(0, clock.GetMillisecondTimestamp());
        }
        public void MarkSuccess_ImmediatelyAfterTrippingButStartedBeforeTripped_DoesntImmediatelyFix()
        {
            // 1. Breaker is near tripping.
            // 2. Operation A and B are Allowed and begin work.
            // 3. Before Operation A completes
            //    a. Operation B has an error and updates metrics.
            //    b. Operation C calls IsAllowing(), which trips breaker.
            // 4. Operation A completes successfully and calls MarkSuccess().
            // 5. Since breaker is tripped but we haven't hit our wait duration yet,
            //    MarkSuccess() should result in the the breaker remaining tripped.


            // Arrange

            var manualClock = new ManualTestClock();

            var mockMetrics = new Mock <ICommandMetrics>();

            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(2, 100)); // 2 ops, 100% failing.

            var mockEvents = new Mock <IMetricEvents>();

            var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>();

            mockConfig.SetupSequence(m => m.GetMinimumOperations(It.IsAny <GroupKey>()))
            .Returns(5)     // First access should be > 1 so that the breaker doesn't trip.
            .Returns(1);    // Second access should be 1, so that we trip the breaker because we have the minimum met.

            mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(1);
            mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(30000);
            mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(false);
            mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(false);

            // 5 ops, 1% failure required to break.
            var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());


            // Act / Assert

            // #2. Operation A is allowed and begins.
            Assert.True(breaker.IsAllowing()); // Haven't hit the 1-operation threshold yet, should be allowed.

            // #3a. Operation B errors. This is easier to simulate by changing the breaker's trip
            //      conditions. The sequenced Mock (above) has the second MinimumOperations returning
            //      1, which would now mean we have enough operations to trip (where we didn't before).

            // #3b. Breaker exceeds metrics thresholds, Operation C tries to IsAllowing() and trips breaker.
            Assert.False(breaker.IsAllowing());

            // #4. Operation A completes successfully.
            // Breaker's internal _lastTrippedTimestamp should be equal to zero (current clock time).
            // Since we say the transaction took 100ms, that'll be before the breaker tripped, and should
            // be ignored.
            breaker.MarkSuccess(100);

            // #5. Make sure we're still tripped and we didn't reset the metrics.
            Assert.False(breaker.IsAllowing());
            mockMetrics.Verify(m => m.Reset(), Times.Never);
        }
        public void MarkSuccess_ForLongRunningSingleTest_FixesBreaker()
        {
            // Since we use the elapsed time in MarkSuccess(), address the following situation:
            // 1. Breaker trips
            // 2. Window passes
            // 3. Single test is allowed
            // 4. Single test takes a long time (say, > 2 windows) to complete (even though it started after we tripped).
            // Verify that:
            // - No other requests are allowed while the single test is waiting.
            // - The single test, if successful, fixes the breaker.

            // This is somewhat unlikely since we probably wont have command timeouts
            // that'd allow this. But worth verifying.


            // Arrange

            var manualClock = new ManualTestClock();

            var mockMetrics = new Mock <ICommandMetrics>();

            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(2, 100)); // 2 ops, 100% failing.

            var mockEvents = new Mock <IMetricEvents>();

            const long trippedDurationMillis = 30000;
            var        mockConfig            = new Mock <IFailurePercentageCircuitBreakerConfig>();

            mockConfig.Setup(m => m.GetMinimumOperations(It.IsAny <GroupKey>())).Returns(1);
            mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(1);
            mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(trippedDurationMillis);
            mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(false);
            mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(false);

            // 1 ops, 1% failure required to break.
            var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());


            // Act / Assert

            Assert.False(breaker.IsAllowing()); // Should immediately trip.

            manualClock.AddMilliseconds(trippedDurationMillis + 10);

            Assert.True(breaker.IsAllowing()); // Single test is allowed.
            Assert.False(breaker.IsAllowing());

            manualClock.AddMilliseconds(trippedDurationMillis * 2);
            breaker.MarkSuccess(trippedDurationMillis * 2);

            // Metrics will have been reset.
            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));

            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
        }
        private ResettingNumbersBucket CreateBucket()
        {
            var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>(MockBehavior.Strict);

            mockConfig.Setup(m => m.GetWindowMillis(It.IsAny <GroupKey>())).Returns(10000);

            var clock = new ManualTestClock();

            return(new ResettingNumbersBucket(AnyGroupKey, clock, mockConfig.Object, new DefaultMjolnirLogFactory()));
        }
        public void Increment_AfterPeriodExceeded_ResetsBeforeIncrementing()
        {
            const long periodMillis = 1000;
            var clock = new ManualTestClock();
            var bucket = new ResettingNumbersBucket(clock, new TransientConfigurableValue<long>(periodMillis));

            bucket.Increment(CounterMetric.CommandSuccess);

            clock.AddMilliseconds(periodMillis + 1);
            Assert.Equal(1, bucket.GetCount(CounterMetric.CommandSuccess));

            bucket.Increment(CounterMetric.CommandSuccess); // Should reset and then count one.
            Assert.Equal(1, bucket.GetCount(CounterMetric.CommandSuccess)); // Should be 1, not 2.
        }
        public void Increment_AfterPeriodExceeded_ResetsBeforeIncrementing()
        {
            const long periodMillis = 1000;
            var        clock        = new ManualTestClock();
            var        bucket       = new ResettingNumbersBucket(clock, new TransientConfigurableValue <long>(periodMillis));

            bucket.Increment(CounterMetric.CommandSuccess);

            clock.AddMilliseconds(periodMillis + 1);
            Assert.Equal(1, bucket.GetCount(CounterMetric.CommandSuccess));

            bucket.Increment(CounterMetric.CommandSuccess);                 // Should reset and then count one.
            Assert.Equal(1, bucket.GetCount(CounterMetric.CommandSuccess)); // Should be 1, not 2.
        }
        public void IsAllowing_AfterTrippedAndWithinWaitPeriod_Rejects()
        {
            var clock       = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
            var breaker     = new BreakerBuilder(1, 1)                // Trip at 1 op, 1% failing.
                              .WithMockMetrics(mockMetrics)
                              .WithWaitMillis(10000)
                              .WithClock(clock)
                              .Create();

            Assert.False(breaker.IsAllowing()); // Should reject and trip.

            clock.AddMilliseconds(5000);        // Half the wait duration.

            Assert.False(breaker.IsAllowing());
        }
        public void IsAllowing_AfterTrippedAndAfterWaitPeriod_SendsSingleTestAndRejectsOthers()
        {
            var clock       = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
            var breaker     = new BreakerBuilder(1, 1)                // Trip at 1 op, 1% failing.
                              .WithMockMetrics(mockMetrics)
                              .WithWaitMillis(10000)
                              .WithClock(clock)
                              .Create();

            Assert.False(breaker.IsAllowing()); // Should reject and trip.

            clock.AddMilliseconds(11000);

            Assert.True(breaker.IsAllowing());  // Allow the first one, it's the single test.
            Assert.False(breaker.IsAllowing()); // Reject the next one, the test was already allowed.
        }
        public void IsAllowing_AfterFailedSingleTest_KeepsBreakerTrippedAndSendsAnotherTestAfterAnotherDuration()
        {
            var clock       = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
            var breaker     = new BreakerBuilder(1, 1)                // Trip at 1 op, 1% failing.
                              .WithMockMetrics(mockMetrics)
                              .WithWaitMillis(10000)
                              .WithClock(clock)
                              .Create();

            Assert.False(breaker.IsAllowing()); // Should reject and trip.

            clock.AddMilliseconds(11000);
            Assert.True(breaker.IsAllowing()); // Single test.

            // Advance to halfway through the duration. We should still be rejecting.
            clock.AddMilliseconds(6000);
            Assert.False(breaker.IsAllowing());

            // No mark success call here. Metrics will probably have a failure added to it.

            clock.AddMilliseconds(5000);       // 6000 + 5000 = 11000 > 10000 (the duration).
            Assert.True(breaker.IsAllowing()); // It's been another duration, we should allow another single test.

            // Advance to halfway through the duration. We should still be rejecting.
            clock.AddMilliseconds(6000);
            Assert.False(breaker.IsAllowing());

            // No mark success call here. Metrics will probably have a failure added to it.

            clock.AddMilliseconds(5000);       // 6000 + 5000 = 11000 > 10000 (the duration).
            Assert.True(breaker.IsAllowing()); // It's been another duration, we should allow another single test.

            // Advance a second or so, pretend the single test took some time.
            clock.AddMilliseconds(500);

            // Let's pretend this test succeeds. Mark it and reset the metrics.
            breaker.MarkSuccess(0);
            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));

            // We should immediately be opened.
            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
        }
        public void GetSnapshot_NotCachedAndCached()
        {
            var mockStats = new Mock<IStats>();
            var clock = new ManualTestClock();
            var metrics = CreateMetrics("Test", mockStats, clock, 30000, 10000);

            clock.AddMilliseconds(10100); // Pass the snapshot TTL.

            metrics.GetSnapshot(); // Should create new snapshot.

            mockStats.Verify(m => m.Elapsed("mjolnir metrics Test CreateSnapshot", null, It.IsAny<TimeSpan>()), Times.Once);
            mockStats.Verify(m => m.Elapsed("mjolnir metrics Test GetSnapshot", null, It.IsAny<TimeSpan>()), Times.Once);

            metrics.GetSnapshot(); // Should grab the cached one without re-creating.

            mockStats.Verify(m => m.Elapsed("mjolnir metrics Test CreateSnapshot", null, It.IsAny<TimeSpan>()), Times.Once); // Still once.
            mockStats.Verify(m => m.Elapsed("mjolnir metrics Test GetSnapshot", null, It.IsAny<TimeSpan>()), Times.Exactly(2)); // One more time.
        }
        public void AllowSingleTest_TrippedAndNotPastWaitDuration()
        {
            var mockStats = new Mock <IStats>();
            var clock     = new ManualTestClock();
            var breaker   = new BreakerBuilder(0, 0, "Test")
                            .WithStats(mockStats.Object)
                            .WithClock(clock)
                            .WithWaitMillis(1000)
                            .Create();

            breaker.IsAllowing(); // Trip.
            mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "NotEligible", It.IsAny <TimeSpan>()), Times.Once);
            // Don't advance the clock.

            breaker.IsAllowing();

            mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "NotEligible", It.IsAny <TimeSpan>()), Times.Exactly(2));
        }
        public void AllowSingleTest_TrippedAndPastWaitDuration()
        {
            var mockStats = new Mock <IStats>();
            var clock     = new ManualTestClock();
            var breaker   = new BreakerBuilder(0, 0, "Test")
                            .WithStats(mockStats.Object)
                            .WithClock(clock)
                            .WithWaitMillis(1000)
                            .Create();

            breaker.IsAllowing();        // Trip.
            mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "NotEligible", It.IsAny <TimeSpan>()), Times.Once);
            clock.AddMilliseconds(2000); // Advance past wait duration.

            breaker.IsAllowing();

            mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "Allowed", It.IsAny <TimeSpan>()), Times.Once);
        }
        public void MarkSuccess_WhenTrippedAndAfterWaitDuration_ResetsMetrics()
        {
            var clock = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(1, 100);
            var breaker = new BreakerBuilder(1, 50)
                .WithMockMetrics(mockMetrics)
                .WithWaitMillis(10000)
                .WithClock(clock)
                .Create();

            Assert.False(breaker.IsAllowing()); // Trip the breaker first.

            clock.AddMilliseconds(11000);

            Assert.True(breaker.IsAllowing()); // Single test...
            breaker.MarkSuccess(0); // ... that's successful.
            mockMetrics.Verify(m => m.Reset(), Times.Once);
        }
예제 #17
0
        public void GetSnapshot_NotCachedAndCached()
        {
            var mockStats = new Mock <IStats>();
            var clock     = new ManualTestClock();
            var metrics   = CreateMetrics("Test", mockStats, clock, 30000, 10000);

            clock.AddMilliseconds(10100); // Pass the snapshot TTL.

            metrics.GetSnapshot();        // Should create new snapshot.

            mockStats.Verify(m => m.Elapsed("mjolnir metrics Test CreateSnapshot", null, It.IsAny <TimeSpan>()), Times.Once);
            mockStats.Verify(m => m.Elapsed("mjolnir metrics Test GetSnapshot", null, It.IsAny <TimeSpan>()), Times.Once);

            metrics.GetSnapshot();                                                                                               // Should grab the cached one without re-creating.

            mockStats.Verify(m => m.Elapsed("mjolnir metrics Test CreateSnapshot", null, It.IsAny <TimeSpan>()), Times.Once);    // Still once.
            mockStats.Verify(m => m.Elapsed("mjolnir metrics Test GetSnapshot", null, It.IsAny <TimeSpan>()), Times.Exactly(2)); // One more time.
        }
        public void MarkSuccess_WhenTrippedAndAfterWaitDuration_ResetsMetrics()
        {
            var clock       = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(1, 100);
            var breaker     = new BreakerBuilder(1, 50)
                              .WithMockMetrics(mockMetrics)
                              .WithWaitMillis(10000)
                              .WithClock(clock)
                              .Create();

            Assert.False(breaker.IsAllowing()); // Trip the breaker first.

            clock.AddMilliseconds(11000);

            Assert.True(breaker.IsAllowing()); // Single test...
            breaker.MarkSuccess(0);            // ... that's successful.
            mockMetrics.Verify(m => m.Reset(), Times.Once);
        }
        public void Increment_AfterPeriodExceeded_ResetsBeforeIncrementing()
        {
            const long periodMillis = 1000;

            var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>(MockBehavior.Strict);

            mockConfig.Setup(m => m.GetWindowMillis(It.IsAny <GroupKey>())).Returns(periodMillis);

            var clock  = new ManualTestClock();
            var bucket = new ResettingNumbersBucket(AnyGroupKey, clock, mockConfig.Object, new DefaultMjolnirLogFactory());

            bucket.Increment(CounterMetric.CommandSuccess);

            clock.AddMilliseconds(periodMillis + 1);
            Assert.Equal(1, bucket.GetCount(CounterMetric.CommandSuccess));

            bucket.Increment(CounterMetric.CommandSuccess);                 // Should reset and then count one.
            Assert.Equal(1, bucket.GetCount(CounterMetric.CommandSuccess)); // Should be 1, not 2.
        }
        public void IsAllowing_WhenPropertiesForceFixedButBreakerWouldNormallyTrip_SilentlyTripsTheBreaker()
        {
            // Arrange

            // Most property values don't matter, IsAllowing() should reject before it tries to use them.
            var manualClock = new ManualTestClock();

            var mockMetrics = new Mock <ICommandMetrics>();

            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(2, 50));

            var mockEvents = new Mock <IMetricEvents>();

            var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>();

            mockConfig.Setup(m => m.GetMinimumOperations(It.IsAny <GroupKey>())).Returns(1);
            mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(25);
            mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(30000);
            mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(false);
            mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(true);

            // We'll call IsAllowing() three times. The first two should force the breaker fixed,
            // the third should un-force it. The breaker should have tripped by then, so the third
            // call should show the breaker disallowing the call.
            mockConfig.SetupSequence(m => m.GetForceFixed(It.IsAny <GroupKey>()))
            .Returns(true)
            .Returns(true)
            .Returns(false);

            var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());


            // Act / Assert

            Assert.True(breaker.IsAllowing()); // Will have tripped internally.
            Assert.True(breaker.IsAllowing()); // Continues to allow even when tripped.

            // Test that when the the forced fix property is no longer true, IsAllowing() then rejects.
            // The Mock sequencing enables this.

            Assert.False(breaker.IsAllowing());
        }
        public void MarkSuccess_ForLongRunningSingleTest_FixesBreaker()
        {
            // Since we use the elapsed time in MarkSuccess(), address the following situation:
            // 1. Breaker trips
            // 2. Window passes
            // 3. Single test is allowed
            // 4. Single test takes a long time (say, > 2 windows) to complete (even though it started after we tripped).
            // Verify that:
            // - No other requests are allowed while the single test is waiting.
            // - The single test, if successful, fixes the breaker.

            // This is somewhat unlikely since we probably wont have command timeouts
            // that'd allow this. But worth verifying.

            var clock       = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(2, 100); // 2 ops, 100% failing.
            var breaker     = new BreakerBuilder(1, 1)
                              .WithMockMetrics(mockMetrics)
                              .WithClock(clock)
                              .Create(); // 1 ops, 1% failure required to break.

            var duration = breaker.Properties.TrippedDurationMillis;

            Assert.False(breaker.IsAllowing()); // Should immediately trip.

            clock.AddMilliseconds(duration.Value + 10);

            Assert.True(breaker.IsAllowing()); // Single test is allowed.
            Assert.False(breaker.IsAllowing());

            clock.AddMilliseconds(duration.Value * 2);
            breaker.MarkSuccess(duration.Value * 2);

            // Metrics will have been reset.
            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));

            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
        }
예제 #22
0
        public void GetSnapshot_WithinCachePeriod_ReturnsPreviousSnapshot()
        {
            var clock = new ManualTestClock();

            // Within the metrics, the last snapshot timestamp will probably be zero.
            // Let's start our clock with something far away from zero.
            clock.AddMilliseconds(new SystemClock().GetMillisecondTimestamp());
            var metrics = new StandardCommandMetrics(
                GroupKey.Named("Test"),
                new TransientConfigurableValue <long>(10000),
                new TransientConfigurableValue <long>(1000),
                clock,
                new IgnoringStats());

            metrics.MarkCommandSuccess();
            metrics.GetSnapshot();       // Take the first snapshot to cache it.
            metrics.MarkCommandSuccess();
            clock.AddMilliseconds(500);  // Still within the snapshot TTL (1000).
            Assert.Equal(1, metrics.GetSnapshot().Total);
            clock.AddMilliseconds(1000); // Push time past the TTL.
            Assert.Equal(2, metrics.GetSnapshot().Total);
        }
        public void GetSnapshot_WithinCachePeriod_ReturnsPreviousSnapshot()
        {
            var clock = new ManualTestClock();

            // Within the metrics, the last snapshot timestamp will probably be zero.
            // Let's start our clock with something far away from zero.
            clock.AddMilliseconds(new SystemClock().GetMillisecondTimestamp());
            var metrics = new StandardCommandMetrics(
                GroupKey.Named("Test"),
                new TransientConfigurableValue<long>(10000),
                new TransientConfigurableValue<long>(1000),
                clock,
                new IgnoringStats());

            metrics.MarkCommandSuccess();
            metrics.GetSnapshot(); // Take the first snapshot to cache it.
            metrics.MarkCommandSuccess();
            clock.AddMilliseconds(500); // Still within the snapshot TTL (1000).
            Assert.Equal(1, metrics.GetSnapshot().Total);
            clock.AddMilliseconds(1000); // Push time past the TTL.
            Assert.Equal(2, metrics.GetSnapshot().Total);
        }
        public void MarkSuccess_ImmediatelyAfterTrippingButStartedBeforeTripped_DoesntImmediatelyFix()
        {
            // 1. Breaker is near tripping.
            // 2. Operation A and B are Allowed and begin work.
            // 3. Before Operation A completes
            //    a. Operation B has an error and updates metrics.
            //    b. Operation C calls IsAllowing(), which trips breaker.
            // 4. Operation A completes successfully and calls MarkSuccess().
            // 5. Since breaker is tripped but we haven't hit our wait duration yet,
            //    MarkSuccess() should result in the the breaker remaining tripped.

            var clock = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(2, 100); // 2 ops, 100% failing.
            var breaker = new BreakerBuilder(5, 1)
                .WithMockMetrics(mockMetrics)
                .WithClock(clock)
                .Create(); // 5 ops, 1% failure required to break.
            
            // #2. Operation A is allowed and begins.
            Assert.True(breaker.IsAllowing()); // Haven't hit the 1-operation threshold yet, should be allowed.
             
            // #3a. Operation B errors
            breaker.Properties.MinimumOperations.Value = 1; // Easier to test by changing the breaker conditions.

            // #3b. Breaker exceeds metrics thresholds, Operation C tries to IsAllowing() and trips breaker.
            Assert.False(breaker.IsAllowing());

            // #4. Operation A completes successfully.
            // Breaker's internal _lastTrippedTimestamp should be equal to zero (current clock time).
            // Since we say the transaction took 100ms, that'll be before the breaker tripped, and should
            // be ignored.
            breaker.MarkSuccess(100);

            // #5. Make sure we're still tripped and we didn't reset the metrics.
            Assert.False(breaker.IsAllowing());
            mockMetrics.Verify(m => m.Reset(), Times.Never);
        }
        public void MarkSuccess_ImmediatelyAfterTrippingButStartedBeforeTripped_DoesntImmediatelyFix()
        {
            // 1. Breaker is near tripping.
            // 2. Operation A and B are Allowed and begin work.
            // 3. Before Operation A completes
            //    a. Operation B has an error and updates metrics.
            //    b. Operation C calls IsAllowing(), which trips breaker.
            // 4. Operation A completes successfully and calls MarkSuccess().
            // 5. Since breaker is tripped but we haven't hit our wait duration yet,
            //    MarkSuccess() should result in the the breaker remaining tripped.

            var clock       = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(2, 100); // 2 ops, 100% failing.
            var breaker     = new BreakerBuilder(5, 1)
                              .WithMockMetrics(mockMetrics)
                              .WithClock(clock)
                              .Create(); // 5 ops, 1% failure required to break.

            // #2. Operation A is allowed and begins.
            Assert.True(breaker.IsAllowing()); // Haven't hit the 1-operation threshold yet, should be allowed.

            // #3a. Operation B errors
            breaker.Properties.MinimumOperations.Value = 1; // Easier to test by changing the breaker conditions.

            // #3b. Breaker exceeds metrics thresholds, Operation C tries to IsAllowing() and trips breaker.
            Assert.False(breaker.IsAllowing());

            // #4. Operation A completes successfully.
            // Breaker's internal _lastTrippedTimestamp should be equal to zero (current clock time).
            // Since we say the transaction took 100ms, that'll be before the breaker tripped, and should
            // be ignored.
            breaker.MarkSuccess(100);

            // #5. Make sure we're still tripped and we didn't reset the metrics.
            Assert.False(breaker.IsAllowing());
            mockMetrics.Verify(m => m.Reset(), Times.Never);
        }
        public void IsAllowing_MultipleDurationsBetweenFailureAndNextFailure_KeepsBreakerTripped()
        {
            const long durationMillis = 10000;

            var clock       = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
            var breaker     = new BreakerBuilder(1, 1)                // Trip at 1 op, 1% failing.
                              .WithMockMetrics(mockMetrics)
                              .WithWaitMillis(durationMillis)
                              .WithClock(clock)
                              .Create();

            Assert.False(breaker.IsAllowing()); // Trip the breaker.

            clock.AddMilliseconds(durationMillis * 5);

            Assert.True(breaker.IsAllowing());  // Single test, but it won't MarkSuccess().
            Assert.False(breaker.IsAllowing()); // Make sure the last one was the single test.

            clock.AddMilliseconds(durationMillis * 5);

            Assert.True(breaker.IsAllowing());  // Single test, but it won't MarkSuccess().
            Assert.False(breaker.IsAllowing()); // Make sure the last one was the single test.
        }
        public void GetSnapshot_WithinCachePeriod_ReturnsPreviousSnapshot()
        {
            var clock = new ManualTestClock();

            var key = AnyGroupKey;

            var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>(MockBehavior.Strict);

            mockConfig.Setup(m => m.GetWindowMillis(key)).Returns(10000);
            mockConfig.Setup(m => m.GetSnapshotTtlMillis(key)).Returns(1000);

            // Within the metrics, the last snapshot timestamp will probably be zero.
            // Let's start our clock with something far away from zero.
            clock.AddMilliseconds(new UtcSystemClock().GetMillisecondTimestamp());
            var metrics = new StandardCommandMetrics(key, mockConfig.Object, clock, new DefaultMjolnirLogFactory());

            metrics.MarkCommandSuccess();
            metrics.GetSnapshot();       // Take the first snapshot to cache it.
            metrics.MarkCommandSuccess();
            clock.AddMilliseconds(500);  // Still within the snapshot TTL (1000).
            Assert.Equal(1, metrics.GetSnapshot().Total);
            clock.AddMilliseconds(1000); // Push time past the TTL.
            Assert.Equal(2, metrics.GetSnapshot().Total);
        }
        public void IsAllowing_MultipleDurationsBetweenFailureAndNextSuccess_FixesBreakerOnSuccess()
        {
            const long durationMillis = 10000;

            var clock       = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
            var breaker     = new BreakerBuilder(1, 1)                // Trip at 1 op, 1% failing.
                              .WithMockMetrics(mockMetrics)
                              .WithWaitMillis(durationMillis)
                              .WithClock(clock)
                              .Create();

            Assert.False(breaker.IsAllowing()); // Trip the breaker.

            clock.AddMilliseconds(durationMillis * 5);

            Assert.True(breaker.IsAllowing()); // Single test.
            breaker.MarkSuccess(0);
            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));

            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
        }
        public void AllowSingleTest_TrippedAndPastWaitDuration()
        {
            var mockStats = new Mock<IStats>();
            var clock = new ManualTestClock();
            var breaker = new BreakerBuilder(0, 0, "Test")
                .WithStats(mockStats.Object)
                .WithClock(clock)
                .WithWaitMillis(1000)
                .Create();
            breaker.IsAllowing(); // Trip.
            mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "NotEligible", It.IsAny<TimeSpan>()), Times.Once);
            clock.AddMilliseconds(2000); // Advance past wait duration.

            breaker.IsAllowing();

            mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "Allowed", It.IsAny<TimeSpan>()), Times.Once);
        }
        public void IsAllowing_MultipleDurationsBetweenFailureAndNextSuccess_FixesBreakerOnSuccess()
        {
            const long durationMillis = 10000;

            var clock = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
            var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing.
                .WithMockMetrics(mockMetrics)
                .WithWaitMillis(durationMillis)
                .WithClock(clock)
                .Create();

            Assert.False(breaker.IsAllowing()); // Trip the breaker.

            clock.AddMilliseconds(durationMillis * 5);

            Assert.True(breaker.IsAllowing()); // Single test.
            breaker.MarkSuccess(0);
            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));

            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
        }
        private ResettingNumbersBucket CreateBucket()
        {
            var clock = new ManualTestClock();

            return(new ResettingNumbersBucket(clock, new TransientConfigurableValue <long>(10000)));
        }
        public void IsAllowing_MultipleDurationsBetweenFailureAndNextFailure_KeepsBreakerTripped()
        {
            const long durationMillis = 10000;

            var clock = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
            var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing.
                .WithMockMetrics(mockMetrics)
                .WithWaitMillis(durationMillis)
                .WithClock(clock)
                .Create();

            Assert.False(breaker.IsAllowing()); // Trip the breaker.

            clock.AddMilliseconds(durationMillis * 5);

            Assert.True(breaker.IsAllowing()); // Single test, but it won't MarkSuccess().
            Assert.False(breaker.IsAllowing()); // Make sure the last one was the single test.

            clock.AddMilliseconds(durationMillis * 5);

            Assert.True(breaker.IsAllowing()); // Single test, but it won't MarkSuccess().
            Assert.False(breaker.IsAllowing()); // Make sure the last one was the single test.
        }
        public void IsAllowing_AfterFailedSingleTest_KeepsBreakerTrippedAndSendsAnotherTestAfterAnotherDuration()
        {
            var clock = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
            var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing.
                .WithMockMetrics(mockMetrics)
                .WithWaitMillis(10000)
                .WithClock(clock)
                .Create();

            Assert.False(breaker.IsAllowing()); // Should reject and trip.

            clock.AddMilliseconds(11000);
            Assert.True(breaker.IsAllowing()); // Single test.

            // Advance to halfway through the duration. We should still be rejecting.
            clock.AddMilliseconds(6000);
            Assert.False(breaker.IsAllowing());

            // No mark success call here. Metrics will probably have a failure added to it.

            clock.AddMilliseconds(5000); // 6000 + 5000 = 11000 > 10000 (the duration).
            Assert.True(breaker.IsAllowing()); // It's been another duration, we should allow another single test.

            // Advance to halfway through the duration. We should still be rejecting.
            clock.AddMilliseconds(6000);
            Assert.False(breaker.IsAllowing());

            // No mark success call here. Metrics will probably have a failure added to it.

            clock.AddMilliseconds(5000); // 6000 + 5000 = 11000 > 10000 (the duration).
            Assert.True(breaker.IsAllowing()); // It's been another duration, we should allow another single test.

            // Advance a second or so, pretend the single test took some time.
            clock.AddMilliseconds(500);

            // Let's pretend this test succeeds. Mark it and reset the metrics.
            breaker.MarkSuccess(0);
            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));

            // We should immediately be opened.
            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
        }
 private ResettingNumbersBucket CreateBucket()
 {
     var clock = new ManualTestClock();
     return new ResettingNumbersBucket(clock, new TransientConfigurableValue<long>(10000));
 }
예제 #35
0
        public void ManualTestClock_GetMillisecondTimestamp_StartsAtZero()
        {
            var clock = new ManualTestClock();

            Assert.Equal(0, clock.GetMillisecondTimestamp());
        }
        public void MarkSuccess_ForLongRunningSingleTest_FixesBreaker()
        {
            // Since we use the elapsed time in MarkSuccess(), address the following situation:
            // 1. Breaker trips
            // 2. Window passes
            // 3. Single test is allowed
            // 4. Single test takes a long time (say, > 2 windows) to complete (even though it started after we tripped).
            // Verify that:
            // - No other requests are allowed while the single test is waiting.
            // - The single test, if successful, fixes the breaker.

            // This is somewhat unlikely since we probably wont have command timeouts
            // that'd allow this. But worth verifying.

            var clock = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(2, 100); // 2 ops, 100% failing.
            var breaker = new BreakerBuilder(1, 1)
                .WithMockMetrics(mockMetrics)
                .WithClock(clock)
                .Create(); // 1 ops, 1% failure required to break.

            var duration = breaker.Properties.TrippedDurationMillis;

            Assert.False(breaker.IsAllowing()); // Should immediately trip.

            clock.AddMilliseconds(duration.Value + 10);

            Assert.True(breaker.IsAllowing()); // Single test is allowed.
            Assert.False(breaker.IsAllowing());

            clock.AddMilliseconds(duration.Value * 2);
            breaker.MarkSuccess(duration.Value * 2);

            // Metrics will have been reset.
            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));

            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
        }
예제 #37
0
        public void ManualTestClock_AddMilliseconds_DisallowsNegatives()
        {
            var clock = new ManualTestClock();

            Assert.Throws <ArgumentException>(() => clock.AddMilliseconds(-10));
        }
        public void IsAllowing_AfterTrippedAndWithinWaitPeriod_Rejects()
        {
            var clock = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
            var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing.
                .WithMockMetrics(mockMetrics)
                .WithWaitMillis(10000)
                .WithClock(clock)
                .Create();

            Assert.False(breaker.IsAllowing()); // Should reject and trip.

            clock.AddMilliseconds(5000); // Half the wait duration.

            Assert.False(breaker.IsAllowing());
        }
        public void IsAllowing_AfterTrippedAndAfterWaitPeriod_SendsSingleTestAndRejectsOthers()
        {
            var clock = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
            var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing.
                .WithMockMetrics(mockMetrics)
                .WithWaitMillis(10000)
                .WithClock(clock)
                .Create();

            Assert.False(breaker.IsAllowing()); // Should reject and trip.

            clock.AddMilliseconds(11000);

            Assert.True(breaker.IsAllowing()); // Allow the first one, it's the single test.
            Assert.False(breaker.IsAllowing()); // Reject the next one, the test was already allowed.
        }
        public void AllowSingleTest_TrippedAndNotPastWaitDuration()
        {
            var mockStats = new Mock<IStats>();
            var clock = new ManualTestClock();
            var breaker = new BreakerBuilder(0, 0, "Test")
                .WithStats(mockStats.Object)
                .WithClock(clock)
                .WithWaitMillis(1000)
                .Create();
            breaker.IsAllowing(); // Trip.
            mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "NotEligible", It.IsAny<TimeSpan>()), Times.Once);
            // Don't advance the clock.

            breaker.IsAllowing();

            mockStats.Verify(m => m.Elapsed("mjolnir breaker Test AllowSingleTest", "NotEligible", It.IsAny<TimeSpan>()), Times.Exactly(2));
        }
        public void IsAllowing_AfterSuccessfulSingleTest_FixesBreaker()
        {
            var clock = new ManualTestClock();
            var mockMetrics = CreateMockMetricsWithSnapshot(10, 100); // 10 ops, 100% failing.
            var breaker = new BreakerBuilder(1, 1) // Trip at 1 op, 1% failing.
                .WithMockMetrics(mockMetrics)
                .WithWaitMillis(10000)
                .WithClock(clock)
                .Create();

            Assert.False(breaker.IsAllowing()); // Should reject and trip.
            
            clock.AddMilliseconds(11000);
            breaker.IsAllowing(); // Send the single test.

            clock.AddMilliseconds(500); // Pretend the test operation took a bit (not really necessary here).
            breaker.MarkSuccess(0); // The single test transaction marks a success.

            // Metrics should be reset.
            mockMetrics.Verify(m => m.Reset(), Times.Once);
            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));

            // Should be fixed now.
            // Test IsAllowing() twice to make sure it's not just the single test we're allowing.
            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
        }