public void MarkSuccess_ImmediatelyAfterTrippingButStartedBeforeTripped_DoesntImmediatelyFix()
        {
            // 1. Breaker is near tripping.
            // 2. Operation A and B are Allowed and begin work.
            // 3. Before Operation A completes
            //    a. Operation B has an error and updates metrics.
            //    b. Operation C calls IsAllowing(), which trips breaker.
            // 4. Operation A completes successfully and calls MarkSuccess().
            // 5. Since breaker is tripped but we haven't hit our wait duration yet,
            //    MarkSuccess() should result in the the breaker remaining tripped.


            // Arrange

            var manualClock = new ManualTestClock();

            var mockMetrics = new Mock <ICommandMetrics>();

            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(2, 100)); // 2 ops, 100% failing.

            var mockEvents = new Mock <IMetricEvents>();

            var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>();

            mockConfig.SetupSequence(m => m.GetMinimumOperations(It.IsAny <GroupKey>()))
            .Returns(5)     // First access should be > 1 so that the breaker doesn't trip.
            .Returns(1);    // Second access should be 1, so that we trip the breaker because we have the minimum met.

            mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(1);
            mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(30000);
            mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(false);
            mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(false);

            // 5 ops, 1% failure required to break.
            var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());


            // Act / Assert

            // #2. Operation A is allowed and begins.
            Assert.True(breaker.IsAllowing()); // Haven't hit the 1-operation threshold yet, should be allowed.

            // #3a. Operation B errors. This is easier to simulate by changing the breaker's trip
            //      conditions. The sequenced Mock (above) has the second MinimumOperations returning
            //      1, which would now mean we have enough operations to trip (where we didn't before).

            // #3b. Breaker exceeds metrics thresholds, Operation C tries to IsAllowing() and trips breaker.
            Assert.False(breaker.IsAllowing());

            // #4. Operation A completes successfully.
            // Breaker's internal _lastTrippedTimestamp should be equal to zero (current clock time).
            // Since we say the transaction took 100ms, that'll be before the breaker tripped, and should
            // be ignored.
            breaker.MarkSuccess(100);

            // #5. Make sure we're still tripped and we didn't reset the metrics.
            Assert.False(breaker.IsAllowing());
            mockMetrics.Verify(m => m.Reset(), Times.Never);
        }
        public void MarkSuccess_ForLongRunningSingleTest_FixesBreaker()
        {
            // Since we use the elapsed time in MarkSuccess(), address the following situation:
            // 1. Breaker trips
            // 2. Window passes
            // 3. Single test is allowed
            // 4. Single test takes a long time (say, > 2 windows) to complete (even though it started after we tripped).
            // Verify that:
            // - No other requests are allowed while the single test is waiting.
            // - The single test, if successful, fixes the breaker.

            // This is somewhat unlikely since we probably wont have command timeouts
            // that'd allow this. But worth verifying.


            // Arrange

            var manualClock = new ManualTestClock();

            var mockMetrics = new Mock <ICommandMetrics>();

            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(2, 100)); // 2 ops, 100% failing.

            var mockEvents = new Mock <IMetricEvents>();

            const long trippedDurationMillis = 30000;
            var        mockConfig            = new Mock <IFailurePercentageCircuitBreakerConfig>();

            mockConfig.Setup(m => m.GetMinimumOperations(It.IsAny <GroupKey>())).Returns(1);
            mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(1);
            mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(trippedDurationMillis);
            mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(false);
            mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(false);

            // 1 ops, 1% failure required to break.
            var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());


            // Act / Assert

            Assert.False(breaker.IsAllowing()); // Should immediately trip.

            manualClock.AddMilliseconds(trippedDurationMillis + 10);

            Assert.True(breaker.IsAllowing()); // Single test is allowed.
            Assert.False(breaker.IsAllowing());

            manualClock.AddMilliseconds(trippedDurationMillis * 2);
            breaker.MarkSuccess(trippedDurationMillis * 2);

            // Metrics will have been reset.
            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));

            Assert.True(breaker.IsAllowing());
            Assert.True(breaker.IsAllowing());
        }
        public void IsAllowing_WhenBothForcePropertiesSet_Rejects()
        {
            // Arrange

            // Most property values don't matter, IsAllowing() should reject before it tries to use them.
            var manualClock = new ManualTestClock();

            var mockMetrics = new Mock <ICommandMetrics>();

            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(0, 0));

            var mockEvents = new Mock <IMetricEvents>();

            var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>();

            mockConfig.Setup(m => m.GetMinimumOperations(It.IsAny <GroupKey>())).Returns(1);
            mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(1);
            mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(30000);
            mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(true); // The config we're testing here.
            mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(true);   // The config we're testing here.

            var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());


            // Act / Assert

            Assert.False(breaker.IsAllowing());
        }
            public void Run()
            {
                var mockMetrics = CreateMockMetricsWithSnapshot(_metricsTotal, _metricsPercent);
                var config      = CreateMockBreakerConfig(_breakerTotal, _breakerPercent, 30000);
                var breaker     = new FailurePercentageCircuitBreaker(GroupKey.Named("Test"), mockMetrics.Object, new IgnoringMetricEvents(), config.Object, new DefaultMjolnirLogFactory());

                Assert.NotEqual(_shouldTrip, breaker.IsAllowing());
            }
            public void Run()
            {
                var mockMetrics = CreateMockMetricsWithSnapshot(_metricsTotal, _metricsPercent);
                var properties  = CreateBreakerProperties(_breakerTotal, _breakerPercent, 30000);
                var breaker     = new FailurePercentageCircuitBreaker(GroupKey.Named("Test"), mockMetrics.Object, new IgnoringStats(), properties);

                Assert.NotEqual(_shouldTrip, breaker.IsAllowing());
            }
        public void IsAllowing_WhenPropertiesForceFixedButBreakerWouldNormallyTrip_SilentlyTripsTheBreaker()
        {
            // Arrange

            // Most property values don't matter, IsAllowing() should reject before it tries to use them.
            var manualClock = new ManualTestClock();

            var mockMetrics = new Mock <ICommandMetrics>();

            mockMetrics.Setup(m => m.GetSnapshot()).Returns(new MetricsSnapshot(2, 50));

            var mockEvents = new Mock <IMetricEvents>();

            var mockConfig = new Mock <IFailurePercentageCircuitBreakerConfig>();

            mockConfig.Setup(m => m.GetMinimumOperations(It.IsAny <GroupKey>())).Returns(1);
            mockConfig.Setup(m => m.GetThresholdPercentage(It.IsAny <GroupKey>())).Returns(25);
            mockConfig.Setup(m => m.GetTrippedDurationMillis(It.IsAny <GroupKey>())).Returns(30000);
            mockConfig.Setup(m => m.GetForceTripped(It.IsAny <GroupKey>())).Returns(false);
            mockConfig.Setup(m => m.GetForceFixed(It.IsAny <GroupKey>())).Returns(true);

            // We'll call IsAllowing() three times. The first two should force the breaker fixed,
            // the third should un-force it. The breaker should have tripped by then, so the third
            // call should show the breaker disallowing the call.
            mockConfig.SetupSequence(m => m.GetForceFixed(It.IsAny <GroupKey>()))
            .Returns(true)
            .Returns(true)
            .Returns(false);

            var breaker = new FailurePercentageCircuitBreaker(AnyKey, manualClock, mockMetrics.Object, mockEvents.Object, mockConfig.Object, new DefaultMjolnirLogFactory());


            // Act / Assert

            Assert.True(breaker.IsAllowing()); // Will have tripped internally.
            Assert.True(breaker.IsAllowing()); // Continues to allow even when tripped.

            // Test that when the the forced fix property is no longer true, IsAllowing() then rejects.
            // The Mock sequencing enables this.

            Assert.False(breaker.IsAllowing());
        }