public void testCreatesBuckets()
        {
            var time = new MockedClock();

            var counter = new RollingNumber(time, 200, 10);

            // confirm the initial settings
            Assert.Equal(200, counter.TimeInMs);
            Assert.Equal(10, counter.NumberOfBuckets);
            Assert.Equal(20, counter.BucketSizeInMs);


            // add a Success in each interval which should result in all 10 buckets being created with 1 Success in each
            for (int i = 0; i < counter.NumberOfBuckets; i++)
            {
                counter.Increment(RollingNumberEvent.SUCCESS);
                time.Increment(counter.BucketSizeInMs);
            }

            // confirm we have all 10 buckets
            var buckets = counter.GetBuckets().ToArray();

            Assert.Equal(10, buckets.Length);

            // add 1 more and we should still only have 10 buckets since that's the max
            counter.Increment(RollingNumberEvent.SUCCESS);
            buckets = counter.GetBuckets().ToArray();
            Assert.Equal(10, buckets.Length);
        }
        public void testCumulativeCounterAfterRolling()
        {
            MockedClock        time    = new MockedClock();
            RollingNumberEvent type    = RollingNumberEvent.SUCCESS;
            RollingNumber      counter = new RollingNumber(time, 20, 2);

            Assert.Equal(0, counter.GetCumulativeSum(type));

            // iterate over 20 buckets on a queue sized for 2
            for (int i = 0; i < 20; i++)
            {
                // first bucket
                counter.Increment(type);
                try
                {
                    time.Increment(counter.BucketSizeInMs);
                }
                catch (Exception)
                {
                    // ignore
                }

                counter.GetValueOfLatestBucket(type);
                Assert.Equal(2, counter.GetValues(type).Count());
            }

            // cumulative count should be 20 (for the number of loops above) regardless of buckets rolling
            Assert.Equal(20, counter.GetCumulativeSum(type));
        }
        public void testTripCircuit()
        {
            var             properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
            var             clock      = new MockedClock();
            CommandMetrics  metrics    = getMetrics(properties, clock);
            ICircuitBreaker cb         = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);

            metrics.MarkSuccess(1000);
            metrics.MarkSuccess(1000);
            metrics.MarkSuccess(1000);
            metrics.MarkSuccess(1000);

            // this should still allow requests as everything has been successful
            Assert.True(cb.AllowRequest);
            Assert.False(cb.IsOpen());


            // fail
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);

            // everything has failed in the test window so we should return false now
            Assert.False(cb.AllowRequest);
            Assert.True(cb.IsOpen());
        }
        public void testCumulativeCounterAfterRollingAndReset3()
        {
            MockedClock        time    = new MockedClock();
            RollingNumberEvent type    = RollingNumberEvent.SUCCESS;
            RollingNumber      counter = new RollingNumber(time, 20, 2);

            Assert.Equal(0, counter.GetCumulativeSum(type));

            counter.Increment(type);
            counter.Increment(type);
            counter.Increment(type);

            // iterate over 20 buckets on a queue sized for 2
            for (int i = 0; i < 20; i++)
            {
                try
                {
                    time.Increment(counter.BucketSizeInMs);
                }
                catch (Exception)
                {
                    // ignore
                }
            }

            // since we are rolling over the buckets it should reset naturally

            // no increments during the loop, just some before and after
            counter.Increment(type);
            counter.Increment(type);

            // cumulative count should be 5 regardless of buckets rolling
            Assert.Equal(5, counter.GetCumulativeSum(type));
        }
        public void testCircuitDoesNotTripOnFailuresBelowThreshold()
        {
            var             properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
            var             clock      = new MockedClock();
            CommandMetrics  metrics    = getMetrics(properties, clock);
            ICircuitBreaker cb         = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);

            // this should start as allowing requests
            Assert.True(cb.AllowRequest);
            Assert.False(cb.IsOpen());

            // success with high latency
            metrics.MarkSuccess(400);
            metrics.MarkSuccess(400);
            metrics.MarkFailure(10);
            metrics.MarkSuccess(400);
            metrics.MarkSuccess(40);
            metrics.MarkSuccess(400);
            metrics.MarkFailure(10);
            metrics.MarkFailure(10);

            // this should remain open as the failure threshold is below the percentage limit
            Assert.True(cb.AllowRequest);
            Assert.False(cb.IsOpen());
        }
        public void testRolling()
        {
            MockedClock        time    = new MockedClock();
            RollingNumberEvent type    = RollingNumberEvent.THREAD_MAX_ACTIVE;
            RollingNumber      counter = new RollingNumber(time, 20, 2);

            // iterate over 20 buckets on a queue sized for 2
            for (int i = 0; i < 20; i++)
            {
                // first bucket
                counter.GetCurrentBucket();
                try
                {
                    time.Increment(counter.BucketSizeInMs);
                }
                catch (Exception)
                {
                    // ignore
                }

                counter.GetValueOfLatestBucket(type);
                Assert.Equal(2, counter.GetValues(type).Count());

                // System.out.println("Head: " + counter.buckets.state.get().head);
                // System.out.println("Tail: " + counter.buckets.state.get().tail);
            }
        }
        public void testCreatesBuckets()
        {
            var time = new MockedClock();

            var counter = new RollingNumber(time, 200, 10);
            // confirm the initial settings
            Assert.Equal(200, counter.TimeInMs);
            Assert.Equal(10, counter.NumberOfBuckets);
            Assert.Equal(20, counter.BucketSizeInMs);


            // add a Success in each interval which should result in all 10 buckets being created with 1 Success in each
            for (int i = 0; i < counter.NumberOfBuckets; i++)
            {
                counter.Increment(RollingNumberEvent.SUCCESS);
                time.Increment(counter.BucketSizeInMs);
            }

            // confirm we have all 10 buckets
            var buckets = counter.GetBuckets().ToArray();
            Assert.Equal(10, buckets.Length);

            // add 1 more and we should still only have 10 buckets since that's the max
            counter.Increment(RollingNumberEvent.SUCCESS);
            buckets = counter.GetBuckets().ToArray();
            Assert.Equal(10, buckets.Length);
        }
        public void testTripCircuitOnTimeoutsAboveThreshold()
        {
            var             properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
            var             clock      = new MockedClock();
            CommandMetrics  metrics    = getMetrics(properties, clock);
            ICircuitBreaker cb         = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);

            // this should start as allowing requests
            Assert.True(cb.AllowRequest);
            Assert.False(cb.IsOpen());

            // success with high latency
            metrics.MarkSuccess(400);
            metrics.MarkSuccess(400);
            metrics.MarkTimeout(10);
            metrics.MarkSuccess(400);
            metrics.MarkTimeout(10);
            metrics.MarkTimeout(10);
            metrics.MarkSuccess(400);
            metrics.MarkTimeout(10);
            metrics.MarkTimeout(10);

            // this should trip the circuit as the error percentage is above the threshold
            Assert.False(cb.AllowRequest);
            Assert.True(cb.IsOpen());
        }
        public void testMaxValue()
        {
            MockedClock time = new MockedClock();

            RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;

            RollingNumber counter = new RollingNumber(time, 200, 10);

            counter.UpdateRollingMax(type, 10);

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs);

            counter.UpdateRollingMax(type, 30);

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs);

            counter.UpdateRollingMax(type, 40);

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs);

            counter.UpdateRollingMax(type, 15);

            Assert.Equal(40, counter.GetRollingMaxValue(type));
        }
        public void testSingleTestOnOpenCircuitAfterTimeWindow()
        {
            int             sleepWindow = 200;
            var             properties  = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
            var             clock       = new MockedClock();
            CommandMetrics  metrics     = getMetrics(properties, clock);
            ICircuitBreaker cb          = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);

            // fail
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);

            // everything has failed in the test window so we should return false now
            Assert.False(cb.AllowRequest);
            Assert.True(cb.IsOpen());

            // wait for sleepWindow to pass
            clock.Increment(sleepWindow + 50);

            // we should now allow 1 request
            Assert.True(cb.AllowRequest);
            // but the circuit should still be open
            Assert.True(cb.IsOpen());
            // and further requests are still blocked
            Assert.False(cb.AllowRequest);
        }
        public void testTimeout()
        {
            MockedClock time = new MockedClock();

            RollingNumber counter = new RollingNumber(time, 200, 10);

            // increment
            counter.Increment(RollingNumberEvent.TIMEOUT);

            var buckets = counter.GetBuckets().ToArray();

            // we should have 1 bucket
            Assert.Equal(1, buckets.Count());

            // the count should be 1
            Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
            Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs * 3);

            // incremenet again in latest bucket
            counter.Increment(RollingNumberEvent.TIMEOUT);

            // we should have 2 buckets
            buckets = counter.GetBuckets().ToArray();
            Assert.Equal(2, buckets.Length);

            // the counts of the last bucket
            Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));

            // the total counts
            Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
        }
        public void testIncrementInSingleBucket()
        {
            MockedClock time = new MockedClock();

            RollingNumber counter = new RollingNumber(time, 200, 10);

            // increment
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.TIMEOUT);

            // we should have 1 bucket
            var buckets = counter.GetBuckets().ToArray();

            Assert.Equal(1, buckets.Length);

            // the count should be 4
            Assert.Equal(4, counter.GetCurrentBucket().GetAdder(RollingNumberEvent.SUCCESS));
            Assert.Equal(2, counter.GetCurrentBucket().GetAdder(RollingNumberEvent.FAILURE));
            Assert.Equal(1, counter.GetCurrentBucket().GetAdder(RollingNumberEvent.TIMEOUT));
        }
        public void testEmptyLatestValue()
        {
            MockedClock        time    = new MockedClock();
            RollingNumberEvent type    = RollingNumberEvent.THREAD_MAX_ACTIVE;
            RollingNumber      counter = new RollingNumber(time, 200, 10);

            Assert.Equal(0, counter.GetValueOfLatestBucket(type));
        }
        public void testEmptySum()
        {
            MockedClock        time    = new MockedClock();
            RollingNumberEvent type    = RollingNumberEvent.SUCCESS;
            RollingNumber      counter = new RollingNumber(time, 200, 10);

            Assert.Equal(0, counter.GetRollingSum(type));
        }
        public void testIncrementInMultipleBuckets()
        {
            MockedClock time = new MockedClock();

            RollingNumber counter = new RollingNumber(time, 200, 10);

            // increment
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.TIMEOUT);
            counter.Increment(RollingNumberEvent.TIMEOUT);
            counter.Increment(RollingNumberEvent.SHORT_CIRCUITED);

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs * 3);

            // increment
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.TIMEOUT);
            counter.Increment(RollingNumberEvent.SHORT_CIRCUITED);

            // we should have 2 buckets
            var buckets = counter.GetBuckets().ToArray();

            Assert.Equal(2, buckets.Length);

            // the counts of the last bucket
            Assert.Equal(2, buckets.First().GetAdder(RollingNumberEvent.SUCCESS));
            Assert.Equal(3, buckets.First().GetAdder(RollingNumberEvent.FAILURE));
            Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
            Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.SHORT_CIRCUITED));

            // the total counts
            Assert.Equal(6, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
            Assert.Equal(5, counter.GetRollingSum(RollingNumberEvent.FAILURE));
            Assert.Equal(3, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
            Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.SHORT_CIRCUITED));

            // wait until window passes
            time.Increment(counter.TimeInMs * 10);

            // increment
            counter.Increment(RollingNumberEvent.SUCCESS);

            // the total counts should now include only the last bucket after a reset since the window passed
            Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
            Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
            Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
        }
        public void testCounterRetrievalRefreshesBuckets()
        {
            MockedClock time = new MockedClock();

            RollingNumber counter = new RollingNumber(time, 200, 10);

            // increment
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.FAILURE);

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs * 3);

            var buckets = counter.GetBuckets().ToArray();

            // we should have 1 bucket since nothing has triggered the update of buckets in the elapsed time
            Assert.Equal(1, buckets.Length);

            // the total counts
            Assert.Equal(4, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
            Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.FAILURE));

            // we should have 2 buckets as the counter 'gets' should have triggered the buckets being created to fill in time
            buckets = counter.GetBuckets().ToArray();
            Assert.Equal(2, buckets.Length);

            // wait until window passes
            time.Increment(counter.TimeInMs);

            // the total counts should all be 0 (and the buckets cleared by the get, not only increment)
            Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
            Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));

            // increment
            counter.Increment(RollingNumberEvent.SUCCESS);

            // the total counts should now include only the last bucket after a reset since the window passed
            Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
            Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
        }
        public void testCircuitClosedAfterSuccessAndClearsStatisticalWindow()
        {
            int statisticalWindow = 200;
            int sleepWindow       = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow
            var properties        = CommandPropertiesTest.GetUnitTestPropertiesSetter()
                                    .WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow)
                                    .WithMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow);
            var             clock   = new MockedClock();
            CommandMetrics  metrics = getMetrics(properties, clock);
            ICircuitBreaker cb      = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);

            // fail
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);

            // everything has failed in the test window so we should return false now
            Assert.False(cb.AllowRequest);
            Assert.True(cb.IsOpen());

            // wait for sleepWindow to pass
            clock.Increment(sleepWindow + 50);

            // we should now allow 1 request
            Assert.True(cb.AllowRequest);
            // but the circuit should still be open
            Assert.True(cb.IsOpen());
            // and further requests are still blocked
            Assert.False(cb.AllowRequest);

            // the 'singleTest' succeeds so should cause the circuit to be closed
            metrics.MarkSuccess(500);
            cb.MarkSuccess();

            // all requests should be open again
            Assert.True(cb.AllowRequest);
            Assert.True(cb.AllowRequest);
            Assert.True(cb.AllowRequest);
            // and the circuit should be closed again
            Assert.False(cb.IsOpen());
        }
        public void testLowVolumeDoesNotTripCircuit()
        {
            int sleepWindow = 200;
            int lowVolume   = 5;

            var             properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithCircuitBreakerRequestVolumeThreshold(lowVolume);
            var             clock      = new MockedClock();
            CommandMetrics  metrics    = getMetrics(properties, clock);
            ICircuitBreaker cb         = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);

            // fail
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);

            // even though it has all failed we won't trip the circuit because the volume is low
            Assert.True(cb.AllowRequest);
            Assert.False(cb.IsOpen());
        }
Beispiel #19
0
        public void testBadRequestsDoNotAffectErrorPercentage()
        {
            var            properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
            var            clock      = new MockedClock();
            CommandMetrics metrics    = getMetrics(properties, clock);;

            metrics.MarkSuccess(100);
            Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage);

            metrics.MarkFailure(1000);
            Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);

            metrics.MarkBadRequest(1);
            metrics.MarkBadRequest(2);
            Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);

            metrics.MarkFailure(45);
            metrics.MarkFailure(55);
            Assert.Equal(75, metrics.GetHealthCounts().ErrorPercentage);
        }
        public void testUpdateMax2()
        {
            MockedClock time = new MockedClock();

            RollingNumber counter = new RollingNumber(time, 200, 10);

            // increment
            counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 10);
            counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
            counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 20);

            // we should have 1 bucket
            var buckets = counter.GetBuckets().ToArray();

            Assert.Equal(1, buckets.Length);

            // the count should be 30
            Assert.Equal(30, counter.buckets.First().GetMaxUpdater(RollingNumberEvent.THREAD_MAX_ACTIVE));
            Assert.Equal(30, counter.GetRollingMaxValue(RollingNumberEvent.THREAD_MAX_ACTIVE));

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs * 3);

            counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
            counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
            counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 50);

            // we should have 2 buckets
            buckets = counter.GetBuckets().ToArray();
            Assert.Equal(2, buckets.Length);

            // the count
            Assert.Equal(50, buckets.First().GetMaxUpdater(RollingNumberEvent.THREAD_MAX_ACTIVE));
            Assert.Equal(50, counter.GetValueOfLatestBucket(RollingNumberEvent.THREAD_MAX_ACTIVE));

            // values per bucket
            var values = counter.GetValues(RollingNumberEvent.THREAD_MAX_ACTIVE).ToArray();

            Assert.Equal(30, values[1]); // oldest bucket
            Assert.Equal(50, values[0]); // latest bucket
        }
        public void testTripCircuitOnTimeouts()
        {
            var             properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
            var             clock      = new MockedClock();
            CommandMetrics  metrics    = getMetrics(properties, clock);
            ICircuitBreaker cb         = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);

            // this should start as allowing requests
            Assert.True(cb.AllowRequest);
            Assert.False(cb.IsOpen());

            // timeouts
            metrics.MarkTimeout(2000);
            metrics.MarkTimeout(2000);
            metrics.MarkTimeout(2000);
            metrics.MarkTimeout(2000);

            // everything has been a timeout so we should not allow any requests
            Assert.False(cb.AllowRequest);
            Assert.True(cb.IsOpen());
        }
        public void testResetBuckets()
        {
            MockedClock time = new MockedClock();
            RollingNumber counter = new RollingNumber(time, 200, 10);

  
            // add 1
            counter.Increment(RollingNumberEvent.SUCCESS);

            // confirm we have 1 bucket
            var buckets = counter.GetBuckets().ToArray();
            Assert.Equal(1, buckets.Length);

            // confirm we still have 1 bucket
            Assert.Equal(1, buckets.Length);

            // add 1
            counter.Increment(RollingNumberEvent.SUCCESS);

            // we should now have a single bucket with no values in it instead of 2 or more buckets
            buckets = counter.GetBuckets().ToArray();
            Assert.Equal(1, buckets.Length);
        }
        public void testResetBuckets()
        {
            MockedClock   time    = new MockedClock();
            RollingNumber counter = new RollingNumber(time, 200, 10);


            // add 1
            counter.Increment(RollingNumberEvent.SUCCESS);

            // confirm we have 1 bucket
            var buckets = counter.GetBuckets().ToArray();

            Assert.Equal(1, buckets.Length);

            // confirm we still have 1 bucket
            Assert.Equal(1, buckets.Length);

            // add 1
            counter.Increment(RollingNumberEvent.SUCCESS);

            // we should now have a single bucket with no values in it instead of 2 or more buckets
            buckets = counter.GetBuckets().ToArray();
            Assert.Equal(1, buckets.Length);
        }
Beispiel #24
0
        public void testErrorPercentage()
        {
            var            properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
            var            clock      = new MockedClock();
            CommandMetrics metrics    = getMetrics(properties, clock);

            metrics.MarkSuccess(100);
            Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage);
            clock.Increment(1);
            metrics.MarkFailure(1000);
            Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);

            clock.Increment(1);
            metrics.MarkSuccess(100);
            metrics.MarkSuccess(100);
            Assert.Equal(25, metrics.GetHealthCounts().ErrorPercentage);

            clock.Increment(1);
            metrics.MarkTimeout(5000);
            metrics.MarkTimeout(5000);
            Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);

            clock.Increment(1);
            metrics.MarkSuccess(100);
            metrics.MarkSuccess(100);
            metrics.MarkSuccess(100);

            // latent
            clock.Increment(1);
            metrics.MarkSuccess(5000);

            // 6 success + 1 latent success + 1 failure + 2 timeout = 10 total
            // latent success not considered error
            // error percentage = 1 failure + 2 timeout / 10
            Assert.Equal(30, metrics.GetHealthCounts().ErrorPercentage);
        }
        public void testIncrementInSingleBucket()
        {
            MockedClock time = new MockedClock();

            RollingNumber counter = new RollingNumber(time, 200, 10);

            // increment
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.TIMEOUT);

            // we should have 1 bucket
            var buckets = counter.GetBuckets().ToArray();
            Assert.Equal(1, buckets.Length);

            // the count should be 4
            Assert.Equal(4, counter.GetCurrentBucket().GetAdder(RollingNumberEvent.SUCCESS));
            Assert.Equal(2, counter.GetCurrentBucket().GetAdder(RollingNumberEvent.FAILURE));
            Assert.Equal(1, counter.GetCurrentBucket().GetAdder(RollingNumberEvent.TIMEOUT));
        }
        public void testSampleDataOverTime2()
        {
            //Console.WriteLine("\n\n***************************** testSampleDataOverTime2 \n");
            MockedClock time = new MockedClock();
            int previousTime = 0;
            RollingPercentileNumber p = new RollingPercentileNumber(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, DynamicProperties.Factory.AsProperty(true));
            for (int i = 0; i < SampleDataHolder2.data.Length/2; i++)
            {
                int timeInMillisecondsSinceStart = SampleDataHolder2.data[i,0];
                int latency = SampleDataHolder2.data[i,1];
                time.Increment(timeInMillisecondsSinceStart - previousTime);
                previousTime = timeInMillisecondsSinceStart;
                p.AddValue(latency);
            }

            //Console.WriteLine("0.01: " + p.GetPercentile(0.01));
            //Console.WriteLine("Median: " + p.GetPercentile(50));
            //Console.WriteLine("90th: " + p.GetPercentile(90));
            //Console.WriteLine("99th: " + p.GetPercentile(99));
            //Console.WriteLine("99.5th: " + p.GetPercentile(99.5));
            //Console.WriteLine("99.99: " + p.GetPercentile(99.99));

            if (p.GetPercentile(50) > 90 || p.GetPercentile(50) < 50)
            {
                throw new Exception("We expect around 60-70 but got: " + p.GetPercentile(50));
            }

            if (p.GetPercentile(99) < 400)
            {
                throw new Exception("We expect to see some high values over 400 but got: " + p.GetPercentile(99));
            }
        }
        public void testSampleDataOverTime1()
        {
            //Console.WriteLine("\n\n***************************** testSampleDataOverTime1 \n");

            MockedClock time = new MockedClock();
            RollingPercentileNumber p = new RollingPercentileNumber(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, DynamicProperties.Factory.AsProperty(true));
            int previousTime = 0;
            for (int i = 0; i < SampleDataHolder1.data.Length/2; i++)
            {
                int timeInMillisecondsSinceStart = SampleDataHolder1.data[i,0];
                int latency = SampleDataHolder1.data[i,1];
                time.Increment(timeInMillisecondsSinceStart - previousTime);
                previousTime = timeInMillisecondsSinceStart;
                p.AddValue(latency);
            }

            //Console.WriteLine("0.01: " + p.GetPercentile(0.01));
            //Console.WriteLine("Median: " + p.GetPercentile(50));
            //Console.WriteLine("90th: " + p.GetPercentile(90));
            //Console.WriteLine("99th: " + p.GetPercentile(99));
            //Console.WriteLine("99.5th: " + p.GetPercentile(99.5));
            //Console.WriteLine("99.99: " + p.GetPercentile(99.99));

            //Console.WriteLine("Median: " + p.GetPercentile(50));
            //Console.WriteLine("Median: " + p.GetPercentile(50));
            //Console.WriteLine("Median: " + p.GetPercentile(50));

            /*
             * In a loop as a use case was found where very different values were calculated in subsequent requests.
             */
            for (int i = 0; i < 10; i++)
            {
                if (p.GetPercentile(50) > 5)
                {
                    throw new Exception("We expect around 2 but got: " + p.GetPercentile(50));
                }

                if (p.GetPercentile(99.5) < 20)
                {
                    throw new Exception("We expect to see some high values over 20 but got: " + p.GetPercentile(99.5));
                }
            }
        }
        public void testValueIsZeroAfterRollingWindowPassesAndNoTraffic()
        {
            MockedClock time = new MockedClock();
            var p = new RollingPercentileNumber(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, DynamicProperties.Factory.AsProperty(true));
            p.AddValue(1000);
            p.AddValue(1000);
            p.AddValue(1000);
            p.AddValue(2000);
            p.AddValue(4000);

            Assert.Equal(1, p.GetBuckets().Count());

            // no bucket turnover yet so percentile not yet generated
            Assert.Equal(0, p.GetPercentile(50));

            time.Increment(6000);

            // still only 1 bucket until we touch it again
            Assert.Equal(1, p.GetBuckets().Count());

            // a bucket has been created so we have a new percentile
            Assert.Equal(1500, p.GetPercentile(50));

            // let 1 minute pass
            time.Increment(60000);

            // no data in a minute should mean all buckets are empty (or reset) so we should not have any percentiles
            Assert.Equal(0, p.GetPercentile(50));
        }
        public void testTimeout()
        {
            MockedClock time = new MockedClock();

            RollingNumber counter = new RollingNumber(time, 200, 10);

            // increment
            counter.Increment(RollingNumberEvent.TIMEOUT);

            var buckets = counter.GetBuckets().ToArray();
            // we should have 1 bucket
            Assert.Equal(1, buckets.Count());

            // the count should be 1
            Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
            Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs * 3);

            // incremenet again in latest bucket
            counter.Increment(RollingNumberEvent.TIMEOUT);

            // we should have 2 buckets
            buckets = counter.GetBuckets().ToArray();
            Assert.Equal(2, buckets.Length);

            // the counts of the last bucket
            Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));

            // the total counts
            Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
        }
        public void testCounterRetrievalRefreshesBuckets()
        {
            MockedClock time = new MockedClock();

            RollingNumber counter = new RollingNumber(time, 200, 10);

            // increment
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.FAILURE);

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs * 3);

            var buckets = counter.GetBuckets().ToArray();
            // we should have 1 bucket since nothing has triggered the update of buckets in the elapsed time
            Assert.Equal(1, buckets.Length);

            // the total counts
            Assert.Equal(4, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
            Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.FAILURE));

            // we should have 2 buckets as the counter 'gets' should have triggered the buckets being created to fill in time
            buckets = counter.GetBuckets().ToArray();
            Assert.Equal(2, buckets.Length);

            // wait until window passes
            time.Increment(counter.TimeInMs);

            // the total counts should all be 0 (and the buckets cleared by the get, not only increment)
            Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
            Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));

            // increment
            counter.Increment(RollingNumberEvent.SUCCESS);

            // the total counts should now include only the last bucket after a reset since the window passed
            Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
            Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
        }
        public void testRolling()
        {
            MockedClock time = new MockedClock();
            RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;
            RollingNumber counter = new RollingNumber(time, 20, 2);
            // iterate over 20 buckets on a queue sized for 2
            for (int i = 0; i < 20; i++)
            {
                // first bucket
                counter.GetCurrentBucket();
                try
                {
                    time.Increment(counter.BucketSizeInMs);
                }
                catch (Exception)
                {
                    // ignore
                }

                counter.GetValueOfLatestBucket(type);
                Assert.Equal(2, counter.GetValues(type).Count());

                // System.out.println("Head: " + counter.buckets.state.get().head);
                // System.out.println("Tail: " + counter.buckets.state.get().tail);
            }
        }
        public void testRolling()
        {
            MockedClock time = new MockedClock();
            RollingPercentileNumber p = new RollingPercentileNumber(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, DynamicProperties.Factory.AsProperty(true));
            p.AddValue(1000);
            p.AddValue(1000);
            p.AddValue(1000);
            p.AddValue(2000);

            Assert.Equal(1, p.GetBuckets().Count());

            // no bucket turnover yet so percentile not yet generated
            Assert.Equal(0, p.GetPercentile(50));

            time.Increment(6000);

            // still only 1 bucket until we touch it again
            Assert.Equal(1, p.GetBuckets().Count());

            // a bucket has been created so we have a new percentile
            Assert.Equal(1000, p.GetPercentile(50));

            // now 2 buckets since getting a percentile causes bucket retrieval
            Assert.Equal(2, p.GetBuckets().Count());

            p.AddValue(1000);
            p.AddValue(500);

            // should still be 2 buckets
            Assert.Equal(2, p.GetBuckets().Count());

            p.AddValue(200);
            p.AddValue(200);
            p.AddValue(1600);
            p.AddValue(200);
            p.AddValue(1600);
            p.AddValue(1600);

            // we haven't progressed to a new bucket so the percentile should be the same and ignore the most recent bucket
            Assert.Equal(1000, p.GetPercentile(50));

            // Increment to another bucket so we include all of the above in the PercentileSnapshot
            time.Increment(6000);

            // the rolling version should have the same data as creating a snapshot like this
            PercentileSnapshot ps = new PercentileSnapshot(1000, 1000, 1000, 2000, 1000, 500, 200, 200, 1600, 200, 1600, 1600);

            Assert.Equal(ps.GetPercentile(0.15), p.GetPercentile(0.15));
            Assert.Equal(ps.GetPercentile(0.50), p.GetPercentile(0.50));
            Assert.Equal(ps.GetPercentile(0.90), p.GetPercentile(0.90));
            Assert.Equal(ps.GetPercentile(0.995), p.GetPercentile(0.995));

            //Console.WriteLine("100th: " + ps.GetPercentile(100) + "  " + p.GetPercentile(100));
            //Console.WriteLine("99.5th: " + ps.GetPercentile(99.5) + "  " + p.GetPercentile(99.5));
            //Console.WriteLine("99th: " + ps.GetPercentile(99) + "  " + p.GetPercentile(99));
            //Console.WriteLine("90th: " + ps.GetPercentile(90) + "  " + p.GetPercentile(90));
            //Console.WriteLine("50th: " + ps.GetPercentile(50) + "  " + p.GetPercentile(50));
            //Console.WriteLine("10th: " + ps.GetPercentile(10) + "  " + p.GetPercentile(10));

            // mean = 1000+1000+1000+2000+1000+500+200+200+1600+200+1600+1600/12
            Assert.Equal(991, ps.Mean);
        }
 public void testEmptySum()
 {
     MockedClock time = new MockedClock();
     RollingNumberEvent type = RollingNumberEvent.SUCCESS;
     RollingNumber counter = new RollingNumber(time, 200, 10);
     Assert.Equal(0, counter.GetRollingSum(type));
 }
 public void testEmptyLatestValue()
 {
     MockedClock time = new MockedClock();
     RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;
     RollingNumber counter = new RollingNumber(time, 200, 10);
     Assert.Equal(0, counter.GetValueOfLatestBucket(type));
 }
        public void testMultipleTimeWindowRetriesBeforeClosingCircuit()
        {
            int             sleepWindow = 200;
            var             properties  = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
            var             clock       = new MockedClock();
            CommandMetrics  metrics     = getMetrics(properties, clock);
            ICircuitBreaker cb          = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);

            // fail
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);
            metrics.MarkFailure(1000);

            // everything has failed in the test window so we should return false now
            Assert.False(cb.AllowRequest);
            Assert.True(cb.IsOpen());

            // wait for sleepWindow to pass
            clock.Increment(sleepWindow + 50);

            // we should now allow 1 request
            Assert.True(cb.AllowRequest);
            // but the circuit should still be open
            Assert.True(cb.IsOpen());
            // and further requests are still blocked
            Assert.False(cb.AllowRequest);

            // the 'singleTest' fails so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep
            metrics.MarkFailure(1000);

            Assert.False(cb.AllowRequest);
            Assert.False(cb.AllowRequest);
            Assert.False(cb.AllowRequest);

            // wait for sleepWindow to pass
            clock.Increment(sleepWindow + 50);

            // we should now allow 1 request
            Assert.True(cb.AllowRequest);
            // but the circuit should still be open
            Assert.True(cb.IsOpen());
            // and further requests are still blocked
            Assert.False(cb.AllowRequest);

            // the 'singleTest' fails again so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep
            metrics.MarkFailure(1000);

            Assert.False(cb.AllowRequest);
            Assert.False(cb.AllowRequest);
            Assert.False(cb.AllowRequest);

            // wait for sleepWindow to pass
            clock.Increment(sleepWindow + 50);

            // we should now allow 1 request
            Assert.True(cb.AllowRequest);
            // but the circuit should still be open
            Assert.True(cb.IsOpen());
            // and further requests are still blocked
            Assert.False(cb.AllowRequest);

            // now it finally succeeds
            metrics.MarkSuccess(200);
            cb.MarkSuccess();

            // all requests should be open again
            Assert.True(cb.AllowRequest);
            Assert.True(cb.AllowRequest);
            Assert.True(cb.AllowRequest);
            // and the circuit should be closed again
            Assert.False(cb.IsOpen());
        }
        public void testMaxValue()
        {
            MockedClock time = new MockedClock();

            RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;

            RollingNumber counter = new RollingNumber(time, 200, 10);

            counter.UpdateRollingMax(type, 10);

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs);

            counter.UpdateRollingMax(type, 30);

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs);

            counter.UpdateRollingMax(type, 40);

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs);

            counter.UpdateRollingMax(type, 15);

            Assert.Equal(40, counter.GetRollingMaxValue(type));

        }
        public void testUpdateMax2()
        {
            MockedClock time = new MockedClock();

            RollingNumber counter = new RollingNumber(time, 200, 10);

            // increment
            counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 10);
            counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
            counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 20);

            // we should have 1 bucket
            var buckets = counter.GetBuckets().ToArray();
            Assert.Equal(1, buckets.Length);

            // the count should be 30
            Assert.Equal(30, counter.buckets.First().GetMaxUpdater(RollingNumberEvent.THREAD_MAX_ACTIVE));
            Assert.Equal(30, counter.GetRollingMaxValue(RollingNumberEvent.THREAD_MAX_ACTIVE));

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs * 3);

            counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
            counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
            counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 50);

            // we should have 2 buckets
            buckets = counter.GetBuckets().ToArray();
            Assert.Equal(2, buckets.Length);
        
            // the count
            Assert.Equal(50, buckets.First().GetMaxUpdater(RollingNumberEvent.THREAD_MAX_ACTIVE));
            Assert.Equal(50, counter.GetValueOfLatestBucket(RollingNumberEvent.THREAD_MAX_ACTIVE));

            // values per bucket
            var values = counter.GetValues(RollingNumberEvent.THREAD_MAX_ACTIVE).ToArray();
            Assert.Equal(30, values[1]); // oldest bucket
            Assert.Equal(50, values[0]); // latest bucket
        }
        public void testThreadSafety()
        {
            MockedClock time = new MockedClock();
            RollingPercentileNumber p = new RollingPercentileNumber(time, 100, 25, 1000, DynamicProperties.Factory.AsProperty(true));

            int NUM_THREADS = 1000;
            int NUM_ITERATIONS = 1000000;

            var tasks = new Task [NUM_THREADS];

            long aggregateMetrics = 0; //same as a blackhole

            Random r = new Random();
            var token = new CancellationTokenSource();
            var metricsPoller = Task.Run(() =>
            {

                while (!token.IsCancellationRequested)
                {
                    Interlocked.Add(ref aggregateMetrics, p.Mean + p.GetPercentile(10) + p.GetPercentile(50) + p.GetPercentile(90));
                    //Console.WriteLine("AGGREGATE : " + p.GetPercentile(10) + " : " + p.GetPercentile(50) + " : " + p.GetPercentile(90));
                }
            });

            for (int i = 0; i < NUM_THREADS; i++)
            {
                int threadId = i;
                tasks[i] = Task.Run(() =>
                {

                    for (int j = 1; j < NUM_ITERATIONS / NUM_THREADS + 1; j++)
                    {
                        int nextInt = r.Next(100);
                        p.AddValue(nextInt);
                        if (threadId == 0)
                        {
                            time.Increment(1);
                        }
                    }
                });
            }

            if( !Task.WaitAll(tasks, 1000))
                throw new Exception("Timeout on all threads writing percentiles");
            token.Cancel();

            Interlocked.Add(ref aggregateMetrics, p.Mean + p.GetPercentile(10) + p.GetPercentile(50) + p.GetPercentile(90));
            //Console.WriteLine(p.Mean + " : " + p.GetPercentile(50) + " : " + p.GetPercentile(75) + " : " + p.GetPercentile(90) + " : " + p.GetPercentile(95) + " : " + p.GetPercentile(99));
        }
        public void testCumulativeCounterAfterRolling()
        {
            MockedClock time = new MockedClock();
            RollingNumberEvent type = RollingNumberEvent.SUCCESS;
            RollingNumber counter = new RollingNumber(time, 20, 2);

            Assert.Equal(0, counter.GetCumulativeSum(type));

            // iterate over 20 buckets on a queue sized for 2
            for (int i = 0; i < 20; i++)
            {
                // first bucket
                counter.Increment(type);
                try
                {
                    time.Increment(counter.BucketSizeInMs);
                }
                catch (Exception)
                {
                    // ignore
                }

                counter.GetValueOfLatestBucket(type);
                Assert.Equal(2, counter.GetValues(type).Count());

            }

            // cumulative count should be 20 (for the number of loops above) regardless of buckets rolling
            Assert.Equal(20, counter.GetCumulativeSum(type));
        }
        public void testWriteThreadSafety()
        {
            MockedClock time = new MockedClock();
            RollingPercentileNumber p = new RollingPercentileNumber(time, 100, 25, 1000, DynamicProperties.Factory.AsProperty(true));

            int NUM_THREADS = 10;
            int NUM_ITERATIONS = 1000;

            var tasks = new Task[NUM_THREADS];

            Random r = new Random();

            long added = 0;

            for (int i = 0; i < NUM_THREADS; i++)
            {
                tasks[i] = Task.Run(() =>
                {

                    for (int j = 1; j < NUM_ITERATIONS / NUM_THREADS + 1; j++)
                    {
                        int nextInt = r.Next(100);
                        p.AddValue(nextInt);
                        Interlocked.Increment(ref added);
                    }
                });
            }

            if (!Task.WaitAll(tasks, 1000))
                throw new Exception("Timeout on all threads writing percentiles");
            Assert.Equal(added, p.GetCurrentBucket().Length);

        }
        public void testCumulativeCounterAfterRollingAndReset3()
        {
            MockedClock time = new MockedClock();
            RollingNumberEvent type = RollingNumberEvent.SUCCESS;
            RollingNumber counter = new RollingNumber(time, 20, 2);

            Assert.Equal(0, counter.GetCumulativeSum(type));

            counter.Increment(type);
            counter.Increment(type);
            counter.Increment(type);

            // iterate over 20 buckets on a queue sized for 2
            for (int i = 0; i < 20; i++)
            {
                try
                {
                    time.Increment(counter.BucketSizeInMs);
                }
                catch (Exception)
                {
                    // ignore
                }
            }

            // since we are rolling over the buckets it should reset naturally

            // no increments during the loop, just some before and after
            counter.Increment(type);
            counter.Increment(type);

            // cumulative count should be 5 regardless of buckets rolling
            Assert.Equal(5, counter.GetCumulativeSum(type));
        }
        public static TestServiceCommand Get(IJellyfishContext ctx, ITestOutputHelper output, string commandName, ExecutionIsolationStrategy strategy,
                                             ExecutionResult executionResult          = ExecutionResult.NONE, FallbackResult fallbackResult = FallbackResult.UNIMPLEMENTED,
                                             Action <CommandPropertiesBuilder> setter = null, TestCircuitBreaker circuitBreaker             = null, IClock clock = null)
        {
            var builder = CommandPropertiesTest.GetUnitTestPropertiesSetter();

            builder.WithExecutionIsolationStrategy(strategy);
            if (setter != null)
            {
                setter(builder);
            }
            if (clock == null)
            {
                clock = new MockedClock();
            }

            var cmd = new TestServiceCommand(ctx, commandName, builder, clock, circuitBreaker ?? new TestCircuitBreaker(clock), new TestExecutionHook(output));

            cmd.Log = msg => output.WriteLine("({0}) - {1}", cmd.Id, msg);

            if (executionResult == ExecutionResult.SUCCESS)
            {
                cmd.Action = () =>
                {
                    cmd.Log("Execution success"); return(Task.FromResult(TestCommandFactory.EXECUTE_VALUE));
                };
            }
            else if (executionResult == ExecutionResult.FAILURE)
            {
                cmd.Action = () => { cmd.Log("Execution failure");; throw new Exception("Execution Failure for TestCommand"); };
            }
            else if (executionResult == ExecutionResult.HYSTRIX_FAILURE)
            {
                cmd.Action = () =>
                {
                    cmd.Log("Execution failure");
                    throw new Exception("Execution  Failure for TestCommand", new Exception("Fallback Failure for TestCommand"));
                };
            }
            else if (executionResult == ExecutionResult.RECOVERABLE_ERROR)
            {
                cmd.Action = () =>
                {
                    cmd.Log("Execution recoverable error");
                    throw new Exception("Execution ERROR for TestCommand");
                };
            }
            else if (executionResult == ExecutionResult.UNRECOVERABLE_ERROR)
            {
                cmd.Action = () =>
                {
                    cmd.Log("Execution unrecoverable error");
                    throw new StackOverflowException("Unrecoverable Error for TestCommand");
                };
            }
            else if (executionResult == ExecutionResult.BAD_REQUEST)
            {
                cmd.Action = () =>
                {
                    cmd.Log("Execution bad request");
                    throw new BadRequestException("Execution BadRequestException for TestCommand");
                };
            }

            if (fallbackResult == FallbackResult.SUCCESS)
            {
                cmd.Fallback = () =>
                {
                    cmd.Log("Execution fallback success");
                    return(Task.FromResult(TestCommandFactory.FALLBACK_VALUE));
                };
            }
            else if (fallbackResult == FallbackResult.FAILURE)
            {
                cmd.Fallback = () => { cmd.Log("Execution fallback error"); throw new Exception("Fallback Failure for TestCommand"); };
            }
            else if (fallbackResult == FallbackResult.UNIMPLEMENTED)
            {
                cmd.Fallback = () =>
                {
                    cmd.Log("Execution fallback unimplemented");
                    throw new NotImplementedException();
                };
            }

            return(cmd);
        }
        public void testIncrementInMultipleBuckets()
        {
            MockedClock time = new MockedClock();

            RollingNumber counter = new RollingNumber(time, 200, 10);

            // increment
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.TIMEOUT);
            counter.Increment(RollingNumberEvent.TIMEOUT);
            counter.Increment(RollingNumberEvent.SHORT_CIRCUITED);

            // sleep to get to a new bucket
            time.Increment(counter.BucketSizeInMs * 3);

            // increment
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.SUCCESS);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.FAILURE);
            counter.Increment(RollingNumberEvent.TIMEOUT);
            counter.Increment(RollingNumberEvent.SHORT_CIRCUITED);

            // we should have 2 buckets
            var buckets = counter.GetBuckets().ToArray();
            Assert.Equal(2, buckets.Length);

            // the counts of the last bucket
            Assert.Equal(2, buckets.First().GetAdder(RollingNumberEvent.SUCCESS));
            Assert.Equal(3, buckets.First().GetAdder(RollingNumberEvent.FAILURE));
            Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
            Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.SHORT_CIRCUITED));

            // the total counts
            Assert.Equal(6, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
            Assert.Equal(5, counter.GetRollingSum(RollingNumberEvent.FAILURE));
            Assert.Equal(3, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
            Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.SHORT_CIRCUITED));

            // wait until window passes
            time.Increment(counter.TimeInMs*10);

            // increment
            counter.Increment(RollingNumberEvent.SUCCESS);

            // the total counts should now include only the last bucket after a reset since the window passed
            Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
            Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
            Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
        }