public void testCreatesBuckets()
{
var time = new MockedClock();
var counter = new RollingNumber(time, 200, 10);
// confirm the initial settings
Assert.Equal(200, counter.TimeInMs);
Assert.Equal(10, counter.NumberOfBuckets);
Assert.Equal(20, counter.BucketSizeInMs);
// add a Success in each interval which should result in all 10 buckets being created with 1 Success in each
for (int i = 0; i < counter.NumberOfBuckets; i++)
{
counter.Increment(RollingNumberEvent.SUCCESS);
time.Increment(counter.BucketSizeInMs);
}
// confirm we have all 10 buckets
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(10, buckets.Length);
// add 1 more and we should still only have 10 buckets since that's the max
counter.Increment(RollingNumberEvent.SUCCESS);
buckets = counter.GetBuckets().ToArray();
Assert.Equal(10, buckets.Length);
}
public void testCumulativeCounterAfterRolling()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.SUCCESS;
RollingNumber counter = new RollingNumber(time, 20, 2);
Assert.Equal(0, counter.GetCumulativeSum(type));
// iterate over 20 buckets on a queue sized for 2
for (int i = 0; i < 20; i++)
{
// first bucket
counter.Increment(type);
try
{
time.Increment(counter.BucketSizeInMs);
}
catch (Exception)
{
// ignore
}
counter.GetValueOfLatestBucket(type);
Assert.Equal(2, counter.GetValues(type).Count());
}
// cumulative count should be 20 (for the number of loops above) regardless of buckets rolling
Assert.Equal(20, counter.GetCumulativeSum(type));
}
public void testTripCircuit()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
// this should still allow requests as everything has been successful
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
}
public void testCumulativeCounterAfterRollingAndReset3()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.SUCCESS;
RollingNumber counter = new RollingNumber(time, 20, 2);
Assert.Equal(0, counter.GetCumulativeSum(type));
counter.Increment(type);
counter.Increment(type);
counter.Increment(type);
// iterate over 20 buckets on a queue sized for 2
for (int i = 0; i < 20; i++)
{
try
{
time.Increment(counter.BucketSizeInMs);
}
catch (Exception)
{
// ignore
}
}
// since we are rolling over the buckets it should reset naturally
// no increments during the loop, just some before and after
counter.Increment(type);
counter.Increment(type);
// cumulative count should be 5 regardless of buckets rolling
Assert.Equal(5, counter.GetCumulativeSum(type));
}
public void testCircuitDoesNotTripOnFailuresBelowThreshold()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// success with high latency
metrics.MarkSuccess(400);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkSuccess(400);
metrics.MarkSuccess(40);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkFailure(10);
// this should remain open as the failure threshold is below the percentage limit
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
}
public void testRolling()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;
RollingNumber counter = new RollingNumber(time, 20, 2);
// iterate over 20 buckets on a queue sized for 2
for (int i = 0; i < 20; i++)
{
// first bucket
counter.GetCurrentBucket();
try
{
time.Increment(counter.BucketSizeInMs);
}
catch (Exception)
{
// ignore
}
counter.GetValueOfLatestBucket(type);
Assert.Equal(2, counter.GetValues(type).Count());
// System.out.println("Head: " + counter.buckets.state.get().head);
// System.out.println("Tail: " + counter.buckets.state.get().tail);
}
}
public void testCreatesBuckets()
{
var time = new MockedClock();
var counter = new RollingNumber(time, 200, 10);
// confirm the initial settings
Assert.Equal(200, counter.TimeInMs);
Assert.Equal(10, counter.NumberOfBuckets);
Assert.Equal(20, counter.BucketSizeInMs);
// add a Success in each interval which should result in all 10 buckets being created with 1 Success in each
for (int i = 0; i < counter.NumberOfBuckets; i++)
{
counter.Increment(RollingNumberEvent.SUCCESS);
time.Increment(counter.BucketSizeInMs);
}
// confirm we have all 10 buckets
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(10, buckets.Length);
// add 1 more and we should still only have 10 buckets since that's the max
counter.Increment(RollingNumberEvent.SUCCESS);
buckets = counter.GetBuckets().ToArray();
Assert.Equal(10, buckets.Length);
}
public void testTripCircuitOnTimeoutsAboveThreshold()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// success with high latency
metrics.MarkSuccess(400);
metrics.MarkSuccess(400);
metrics.MarkTimeout(10);
metrics.MarkSuccess(400);
metrics.MarkTimeout(10);
metrics.MarkTimeout(10);
metrics.MarkSuccess(400);
metrics.MarkTimeout(10);
metrics.MarkTimeout(10);
// this should trip the circuit as the error percentage is above the threshold
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
}
public void testMaxValue()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;
RollingNumber counter = new RollingNumber(time, 200, 10);
counter.UpdateRollingMax(type, 10);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs);
counter.UpdateRollingMax(type, 30);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs);
counter.UpdateRollingMax(type, 40);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs);
counter.UpdateRollingMax(type, 15);
Assert.Equal(40, counter.GetRollingMaxValue(type));
}
public void testSingleTestOnOpenCircuitAfterTimeWindow()
{
int sleepWindow = 200;
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
}
public void testTimeout()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.TIMEOUT);
var buckets = counter.GetBuckets().ToArray();
// we should have 1 bucket
Assert.Equal(1, buckets.Count());
// the count should be 1
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
// incremenet again in latest bucket
counter.Increment(RollingNumberEvent.TIMEOUT);
// we should have 2 buckets
buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// the counts of the last bucket
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
// the total counts
Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
}
public void testIncrementInSingleBucket()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.TIMEOUT);
// we should have 1 bucket
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(1, buckets.Length);
// the count should be 4
Assert.Equal(4, counter.GetCurrentBucket().GetAdder(RollingNumberEvent.SUCCESS));
Assert.Equal(2, counter.GetCurrentBucket().GetAdder(RollingNumberEvent.FAILURE));
Assert.Equal(1, counter.GetCurrentBucket().GetAdder(RollingNumberEvent.TIMEOUT));
}
public void testEmptyLatestValue()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;
RollingNumber counter = new RollingNumber(time, 200, 10);
Assert.Equal(0, counter.GetValueOfLatestBucket(type));
}
public void testEmptySum()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.SUCCESS;
RollingNumber counter = new RollingNumber(time, 200, 10);
Assert.Equal(0, counter.GetRollingSum(type));
}
public void testIncrementInMultipleBuckets()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.TIMEOUT);
counter.Increment(RollingNumberEvent.TIMEOUT);
counter.Increment(RollingNumberEvent.SHORT_CIRCUITED);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.TIMEOUT);
counter.Increment(RollingNumberEvent.SHORT_CIRCUITED);
// we should have 2 buckets
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// the counts of the last bucket
Assert.Equal(2, buckets.First().GetAdder(RollingNumberEvent.SUCCESS));
Assert.Equal(3, buckets.First().GetAdder(RollingNumberEvent.FAILURE));
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.SHORT_CIRCUITED));
// the total counts
Assert.Equal(6, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(5, counter.GetRollingSum(RollingNumberEvent.FAILURE));
Assert.Equal(3, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.SHORT_CIRCUITED));
// wait until window passes
time.Increment(counter.TimeInMs * 10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
// the total counts should now include only the last bucket after a reset since the window passed
Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
}
public void testCounterRetrievalRefreshesBuckets()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
var buckets = counter.GetBuckets().ToArray();
// we should have 1 bucket since nothing has triggered the update of buckets in the elapsed time
Assert.Equal(1, buckets.Length);
// the total counts
Assert.Equal(4, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.FAILURE));
// we should have 2 buckets as the counter 'gets' should have triggered the buckets being created to fill in time
buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// wait until window passes
time.Increment(counter.TimeInMs);
// the total counts should all be 0 (and the buckets cleared by the get, not only increment)
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
// the total counts should now include only the last bucket after a reset since the window passed
Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
}
public void testCircuitClosedAfterSuccessAndClearsStatisticalWindow()
{
int statisticalWindow = 200;
int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter()
.WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow)
.WithMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' succeeds so should cause the circuit to be closed
metrics.MarkSuccess(500);
cb.MarkSuccess();
// all requests should be open again
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen());
}
public void testLowVolumeDoesNotTripCircuit()
{
int sleepWindow = 200;
int lowVolume = 5;
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithCircuitBreakerRequestVolumeThreshold(lowVolume);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// even though it has all failed we won't trip the circuit because the volume is low
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
}
public void testBadRequestsDoNotAffectErrorPercentage()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);;
metrics.MarkSuccess(100);
Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage);
metrics.MarkFailure(1000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
metrics.MarkBadRequest(1);
metrics.MarkBadRequest(2);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
metrics.MarkFailure(45);
metrics.MarkFailure(55);
Assert.Equal(75, metrics.GetHealthCounts().ErrorPercentage);
}
public void testUpdateMax2()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 10);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 20);
// we should have 1 bucket
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(1, buckets.Length);
// the count should be 30
Assert.Equal(30, counter.buckets.First().GetMaxUpdater(RollingNumberEvent.THREAD_MAX_ACTIVE));
Assert.Equal(30, counter.GetRollingMaxValue(RollingNumberEvent.THREAD_MAX_ACTIVE));
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 50);
// we should have 2 buckets
buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// the count
Assert.Equal(50, buckets.First().GetMaxUpdater(RollingNumberEvent.THREAD_MAX_ACTIVE));
Assert.Equal(50, counter.GetValueOfLatestBucket(RollingNumberEvent.THREAD_MAX_ACTIVE));
// values per bucket
var values = counter.GetValues(RollingNumberEvent.THREAD_MAX_ACTIVE).ToArray();
Assert.Equal(30, values[1]); // oldest bucket
Assert.Equal(50, values[0]); // latest bucket
}
public void testTripCircuitOnTimeouts()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// timeouts
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
// everything has been a timeout so we should not allow any requests
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
}
public void testResetBuckets()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// add 1
counter.Increment(RollingNumberEvent.SUCCESS);
// confirm we have 1 bucket
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(1, buckets.Length);
// confirm we still have 1 bucket
Assert.Equal(1, buckets.Length);
// add 1
counter.Increment(RollingNumberEvent.SUCCESS);
// we should now have a single bucket with no values in it instead of 2 or more buckets
buckets = counter.GetBuckets().ToArray();
Assert.Equal(1, buckets.Length);
}
public void testResetBuckets()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// add 1
counter.Increment(RollingNumberEvent.SUCCESS);
// confirm we have 1 bucket
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(1, buckets.Length);
// confirm we still have 1 bucket
Assert.Equal(1, buckets.Length);
// add 1
counter.Increment(RollingNumberEvent.SUCCESS);
// we should now have a single bucket with no values in it instead of 2 or more buckets
buckets = counter.GetBuckets().ToArray();
Assert.Equal(1, buckets.Length);
}
public void testErrorPercentage()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
metrics.MarkSuccess(100);
Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkFailure(1000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
Assert.Equal(25, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkTimeout(5000);
metrics.MarkTimeout(5000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
// latent
clock.Increment(1);
metrics.MarkSuccess(5000);
// 6 success + 1 latent success + 1 failure + 2 timeout = 10 total
// latent success not considered error
// error percentage = 1 failure + 2 timeout / 10
Assert.Equal(30, metrics.GetHealthCounts().ErrorPercentage);
}
public void testIncrementInSingleBucket()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.TIMEOUT);
// we should have 1 bucket
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(1, buckets.Length);
// the count should be 4
Assert.Equal(4, counter.GetCurrentBucket().GetAdder(RollingNumberEvent.SUCCESS));
Assert.Equal(2, counter.GetCurrentBucket().GetAdder(RollingNumberEvent.FAILURE));
Assert.Equal(1, counter.GetCurrentBucket().GetAdder(RollingNumberEvent.TIMEOUT));
}
public void testSampleDataOverTime2()
{
//Console.WriteLine("\n\n***************************** testSampleDataOverTime2 \n");
MockedClock time = new MockedClock();
int previousTime = 0;
RollingPercentileNumber p = new RollingPercentileNumber(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, DynamicProperties.Factory.AsProperty(true));
for (int i = 0; i < SampleDataHolder2.data.Length/2; i++)
{
int timeInMillisecondsSinceStart = SampleDataHolder2.data[i,0];
int latency = SampleDataHolder2.data[i,1];
time.Increment(timeInMillisecondsSinceStart - previousTime);
previousTime = timeInMillisecondsSinceStart;
p.AddValue(latency);
}
//Console.WriteLine("0.01: " + p.GetPercentile(0.01));
//Console.WriteLine("Median: " + p.GetPercentile(50));
//Console.WriteLine("90th: " + p.GetPercentile(90));
//Console.WriteLine("99th: " + p.GetPercentile(99));
//Console.WriteLine("99.5th: " + p.GetPercentile(99.5));
//Console.WriteLine("99.99: " + p.GetPercentile(99.99));
if (p.GetPercentile(50) > 90 || p.GetPercentile(50) < 50)
{
throw new Exception("We expect around 60-70 but got: " + p.GetPercentile(50));
}
if (p.GetPercentile(99) < 400)
{
throw new Exception("We expect to see some high values over 400 but got: " + p.GetPercentile(99));
}
}
public void testSampleDataOverTime1()
{
//Console.WriteLine("\n\n***************************** testSampleDataOverTime1 \n");
MockedClock time = new MockedClock();
RollingPercentileNumber p = new RollingPercentileNumber(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, DynamicProperties.Factory.AsProperty(true));
int previousTime = 0;
for (int i = 0; i < SampleDataHolder1.data.Length/2; i++)
{
int timeInMillisecondsSinceStart = SampleDataHolder1.data[i,0];
int latency = SampleDataHolder1.data[i,1];
time.Increment(timeInMillisecondsSinceStart - previousTime);
previousTime = timeInMillisecondsSinceStart;
p.AddValue(latency);
}
//Console.WriteLine("0.01: " + p.GetPercentile(0.01));
//Console.WriteLine("Median: " + p.GetPercentile(50));
//Console.WriteLine("90th: " + p.GetPercentile(90));
//Console.WriteLine("99th: " + p.GetPercentile(99));
//Console.WriteLine("99.5th: " + p.GetPercentile(99.5));
//Console.WriteLine("99.99: " + p.GetPercentile(99.99));
//Console.WriteLine("Median: " + p.GetPercentile(50));
//Console.WriteLine("Median: " + p.GetPercentile(50));
//Console.WriteLine("Median: " + p.GetPercentile(50));
/*
* In a loop as a use case was found where very different values were calculated in subsequent requests.
*/
for (int i = 0; i < 10; i++)
{
if (p.GetPercentile(50) > 5)
{
throw new Exception("We expect around 2 but got: " + p.GetPercentile(50));
}
if (p.GetPercentile(99.5) < 20)
{
throw new Exception("We expect to see some high values over 20 but got: " + p.GetPercentile(99.5));
}
}
}
public void testValueIsZeroAfterRollingWindowPassesAndNoTraffic()
{
MockedClock time = new MockedClock();
var p = new RollingPercentileNumber(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, DynamicProperties.Factory.AsProperty(true));
p.AddValue(1000);
p.AddValue(1000);
p.AddValue(1000);
p.AddValue(2000);
p.AddValue(4000);
Assert.Equal(1, p.GetBuckets().Count());
// no bucket turnover yet so percentile not yet generated
Assert.Equal(0, p.GetPercentile(50));
time.Increment(6000);
// still only 1 bucket until we touch it again
Assert.Equal(1, p.GetBuckets().Count());
// a bucket has been created so we have a new percentile
Assert.Equal(1500, p.GetPercentile(50));
// let 1 minute pass
time.Increment(60000);
// no data in a minute should mean all buckets are empty (or reset) so we should not have any percentiles
Assert.Equal(0, p.GetPercentile(50));
}
public void testTimeout()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.TIMEOUT);
var buckets = counter.GetBuckets().ToArray();
// we should have 1 bucket
Assert.Equal(1, buckets.Count());
// the count should be 1
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
// incremenet again in latest bucket
counter.Increment(RollingNumberEvent.TIMEOUT);
// we should have 2 buckets
buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// the counts of the last bucket
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
// the total counts
Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
}
public void testCounterRetrievalRefreshesBuckets()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
var buckets = counter.GetBuckets().ToArray();
// we should have 1 bucket since nothing has triggered the update of buckets in the elapsed time
Assert.Equal(1, buckets.Length);
// the total counts
Assert.Equal(4, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.FAILURE));
// we should have 2 buckets as the counter 'gets' should have triggered the buckets being created to fill in time
buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// wait until window passes
time.Increment(counter.TimeInMs);
// the total counts should all be 0 (and the buckets cleared by the get, not only increment)
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
// the total counts should now include only the last bucket after a reset since the window passed
Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
}
public void testRolling()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;
RollingNumber counter = new RollingNumber(time, 20, 2);
// iterate over 20 buckets on a queue sized for 2
for (int i = 0; i < 20; i++)
{
// first bucket
counter.GetCurrentBucket();
try
{
time.Increment(counter.BucketSizeInMs);
}
catch (Exception)
{
// ignore
}
counter.GetValueOfLatestBucket(type);
Assert.Equal(2, counter.GetValues(type).Count());
// System.out.println("Head: " + counter.buckets.state.get().head);
// System.out.println("Tail: " + counter.buckets.state.get().tail);
}
}
public void testRolling()
{
MockedClock time = new MockedClock();
RollingPercentileNumber p = new RollingPercentileNumber(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, DynamicProperties.Factory.AsProperty(true));
p.AddValue(1000);
p.AddValue(1000);
p.AddValue(1000);
p.AddValue(2000);
Assert.Equal(1, p.GetBuckets().Count());
// no bucket turnover yet so percentile not yet generated
Assert.Equal(0, p.GetPercentile(50));
time.Increment(6000);
// still only 1 bucket until we touch it again
Assert.Equal(1, p.GetBuckets().Count());
// a bucket has been created so we have a new percentile
Assert.Equal(1000, p.GetPercentile(50));
// now 2 buckets since getting a percentile causes bucket retrieval
Assert.Equal(2, p.GetBuckets().Count());
p.AddValue(1000);
p.AddValue(500);
// should still be 2 buckets
Assert.Equal(2, p.GetBuckets().Count());
p.AddValue(200);
p.AddValue(200);
p.AddValue(1600);
p.AddValue(200);
p.AddValue(1600);
p.AddValue(1600);
// we haven't progressed to a new bucket so the percentile should be the same and ignore the most recent bucket
Assert.Equal(1000, p.GetPercentile(50));
// Increment to another bucket so we include all of the above in the PercentileSnapshot
time.Increment(6000);
// the rolling version should have the same data as creating a snapshot like this
PercentileSnapshot ps = new PercentileSnapshot(1000, 1000, 1000, 2000, 1000, 500, 200, 200, 1600, 200, 1600, 1600);
Assert.Equal(ps.GetPercentile(0.15), p.GetPercentile(0.15));
Assert.Equal(ps.GetPercentile(0.50), p.GetPercentile(0.50));
Assert.Equal(ps.GetPercentile(0.90), p.GetPercentile(0.90));
Assert.Equal(ps.GetPercentile(0.995), p.GetPercentile(0.995));
//Console.WriteLine("100th: " + ps.GetPercentile(100) + " " + p.GetPercentile(100));
//Console.WriteLine("99.5th: " + ps.GetPercentile(99.5) + " " + p.GetPercentile(99.5));
//Console.WriteLine("99th: " + ps.GetPercentile(99) + " " + p.GetPercentile(99));
//Console.WriteLine("90th: " + ps.GetPercentile(90) + " " + p.GetPercentile(90));
//Console.WriteLine("50th: " + ps.GetPercentile(50) + " " + p.GetPercentile(50));
//Console.WriteLine("10th: " + ps.GetPercentile(10) + " " + p.GetPercentile(10));
// mean = 1000+1000+1000+2000+1000+500+200+200+1600+200+1600+1600/12
Assert.Equal(991, ps.Mean);
}
public void testEmptySum()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.SUCCESS;
RollingNumber counter = new RollingNumber(time, 200, 10);
Assert.Equal(0, counter.GetRollingSum(type));
}
public void testEmptyLatestValue()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;
RollingNumber counter = new RollingNumber(time, 200, 10);
Assert.Equal(0, counter.GetValueOfLatestBucket(type));
}
public void testMultipleTimeWindowRetriesBeforeClosingCircuit()
{
int sleepWindow = 200;
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' fails so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep
metrics.MarkFailure(1000);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' fails again so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep
metrics.MarkFailure(1000);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// now it finally succeeds
metrics.MarkSuccess(200);
cb.MarkSuccess();
// all requests should be open again
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen());
}
public void testMaxValue()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;
RollingNumber counter = new RollingNumber(time, 200, 10);
counter.UpdateRollingMax(type, 10);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs);
counter.UpdateRollingMax(type, 30);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs);
counter.UpdateRollingMax(type, 40);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs);
counter.UpdateRollingMax(type, 15);
Assert.Equal(40, counter.GetRollingMaxValue(type));
}
public void testUpdateMax2()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 10);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 20);
// we should have 1 bucket
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(1, buckets.Length);
// the count should be 30
Assert.Equal(30, counter.buckets.First().GetMaxUpdater(RollingNumberEvent.THREAD_MAX_ACTIVE));
Assert.Equal(30, counter.GetRollingMaxValue(RollingNumberEvent.THREAD_MAX_ACTIVE));
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 50);
// we should have 2 buckets
buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// the count
Assert.Equal(50, buckets.First().GetMaxUpdater(RollingNumberEvent.THREAD_MAX_ACTIVE));
Assert.Equal(50, counter.GetValueOfLatestBucket(RollingNumberEvent.THREAD_MAX_ACTIVE));
// values per bucket
var values = counter.GetValues(RollingNumberEvent.THREAD_MAX_ACTIVE).ToArray();
Assert.Equal(30, values[1]); // oldest bucket
Assert.Equal(50, values[0]); // latest bucket
}
public void testThreadSafety()
{
MockedClock time = new MockedClock();
RollingPercentileNumber p = new RollingPercentileNumber(time, 100, 25, 1000, DynamicProperties.Factory.AsProperty(true));
int NUM_THREADS = 1000;
int NUM_ITERATIONS = 1000000;
var tasks = new Task [NUM_THREADS];
long aggregateMetrics = 0; //same as a blackhole
Random r = new Random();
var token = new CancellationTokenSource();
var metricsPoller = Task.Run(() =>
{
while (!token.IsCancellationRequested)
{
Interlocked.Add(ref aggregateMetrics, p.Mean + p.GetPercentile(10) + p.GetPercentile(50) + p.GetPercentile(90));
//Console.WriteLine("AGGREGATE : " + p.GetPercentile(10) + " : " + p.GetPercentile(50) + " : " + p.GetPercentile(90));
}
});
for (int i = 0; i < NUM_THREADS; i++)
{
int threadId = i;
tasks[i] = Task.Run(() =>
{
for (int j = 1; j < NUM_ITERATIONS / NUM_THREADS + 1; j++)
{
int nextInt = r.Next(100);
p.AddValue(nextInt);
if (threadId == 0)
{
time.Increment(1);
}
}
});
}
if( !Task.WaitAll(tasks, 1000))
throw new Exception("Timeout on all threads writing percentiles");
token.Cancel();
Interlocked.Add(ref aggregateMetrics, p.Mean + p.GetPercentile(10) + p.GetPercentile(50) + p.GetPercentile(90));
//Console.WriteLine(p.Mean + " : " + p.GetPercentile(50) + " : " + p.GetPercentile(75) + " : " + p.GetPercentile(90) + " : " + p.GetPercentile(95) + " : " + p.GetPercentile(99));
}
public void testCumulativeCounterAfterRolling()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.SUCCESS;
RollingNumber counter = new RollingNumber(time, 20, 2);
Assert.Equal(0, counter.GetCumulativeSum(type));
// iterate over 20 buckets on a queue sized for 2
for (int i = 0; i < 20; i++)
{
// first bucket
counter.Increment(type);
try
{
time.Increment(counter.BucketSizeInMs);
}
catch (Exception)
{
// ignore
}
counter.GetValueOfLatestBucket(type);
Assert.Equal(2, counter.GetValues(type).Count());
}
// cumulative count should be 20 (for the number of loops above) regardless of buckets rolling
Assert.Equal(20, counter.GetCumulativeSum(type));
}
public void testWriteThreadSafety()
{
MockedClock time = new MockedClock();
RollingPercentileNumber p = new RollingPercentileNumber(time, 100, 25, 1000, DynamicProperties.Factory.AsProperty(true));
int NUM_THREADS = 10;
int NUM_ITERATIONS = 1000;
var tasks = new Task[NUM_THREADS];
Random r = new Random();
long added = 0;
for (int i = 0; i < NUM_THREADS; i++)
{
tasks[i] = Task.Run(() =>
{
for (int j = 1; j < NUM_ITERATIONS / NUM_THREADS + 1; j++)
{
int nextInt = r.Next(100);
p.AddValue(nextInt);
Interlocked.Increment(ref added);
}
});
}
if (!Task.WaitAll(tasks, 1000))
throw new Exception("Timeout on all threads writing percentiles");
Assert.Equal(added, p.GetCurrentBucket().Length);
}
public void testCumulativeCounterAfterRollingAndReset3()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.SUCCESS;
RollingNumber counter = new RollingNumber(time, 20, 2);
Assert.Equal(0, counter.GetCumulativeSum(type));
counter.Increment(type);
counter.Increment(type);
counter.Increment(type);
// iterate over 20 buckets on a queue sized for 2
for (int i = 0; i < 20; i++)
{
try
{
time.Increment(counter.BucketSizeInMs);
}
catch (Exception)
{
// ignore
}
}
// since we are rolling over the buckets it should reset naturally
// no increments during the loop, just some before and after
counter.Increment(type);
counter.Increment(type);
// cumulative count should be 5 regardless of buckets rolling
Assert.Equal(5, counter.GetCumulativeSum(type));
}
public static TestServiceCommand Get(IJellyfishContext ctx, ITestOutputHelper output, string commandName, ExecutionIsolationStrategy strategy,
ExecutionResult executionResult = ExecutionResult.NONE, FallbackResult fallbackResult = FallbackResult.UNIMPLEMENTED,
Action <CommandPropertiesBuilder> setter = null, TestCircuitBreaker circuitBreaker = null, IClock clock = null)
{
var builder = CommandPropertiesTest.GetUnitTestPropertiesSetter();
builder.WithExecutionIsolationStrategy(strategy);
if (setter != null)
{
setter(builder);
}
if (clock == null)
{
clock = new MockedClock();
}
var cmd = new TestServiceCommand(ctx, commandName, builder, clock, circuitBreaker ?? new TestCircuitBreaker(clock), new TestExecutionHook(output));
cmd.Log = msg => output.WriteLine("({0}) - {1}", cmd.Id, msg);
if (executionResult == ExecutionResult.SUCCESS)
{
cmd.Action = () =>
{
cmd.Log("Execution success"); return(Task.FromResult(TestCommandFactory.EXECUTE_VALUE));
};
}
else if (executionResult == ExecutionResult.FAILURE)
{
cmd.Action = () => { cmd.Log("Execution failure");; throw new Exception("Execution Failure for TestCommand"); };
}
else if (executionResult == ExecutionResult.HYSTRIX_FAILURE)
{
cmd.Action = () =>
{
cmd.Log("Execution failure");
throw new Exception("Execution Failure for TestCommand", new Exception("Fallback Failure for TestCommand"));
};
}
else if (executionResult == ExecutionResult.RECOVERABLE_ERROR)
{
cmd.Action = () =>
{
cmd.Log("Execution recoverable error");
throw new Exception("Execution ERROR for TestCommand");
};
}
else if (executionResult == ExecutionResult.UNRECOVERABLE_ERROR)
{
cmd.Action = () =>
{
cmd.Log("Execution unrecoverable error");
throw new StackOverflowException("Unrecoverable Error for TestCommand");
};
}
else if (executionResult == ExecutionResult.BAD_REQUEST)
{
cmd.Action = () =>
{
cmd.Log("Execution bad request");
throw new BadRequestException("Execution BadRequestException for TestCommand");
};
}
if (fallbackResult == FallbackResult.SUCCESS)
{
cmd.Fallback = () =>
{
cmd.Log("Execution fallback success");
return(Task.FromResult(TestCommandFactory.FALLBACK_VALUE));
};
}
else if (fallbackResult == FallbackResult.FAILURE)
{
cmd.Fallback = () => { cmd.Log("Execution fallback error"); throw new Exception("Fallback Failure for TestCommand"); };
}
else if (fallbackResult == FallbackResult.UNIMPLEMENTED)
{
cmd.Fallback = () =>
{
cmd.Log("Execution fallback unimplemented");
throw new NotImplementedException();
};
}
return(cmd);
}
public void testIncrementInMultipleBuckets()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.TIMEOUT);
counter.Increment(RollingNumberEvent.TIMEOUT);
counter.Increment(RollingNumberEvent.SHORT_CIRCUITED);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.TIMEOUT);
counter.Increment(RollingNumberEvent.SHORT_CIRCUITED);
// we should have 2 buckets
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// the counts of the last bucket
Assert.Equal(2, buckets.First().GetAdder(RollingNumberEvent.SUCCESS));
Assert.Equal(3, buckets.First().GetAdder(RollingNumberEvent.FAILURE));
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.SHORT_CIRCUITED));
// the total counts
Assert.Equal(6, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(5, counter.GetRollingSum(RollingNumberEvent.FAILURE));
Assert.Equal(3, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.SHORT_CIRCUITED));
// wait until window passes
time.Increment(counter.TimeInMs*10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
// the total counts should now include only the last bucket after a reset since the window passed
Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
}