public void testMaxValue()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;
RollingNumber counter = new RollingNumber(time, 200, 10);
counter.UpdateRollingMax(type, 10);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs);
counter.UpdateRollingMax(type, 30);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs);
counter.UpdateRollingMax(type, 40);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs);
counter.UpdateRollingMax(type, 15);
Assert.Equal(40, counter.GetRollingMaxValue(type));
}
public void testIncrementInMultipleBuckets()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.TIMEOUT);
counter.Increment(RollingNumberEvent.TIMEOUT);
counter.Increment(RollingNumberEvent.SHORT_CIRCUITED);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.TIMEOUT);
counter.Increment(RollingNumberEvent.SHORT_CIRCUITED);
// we should have 2 buckets
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// the counts of the last bucket
Assert.Equal(2, buckets.First().GetAdder(RollingNumberEvent.SUCCESS));
Assert.Equal(3, buckets.First().GetAdder(RollingNumberEvent.FAILURE));
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.SHORT_CIRCUITED));
// the total counts
Assert.Equal(6, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(5, counter.GetRollingSum(RollingNumberEvent.FAILURE));
Assert.Equal(3, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.SHORT_CIRCUITED));
// wait until window passes
time.Increment(counter.TimeInMs * 10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
// the total counts should now include only the last bucket after a reset since the window passed
Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
}
public void testTimeout()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.TIMEOUT);
var buckets = counter.GetBuckets().ToArray();
// we should have 1 bucket
Assert.Equal(1, buckets.Count());
// the count should be 1
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
// incremenet again in latest bucket
counter.Increment(RollingNumberEvent.TIMEOUT);
// we should have 2 buckets
buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// the counts of the last bucket
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
// the total counts
Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
}
public void testCumulativeCounterAfterRollingAndReset3()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.SUCCESS;
RollingNumber counter = new RollingNumber(time, 20, 2);
Assert.Equal(0, counter.GetCumulativeSum(type));
counter.Increment(type);
counter.Increment(type);
counter.Increment(type);
// iterate over 20 buckets on a queue sized for 2
for (int i = 0; i < 20; i++)
{
try
{
time.Increment(counter.BucketSizeInMs);
}
catch (Exception)
{
// ignore
}
}
// since we are rolling over the buckets it should reset naturally
// no increments during the loop, just some before and after
counter.Increment(type);
counter.Increment(type);
// cumulative count should be 5 regardless of buckets rolling
Assert.Equal(5, counter.GetCumulativeSum(type));
}
public void testCumulativeCounterAfterRolling()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.SUCCESS;
RollingNumber counter = new RollingNumber(time, 20, 2);
Assert.Equal(0, counter.GetCumulativeSum(type));
// iterate over 20 buckets on a queue sized for 2
for (int i = 0; i < 20; i++)
{
// first bucket
counter.Increment(type);
try
{
time.Increment(counter.BucketSizeInMs);
}
catch (Exception)
{
// ignore
}
counter.GetValueOfLatestBucket(type);
Assert.Equal(2, counter.GetValues(type).Count());
}
// cumulative count should be 20 (for the number of loops above) regardless of buckets rolling
Assert.Equal(20, counter.GetCumulativeSum(type));
}
public void testRolling()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;
RollingNumber counter = new RollingNumber(time, 20, 2);
// iterate over 20 buckets on a queue sized for 2
for (int i = 0; i < 20; i++)
{
// first bucket
counter.GetCurrentBucket();
try
{
time.Increment(counter.BucketSizeInMs);
}
catch (Exception)
{
// ignore
}
counter.GetValueOfLatestBucket(type);
Assert.Equal(2, counter.GetValues(type).Count());
// System.out.println("Head: " + counter.buckets.state.get().head);
// System.out.println("Tail: " + counter.buckets.state.get().tail);
}
}
public void testCreatesBuckets()
{
var time = new MockedClock();
var counter = new RollingNumber(time, 200, 10);
// confirm the initial settings
Assert.Equal(200, counter.TimeInMs);
Assert.Equal(10, counter.NumberOfBuckets);
Assert.Equal(20, counter.BucketSizeInMs);
// add a Success in each interval which should result in all 10 buckets being created with 1 Success in each
for (int i = 0; i < counter.NumberOfBuckets; i++)
{
counter.Increment(RollingNumberEvent.SUCCESS);
time.Increment(counter.BucketSizeInMs);
}
// confirm we have all 10 buckets
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(10, buckets.Length);
// add 1 more and we should still only have 10 buckets since that's the max
counter.Increment(RollingNumberEvent.SUCCESS);
buckets = counter.GetBuckets().ToArray();
Assert.Equal(10, buckets.Length);
}
public void testSingleTestOnOpenCircuitAfterTimeWindow()
{
int sleepWindow = 200;
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
}
public void testCreatesBuckets()
{
var time = new MockedClock();
var counter = new RollingNumber(time, 200, 10);
// confirm the initial settings
Assert.Equal(200, counter.TimeInMs);
Assert.Equal(10, counter.NumberOfBuckets);
Assert.Equal(20, counter.BucketSizeInMs);
// add a Success in each interval which should result in all 10 buckets being created with 1 Success in each
for (int i = 0; i < counter.NumberOfBuckets; i++)
{
counter.Increment(RollingNumberEvent.SUCCESS);
time.Increment(counter.BucketSizeInMs);
}
// confirm we have all 10 buckets
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(10, buckets.Length);
// add 1 more and we should still only have 10 buckets since that's the max
counter.Increment(RollingNumberEvent.SUCCESS);
buckets = counter.GetBuckets().ToArray();
Assert.Equal(10, buckets.Length);
}
public void testCounterRetrievalRefreshesBuckets()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
var buckets = counter.GetBuckets().ToArray();
// we should have 1 bucket since nothing has triggered the update of buckets in the elapsed time
Assert.Equal(1, buckets.Length);
// the total counts
Assert.Equal(4, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.FAILURE));
// we should have 2 buckets as the counter 'gets' should have triggered the buckets being created to fill in time
buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// wait until window passes
time.Increment(counter.TimeInMs);
// the total counts should all be 0 (and the buckets cleared by the get, not only increment)
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
// the total counts should now include only the last bucket after a reset since the window passed
Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
}
public void testErrorPercentage()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
metrics.MarkSuccess(100);
Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkFailure(1000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
Assert.Equal(25, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkTimeout(5000);
metrics.MarkTimeout(5000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
// latent
clock.Increment(1);
metrics.MarkSuccess(5000);
// 6 success + 1 latent success + 1 failure + 2 timeout = 10 total
// latent success not considered error
// error percentage = 1 failure + 2 timeout / 10
Assert.Equal(30, metrics.GetHealthCounts().ErrorPercentage);
}
public void testCircuitClosedAfterSuccessAndClearsStatisticalWindow()
{
int statisticalWindow = 200;
int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter()
.WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow)
.WithMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' succeeds so should cause the circuit to be closed
metrics.MarkSuccess(500);
cb.MarkSuccess();
// all requests should be open again
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen());
}
public void testUpdateMax2()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 10);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 20);
// we should have 1 bucket
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(1, buckets.Length);
// the count should be 30
Assert.Equal(30, counter.buckets.First().GetMaxUpdater(RollingNumberEvent.THREAD_MAX_ACTIVE));
Assert.Equal(30, counter.GetRollingMaxValue(RollingNumberEvent.THREAD_MAX_ACTIVE));
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 50);
// we should have 2 buckets
buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// the count
Assert.Equal(50, buckets.First().GetMaxUpdater(RollingNumberEvent.THREAD_MAX_ACTIVE));
Assert.Equal(50, counter.GetValueOfLatestBucket(RollingNumberEvent.THREAD_MAX_ACTIVE));
// values per bucket
var values = counter.GetValues(RollingNumberEvent.THREAD_MAX_ACTIVE).ToArray();
Assert.Equal(30, values[1]); // oldest bucket
Assert.Equal(50, values[0]); // latest bucket
}
public void testRolling()
{
MockedClock time = new MockedClock();
RollingPercentileNumber p = new RollingPercentileNumber(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, DynamicProperties.Factory.AsProperty(true));
p.AddValue(1000);
p.AddValue(1000);
p.AddValue(1000);
p.AddValue(2000);
Assert.Equal(1, p.GetBuckets().Count());
// no bucket turnover yet so percentile not yet generated
Assert.Equal(0, p.GetPercentile(50));
time.Increment(6000);
// still only 1 bucket until we touch it again
Assert.Equal(1, p.GetBuckets().Count());
// a bucket has been created so we have a new percentile
Assert.Equal(1000, p.GetPercentile(50));
// now 2 buckets since getting a percentile causes bucket retrieval
Assert.Equal(2, p.GetBuckets().Count());
p.AddValue(1000);
p.AddValue(500);
// should still be 2 buckets
Assert.Equal(2, p.GetBuckets().Count());
p.AddValue(200);
p.AddValue(200);
p.AddValue(1600);
p.AddValue(200);
p.AddValue(1600);
p.AddValue(1600);
// we haven't progressed to a new bucket so the percentile should be the same and ignore the most recent bucket
Assert.Equal(1000, p.GetPercentile(50));
// Increment to another bucket so we include all of the above in the PercentileSnapshot
time.Increment(6000);
// the rolling version should have the same data as creating a snapshot like this
PercentileSnapshot ps = new PercentileSnapshot(1000, 1000, 1000, 2000, 1000, 500, 200, 200, 1600, 200, 1600, 1600);
Assert.Equal(ps.GetPercentile(0.15), p.GetPercentile(0.15));
Assert.Equal(ps.GetPercentile(0.50), p.GetPercentile(0.50));
Assert.Equal(ps.GetPercentile(0.90), p.GetPercentile(0.90));
Assert.Equal(ps.GetPercentile(0.995), p.GetPercentile(0.995));
//Console.WriteLine("100th: " + ps.GetPercentile(100) + " " + p.GetPercentile(100));
//Console.WriteLine("99.5th: " + ps.GetPercentile(99.5) + " " + p.GetPercentile(99.5));
//Console.WriteLine("99th: " + ps.GetPercentile(99) + " " + p.GetPercentile(99));
//Console.WriteLine("90th: " + ps.GetPercentile(90) + " " + p.GetPercentile(90));
//Console.WriteLine("50th: " + ps.GetPercentile(50) + " " + p.GetPercentile(50));
//Console.WriteLine("10th: " + ps.GetPercentile(10) + " " + p.GetPercentile(10));
// mean = 1000+1000+1000+2000+1000+500+200+200+1600+200+1600+1600/12
Assert.Equal(991, ps.Mean);
}
public void testThreadSafety()
{
MockedClock time = new MockedClock();
RollingPercentileNumber p = new RollingPercentileNumber(time, 100, 25, 1000, DynamicProperties.Factory.AsProperty(true));
int NUM_THREADS = 1000;
int NUM_ITERATIONS = 1000000;
var tasks = new Task [NUM_THREADS];
long aggregateMetrics = 0; //same as a blackhole
Random r = new Random();
var token = new CancellationTokenSource();
var metricsPoller = Task.Run(() =>
{
while (!token.IsCancellationRequested)
{
Interlocked.Add(ref aggregateMetrics, p.Mean + p.GetPercentile(10) + p.GetPercentile(50) + p.GetPercentile(90));
//Console.WriteLine("AGGREGATE : " + p.GetPercentile(10) + " : " + p.GetPercentile(50) + " : " + p.GetPercentile(90));
}
});
for (int i = 0; i < NUM_THREADS; i++)
{
int threadId = i;
tasks[i] = Task.Run(() =>
{
for (int j = 1; j < NUM_ITERATIONS / NUM_THREADS + 1; j++)
{
int nextInt = r.Next(100);
p.AddValue(nextInt);
if (threadId == 0)
{
time.Increment(1);
}
}
});
}
if( !Task.WaitAll(tasks, 1000))
throw new Exception("Timeout on all threads writing percentiles");
token.Cancel();
Interlocked.Add(ref aggregateMetrics, p.Mean + p.GetPercentile(10) + p.GetPercentile(50) + p.GetPercentile(90));
//Console.WriteLine(p.Mean + " : " + p.GetPercentile(50) + " : " + p.GetPercentile(75) + " : " + p.GetPercentile(90) + " : " + p.GetPercentile(95) + " : " + p.GetPercentile(99));
}
public void testSampleDataOverTime2()
{
//Console.WriteLine("\n\n***************************** testSampleDataOverTime2 \n");
MockedClock time = new MockedClock();
int previousTime = 0;
RollingPercentileNumber p = new RollingPercentileNumber(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, DynamicProperties.Factory.AsProperty(true));
for (int i = 0; i < SampleDataHolder2.data.Length/2; i++)
{
int timeInMillisecondsSinceStart = SampleDataHolder2.data[i,0];
int latency = SampleDataHolder2.data[i,1];
time.Increment(timeInMillisecondsSinceStart - previousTime);
previousTime = timeInMillisecondsSinceStart;
p.AddValue(latency);
}
//Console.WriteLine("0.01: " + p.GetPercentile(0.01));
//Console.WriteLine("Median: " + p.GetPercentile(50));
//Console.WriteLine("90th: " + p.GetPercentile(90));
//Console.WriteLine("99th: " + p.GetPercentile(99));
//Console.WriteLine("99.5th: " + p.GetPercentile(99.5));
//Console.WriteLine("99.99: " + p.GetPercentile(99.99));
if (p.GetPercentile(50) > 90 || p.GetPercentile(50) < 50)
{
throw new Exception("We expect around 60-70 but got: " + p.GetPercentile(50));
}
if (p.GetPercentile(99) < 400)
{
throw new Exception("We expect to see some high values over 400 but got: " + p.GetPercentile(99));
}
}
public void testSampleDataOverTime1()
{
//Console.WriteLine("\n\n***************************** testSampleDataOverTime1 \n");
MockedClock time = new MockedClock();
RollingPercentileNumber p = new RollingPercentileNumber(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, DynamicProperties.Factory.AsProperty(true));
int previousTime = 0;
for (int i = 0; i < SampleDataHolder1.data.Length/2; i++)
{
int timeInMillisecondsSinceStart = SampleDataHolder1.data[i,0];
int latency = SampleDataHolder1.data[i,1];
time.Increment(timeInMillisecondsSinceStart - previousTime);
previousTime = timeInMillisecondsSinceStart;
p.AddValue(latency);
}
//Console.WriteLine("0.01: " + p.GetPercentile(0.01));
//Console.WriteLine("Median: " + p.GetPercentile(50));
//Console.WriteLine("90th: " + p.GetPercentile(90));
//Console.WriteLine("99th: " + p.GetPercentile(99));
//Console.WriteLine("99.5th: " + p.GetPercentile(99.5));
//Console.WriteLine("99.99: " + p.GetPercentile(99.99));
//Console.WriteLine("Median: " + p.GetPercentile(50));
//Console.WriteLine("Median: " + p.GetPercentile(50));
//Console.WriteLine("Median: " + p.GetPercentile(50));
/*
* In a loop as a use case was found where very different values were calculated in subsequent requests.
*/
for (int i = 0; i < 10; i++)
{
if (p.GetPercentile(50) > 5)
{
throw new Exception("We expect around 2 but got: " + p.GetPercentile(50));
}
if (p.GetPercentile(99.5) < 20)
{
throw new Exception("We expect to see some high values over 20 but got: " + p.GetPercentile(99.5));
}
}
}
public void testValueIsZeroAfterRollingWindowPassesAndNoTraffic()
{
MockedClock time = new MockedClock();
var p = new RollingPercentileNumber(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, DynamicProperties.Factory.AsProperty(true));
p.AddValue(1000);
p.AddValue(1000);
p.AddValue(1000);
p.AddValue(2000);
p.AddValue(4000);
Assert.Equal(1, p.GetBuckets().Count());
// no bucket turnover yet so percentile not yet generated
Assert.Equal(0, p.GetPercentile(50));
time.Increment(6000);
// still only 1 bucket until we touch it again
Assert.Equal(1, p.GetBuckets().Count());
// a bucket has been created so we have a new percentile
Assert.Equal(1500, p.GetPercentile(50));
// let 1 minute pass
time.Increment(60000);
// no data in a minute should mean all buckets are empty (or reset) so we should not have any percentiles
Assert.Equal(0, p.GetPercentile(50));
}
public void testTimeout()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.TIMEOUT);
var buckets = counter.GetBuckets().ToArray();
// we should have 1 bucket
Assert.Equal(1, buckets.Count());
// the count should be 1
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
// incremenet again in latest bucket
counter.Increment(RollingNumberEvent.TIMEOUT);
// we should have 2 buckets
buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// the counts of the last bucket
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
// the total counts
Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
}
public void testCumulativeCounterAfterRollingAndReset3()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.SUCCESS;
RollingNumber counter = new RollingNumber(time, 20, 2);
Assert.Equal(0, counter.GetCumulativeSum(type));
counter.Increment(type);
counter.Increment(type);
counter.Increment(type);
// iterate over 20 buckets on a queue sized for 2
for (int i = 0; i < 20; i++)
{
try
{
time.Increment(counter.BucketSizeInMs);
}
catch (Exception)
{
// ignore
}
}
// since we are rolling over the buckets it should reset naturally
// no increments during the loop, just some before and after
counter.Increment(type);
counter.Increment(type);
// cumulative count should be 5 regardless of buckets rolling
Assert.Equal(5, counter.GetCumulativeSum(type));
}
public void testCumulativeCounterAfterRolling()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.SUCCESS;
RollingNumber counter = new RollingNumber(time, 20, 2);
Assert.Equal(0, counter.GetCumulativeSum(type));
// iterate over 20 buckets on a queue sized for 2
for (int i = 0; i < 20; i++)
{
// first bucket
counter.Increment(type);
try
{
time.Increment(counter.BucketSizeInMs);
}
catch (Exception)
{
// ignore
}
counter.GetValueOfLatestBucket(type);
Assert.Equal(2, counter.GetValues(type).Count());
}
// cumulative count should be 20 (for the number of loops above) regardless of buckets rolling
Assert.Equal(20, counter.GetCumulativeSum(type));
}
public void testRolling()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;
RollingNumber counter = new RollingNumber(time, 20, 2);
// iterate over 20 buckets on a queue sized for 2
for (int i = 0; i < 20; i++)
{
// first bucket
counter.GetCurrentBucket();
try
{
time.Increment(counter.BucketSizeInMs);
}
catch (Exception)
{
// ignore
}
counter.GetValueOfLatestBucket(type);
Assert.Equal(2, counter.GetValues(type).Count());
// System.out.println("Head: " + counter.buckets.state.get().head);
// System.out.println("Tail: " + counter.buckets.state.get().tail);
}
}
public void testMaxValue()
{
MockedClock time = new MockedClock();
RollingNumberEvent type = RollingNumberEvent.THREAD_MAX_ACTIVE;
RollingNumber counter = new RollingNumber(time, 200, 10);
counter.UpdateRollingMax(type, 10);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs);
counter.UpdateRollingMax(type, 30);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs);
counter.UpdateRollingMax(type, 40);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs);
counter.UpdateRollingMax(type, 15);
Assert.Equal(40, counter.GetRollingMaxValue(type));
}
public void testUpdateMax2()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 10);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 20);
// we should have 1 bucket
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(1, buckets.Length);
// the count should be 30
Assert.Equal(30, counter.buckets.First().GetMaxUpdater(RollingNumberEvent.THREAD_MAX_ACTIVE));
Assert.Equal(30, counter.GetRollingMaxValue(RollingNumberEvent.THREAD_MAX_ACTIVE));
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 30);
counter.UpdateRollingMax(RollingNumberEvent.THREAD_MAX_ACTIVE, 50);
// we should have 2 buckets
buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// the count
Assert.Equal(50, buckets.First().GetMaxUpdater(RollingNumberEvent.THREAD_MAX_ACTIVE));
Assert.Equal(50, counter.GetValueOfLatestBucket(RollingNumberEvent.THREAD_MAX_ACTIVE));
// values per bucket
var values = counter.GetValues(RollingNumberEvent.THREAD_MAX_ACTIVE).ToArray();
Assert.Equal(30, values[1]); // oldest bucket
Assert.Equal(50, values[0]); // latest bucket
}
public void testCounterRetrievalRefreshesBuckets()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
var buckets = counter.GetBuckets().ToArray();
// we should have 1 bucket since nothing has triggered the update of buckets in the elapsed time
Assert.Equal(1, buckets.Length);
// the total counts
Assert.Equal(4, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.FAILURE));
// we should have 2 buckets as the counter 'gets' should have triggered the buckets being created to fill in time
buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// wait until window passes
time.Increment(counter.TimeInMs);
// the total counts should all be 0 (and the buckets cleared by the get, not only increment)
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
// the total counts should now include only the last bucket after a reset since the window passed
Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
}
public void testIncrementInMultipleBuckets()
{
MockedClock time = new MockedClock();
RollingNumber counter = new RollingNumber(time, 200, 10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.TIMEOUT);
counter.Increment(RollingNumberEvent.TIMEOUT);
counter.Increment(RollingNumberEvent.SHORT_CIRCUITED);
// sleep to get to a new bucket
time.Increment(counter.BucketSizeInMs * 3);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.SUCCESS);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.FAILURE);
counter.Increment(RollingNumberEvent.TIMEOUT);
counter.Increment(RollingNumberEvent.SHORT_CIRCUITED);
// we should have 2 buckets
var buckets = counter.GetBuckets().ToArray();
Assert.Equal(2, buckets.Length);
// the counts of the last bucket
Assert.Equal(2, buckets.First().GetAdder(RollingNumberEvent.SUCCESS));
Assert.Equal(3, buckets.First().GetAdder(RollingNumberEvent.FAILURE));
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.TIMEOUT));
Assert.Equal(1, buckets.First().GetAdder(RollingNumberEvent.SHORT_CIRCUITED));
// the total counts
Assert.Equal(6, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(5, counter.GetRollingSum(RollingNumberEvent.FAILURE));
Assert.Equal(3, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
Assert.Equal(2, counter.GetRollingSum(RollingNumberEvent.SHORT_CIRCUITED));
// wait until window passes
time.Increment(counter.TimeInMs*10);
// increment
counter.Increment(RollingNumberEvent.SUCCESS);
// the total counts should now include only the last bucket after a reset since the window passed
Assert.Equal(1, counter.GetRollingSum(RollingNumberEvent.SUCCESS));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.FAILURE));
Assert.Equal(0, counter.GetRollingSum(RollingNumberEvent.TIMEOUT));
}
public void testMultipleTimeWindowRetriesBeforeClosingCircuit()
{
int sleepWindow = 200;
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' fails so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep
metrics.MarkFailure(1000);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' fails again so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep
metrics.MarkFailure(1000);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// now it finally succeeds
metrics.MarkSuccess(200);
cb.MarkSuccess();
// all requests should be open again
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen());
}