public JsonMetricsPublisherCommand(CommandMetrics metrics, ICircuitBreaker circuitBreaker = null)
{
this.commandName = metrics.CommandName;
this.metrics = metrics;
this.circuitBreaker = circuitBreaker ?? CircuitBreaker.CircuitBreakerFactory.GetInstance(commandName);
this.properties = metrics.Properties;
}
public JsonMetricsPublisherCommand(string commandName, CommandMetrics metrics, ICircuitBreaker circuitBreaker, CommandProperties properties)
{
this.commandName = commandName;
this.metrics = metrics;
this.circuitBreaker = circuitBreaker;
this.properties = properties;
}
public void testTripCircuitOnTimeoutsAboveThreshold()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// success with high latency
metrics.MarkSuccess(400);
metrics.MarkSuccess(400);
metrics.MarkTimeout(10);
metrics.MarkSuccess(400);
metrics.MarkTimeout(10);
metrics.MarkTimeout(10);
metrics.MarkSuccess(400);
metrics.MarkTimeout(10);
metrics.MarkTimeout(10);
// this should trip the circuit as the error percentage is above the threshold
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
}
public void testSingleTestOnOpenCircuitAfterTimeWindow()
{
int sleepWindow = 200;
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
}
public void testCircuitDoesNotTripOnFailuresBelowThreshold()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// success with high latency
metrics.MarkSuccess(400);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkSuccess(400);
metrics.MarkSuccess(40);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkFailure(10);
// this should remain open as the failure threshold is below the percentage limit
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
}
public JsonMetricsPublisherCommand(CommandMetrics metrics, ICircuitBreaker circuitBreaker=null)
{
this.commandName = metrics.CommandName;
this.metrics = metrics;
this.circuitBreaker = circuitBreaker ?? CircuitBreaker.CircuitBreakerFactory.GetInstance(commandName);
this.properties = metrics.Properties;
}
public void testTripCircuit()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
// this should still allow requests as everything has been successful
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
}
private IMetricsPublisherCommand GetOrCreatePublisherForCommand(string commandName, CommandMetrics metrics, ICircuitBreaker circuitBreaker)
{
// attempt to retrieve from cache first
IMetricsPublisherCommand publisher;
if (_commandPublishers.TryGetValue(commandName, out publisher))
{
return publisher;
}
// it doesn't exist so we need to create it
publisher = _context.GetService<IMetricsPublisherCommand>() ?? new DefaultPublisherCommand();
// attempt to store it (race other threads)
return _commandPublishers.GetOrAdd(commandName, publisher);
}
public void testCircuitClosedAfterSuccessAndClearsStatisticalWindow()
{
int statisticalWindow = 200;
int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter()
.WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow)
.WithMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' succeeds so should cause the circuit to be closed
metrics.MarkSuccess(500);
cb.MarkSuccess();
// all requests should be open again
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen());
}
public void testLowVolumeDoesNotTripCircuit()
{
int sleepWindow = 200;
int lowVolume = 5;
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithCircuitBreakerRequestVolumeThreshold(lowVolume);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// even though it has all failed we won't trip the circuit because the volume is low
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
}
public void testBadRequestsDoNotAffectErrorPercentage()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);;
metrics.MarkSuccess(100);
Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage);
metrics.MarkFailure(1000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
metrics.MarkBadRequest(1);
metrics.MarkBadRequest(2);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
metrics.MarkFailure(45);
metrics.MarkFailure(55);
Assert.Equal(75, metrics.GetHealthCounts().ErrorPercentage);
}
public void testTripCircuitOnTimeouts()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// this should start as allowing requests
Assert.True(cb.AllowRequest);
Assert.False(cb.IsOpen());
// timeouts
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
// everything has been a timeout so we should not allow any requests
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
}
public void testCurrentConcurrentExecutionCount()
{
CommandMetrics metrics = null;
var ctx = new JellyfishContext();
int NUM_CMDS = 10;
Task[] tasks = new Task[NUM_CMDS];
for (int i = 0; i < NUM_CMDS; i++)
{
LatentCommand cmd = new LatentCommand(ctx, 400);
if (metrics == null)
{
metrics = cmd.Metrics;
}
tasks[i] = cmd.ExecuteAsync();
}
Task.WaitAll(tasks);
Assert.Equal(NUM_CMDS, metrics.GetRollingMaxConcurrentExecutions());
}
public void testErrorPercentage()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
metrics.MarkSuccess(100);
Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkFailure(1000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
Assert.Equal(25, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkTimeout(5000);
metrics.MarkTimeout(5000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
// latent
clock.Increment(1);
metrics.MarkSuccess(5000);
// 6 success + 1 latent success + 1 failure + 2 timeout = 10 total
// latent success not considered error
// error percentage = 1 failure + 2 timeout / 10
Assert.Equal(30, metrics.GetHealthCounts().ErrorPercentage);
}
private IMetricsPublisherCommand GetOrCreatePublisherForCommand(string commandName, CommandMetrics metrics, ICircuitBreaker circuitBreaker)
{
// attempt to retrieve from cache first
IMetricsPublisherCommand publisher;
if (_commandPublishers.TryGetValue(commandName, out publisher))
{
return(publisher);
}
// it doesn't exist so we need to create it
publisher = _context.GetService <IMetricsPublisherCommand>() ?? new DefaultPublisherCommand();
// attempt to store it (race other threads)
return(_commandPublishers.GetOrAdd(commandName, publisher));
}
internal static ICircuitBreaker GetOrCreateInstance(string name, CommandProperties properties, CommandMetrics metrics, IClock clock)
{
Contract.Assert(!String.IsNullOrEmpty(name));
Contract.Assert(properties != null);
Contract.Assert(metrics != null);
Contract.Assert(clock != null);
return(_circuitBreakers.GetOrAdd(name, n => new DefaultCircuitBreaker(properties, metrics, clock)));
}
public IMetricsPublisherCommand CreateOrRetrievePublisherForCommand(string commandName, CommandMetrics metrics, ICircuitBreaker circuitBreaker)
{
return(GetOrCreatePublisherForCommand(commandName, metrics, circuitBreaker));
}
private T InnerExecute <T>(Func <T> primaryFunction, Func <IEnumerable <Exception>, T> fallbackFunction, CancellationTokenSource cancellationTokenSource)
{
if (!ConfigurationService.GetHystrixCommandEnabled())
{
return(primaryFunction.Invoke());
}
var innerExceptions = new List <Exception>();
Stopwatch userThreadStopWatch = Stopwatch.StartNew();
if (CircuitBreaker.AllowRequest())
{
CommandMetrics.IncrementConcurrentExecutionCount();
ThreadPoolMetrics.MarkThreadExecution();
Stopwatch commandStopWatch = Stopwatch.StartNew();
try
{
var result = timeoutWrapper.Execute(primaryFunction, cancellationTokenSource);
commandStopWatch.Stop();
CircuitBreaker.CloseCircuit();
CommandMetrics.MarkSuccess();
return(result);
}
catch (HystrixTimeoutException hte)
{
commandStopWatch.Stop();
CommandMetrics.MarkTimeout();
innerExceptions.Add(hte);
}
catch (Exception ex)
{
commandStopWatch.Stop();
CommandMetrics.MarkFailure();
CommandMetrics.MarkExceptionThrown();
innerExceptions.Add(ex);
}
finally
{
// track command execution time
commandStopWatch.Stop();
CommandMetrics.AddCommandExecutionTime(commandStopWatch.Elapsed.TotalMilliseconds);
CommandMetrics.DecrementConcurrentExecutionCount();
ThreadPoolMetrics.MarkThreadCompletion();
// track execution time including threading overhead
userThreadStopWatch.Stop();
CommandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds);
}
}
else
{
CommandMetrics.MarkShortCircuited();
// track execution time including threading overhead
userThreadStopWatch.Stop();
CommandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds);
}
T fallbackResult = fallbackFunction.Invoke(innerExceptions);
CommandMetrics.MarkFallbackSuccess();
return(fallbackResult);
}
public IMetricsPublisherCommand CreateOrRetrievePublisherForCommand(string commandName, CommandMetrics metrics, ICircuitBreaker circuitBreaker)
{
return GetOrCreatePublisherForCommand(commandName, metrics, circuitBreaker);
}
internal ServiceCommand(IJellyfishContext context, IClock clock, string commandGroup = null, string commandName = null, string threadPoolKey = null, CommandPropertiesBuilder properties = null, ICircuitBreaker circuitBreaker = null, CommandMetrics metrics = null, CommandExecutionHook executionHook = null)
{
Contract.Requires(context != null);
CommandState state = _states.GetOrAdd(this.GetType().FullName, (n) => ServiceCommandHelper.PrepareInternal(this.GetType(), context, commandGroup, commandName, properties, clock, metrics, circuitBreaker));
if (String.IsNullOrEmpty(state.CommandGroup))
{
throw new ArgumentException("commandGroup can not be null or empty.");
}
Logger = context.GetService <ILoggerFactory>()?.CreateLogger(this.GetType().FullName) ?? EmptyLogger.Instance;
CommandGroup = state.CommandGroup;
CommandName = state.CommandName;
_clock = clock ?? Clock.GetInstance(); // for test
Properties = properties?.Build(CommandName) ?? new CommandProperties(CommandName);
Metrics = metrics ?? CommandMetricsFactory.GetInstance(CommandName, CommandGroup, Properties, _clock);
_circuitBreaker = circuitBreaker ?? (Properties.CircuitBreakerEnabled.Value ? CircuitBreakerFactory.GetOrCreateInstance(CommandName, Properties, Metrics, _clock) : new NoOpCircuitBreaker());
context.MetricsPublisher.CreateOrRetrievePublisherForCommand(CommandGroup, Metrics, _circuitBreaker);
this._flags = state.Flags;
_threadPoolKey = threadPoolKey ?? CommandGroup;
_executionResult = new ExecutionResult();
_executionHook = executionHook ?? context.CommandExecutionHook;
var executionPolicy = Properties.ExecutionIsolationStrategy.Value;
if (executionPolicy == ExecutionIsolationStrategy.Semaphore)
{
_flags |= ServiceCommandOptions.SemaphoreExecutionStrategy;
}
if (executionPolicy == ExecutionIsolationStrategy.Thread)
{
_flags |= ServiceCommandOptions.ThreadExecutionStrategy;
}
if (Properties.RequestLogEnabled.Value)
{
_currentRequestLog = context.GetRequestLog();
}
if ((_flags & ServiceCommandOptions.HasCacheKey) == ServiceCommandOptions.HasCacheKey && Properties.RequestCacheEnabled.Value)
{
_requestCache = context.GetCache <CacheItem>(CommandName);
}
}
internal static CommandState PrepareInternal(Type commandType, IJellyfishContext context, string commandGroup, string commandName, CommandPropertiesBuilder propertiesBuilder = null, IClock clock = null, CommandMetrics metrics = null, ICircuitBreaker circuitBreaker = null)
{
var state = new CommandState();
state.CommandName = commandName ?? commandType.FullName;
state.CommandGroup = commandGroup ?? state.CommandName;
clock = clock ?? Clock.GetInstance();
var properties = propertiesBuilder?.Build(state.CommandName) ?? new CommandProperties(state.CommandName);
metrics = metrics ?? CommandMetricsFactory.GetInstance(state.CommandName, state.CommandGroup, properties, clock);
circuitBreaker = circuitBreaker ?? (properties.CircuitBreakerEnabled.Value ? CircuitBreakerFactory.GetOrCreateInstance(state.CommandName, properties, metrics, clock) : new NoOpCircuitBreaker());
context.MetricsPublisher.CreateOrRetrievePublisherForCommand(state.CommandGroup, metrics, circuitBreaker);
ServiceCommandOptions flags = ServiceCommandOptions.None;
if (IsMethodImplemented(commandType, "GetFallback"))
{
flags |= ServiceCommandOptions.HasFallBack;
}
if (IsMethodImplemented(commandType, "GetCacheKey"))
{
flags |= ServiceCommandOptions.HasCacheKey;
}
state.Flags = flags;
return(state);
}
public TestCircuitBreaker(IClock clock = null)
{
Clock = clock ?? Jellyfish.Commands.Utils.Clock.GetInstance();
this.Metrics = CircuitBreakerTests.getMetrics(CommandPropertiesTest.GetUnitTestPropertiesSetter(), Clock);
forceShortCircuit = false;
}
internal DefaultCircuitBreaker([NotNull] CommandProperties properties, [NotNull] CommandMetrics metrics, [NotNull] IClock clock)
{
_clock = clock;
_properties = properties;
_metrics = metrics;
}
public IMetricsPublisherCommand Create(string commandName, CommandMetrics metrics, ICircuitBreaker circuitBreaker, CommandProperties properties)
{
return new JsonMetricsPublisherCommand(commandName, metrics, circuitBreaker, properties);
}
private String GetCommandJson(CommandMetrics commandMetrics)
{
var circuitBreaker = CircuitBreakerFactory.GetInstance(commandMetrics.CommandName);
var sb = new StringBuilder(1024);
StringWriter sw = new StringWriter(sb, CultureInfo.InvariantCulture);
var json = new JsonWriter(sw);
json.writeStartObject();
json.writeStringField("type", "HystrixCommand");
json.writeStringField("name", commandMetrics.CommandName);
// json.writeStringField("group", commandMetrics.CommandName);
json.writeNumberField("currentTime", DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond); // TODO check this
// circuit breaker
if (circuitBreaker == null)
{
// circuit breaker is disabled and thus never open
json.writeBooleanField("isCircuitBreakerOpen", false);
}
else
{
json.writeBooleanField("isCircuitBreakerOpen", circuitBreaker.IsOpen());
}
HealthCounts healthCounts = commandMetrics.GetHealthCounts();
json.writeNumberField("errorPercentage", healthCounts.ErrorPercentage);
json.writeNumberField("errorCount", healthCounts.ErrorCount);
json.writeNumberField("requestCount", healthCounts.TotalRequests);
// rolling counters
json.writeNumberField("rollingCountBadRequests", commandMetrics.GetRollingCount(RollingNumberEvent.BAD_REQUEST));
// json.writeNumberField("rollingCountCollapsedRequests", commandMetrics.GetRollingCount(RollingNumberEvent.COLLAPSED));
//json.writeNumberField("rollingCountEmit", commandMetrics.GetRollingCount(RollingNumberEvent.EMIT));
json.writeNumberField("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(RollingNumberEvent.EXCEPTION_THROWN));
json.writeNumberField("rollingCountFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FAILURE));
//json.writeNumberField("rollingCountEmit", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_EMIT));
json.writeNumberField("rollingCountFallbackFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_FAILURE));
json.writeNumberField("rollingCountFallbackRejection", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_REJECTION));
json.writeNumberField("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_SUCCESS));
json.writeNumberField("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(RollingNumberEvent.RESPONSE_FROM_CACHE));
json.writeNumberField("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(RollingNumberEvent.SEMAPHORE_REJECTED));
json.writeNumberField("rollingCountShortCircuited", commandMetrics.GetRollingCount(RollingNumberEvent.SHORT_CIRCUITED));
json.writeNumberField("rollingCountSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.SUCCESS));
json.writeNumberField("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(RollingNumberEvent.THREAD_POOL_REJECTED));
json.writeNumberField("rollingCountTimeout", commandMetrics.GetRollingCount(RollingNumberEvent.TIMEOUT));
json.writeNumberField("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount);
json.writeNumberField("rollingMaxConcurrentExecutionCount", commandMetrics.GetRollingMaxConcurrentExecutions());
// latency percentiles
json.writeNumberField("latencyExecute_mean", commandMetrics.GetExecutionTimeMean());
json.writeObjectFieldStart("latencyExecute");
json.writeNumberField("0", commandMetrics.GetExecutionTimePercentile(0));
json.writeNumberField("25", commandMetrics.GetExecutionTimePercentile(25));
json.writeNumberField("50", commandMetrics.GetExecutionTimePercentile(50));
json.writeNumberField("75", commandMetrics.GetExecutionTimePercentile(75));
json.writeNumberField("90", commandMetrics.GetExecutionTimePercentile(90));
json.writeNumberField("95", commandMetrics.GetExecutionTimePercentile(95));
json.writeNumberField("99", commandMetrics.GetExecutionTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.GetExecutionTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.GetExecutionTimePercentile(100));
json.writeEndObject();
//
json.writeNumberField("latencyTotal_mean", commandMetrics.GetTotalTimeMean());
json.writeObjectFieldStart("latencyTotal");
json.writeNumberField("0", commandMetrics.GetTotalTimePercentile(0));
json.writeNumberField("25", commandMetrics.GetTotalTimePercentile(25));
json.writeNumberField("50", commandMetrics.GetTotalTimePercentile(50));
json.writeNumberField("75", commandMetrics.GetTotalTimePercentile(75));
json.writeNumberField("90", commandMetrics.GetTotalTimePercentile(90));
json.writeNumberField("95", commandMetrics.GetTotalTimePercentile(95));
json.writeNumberField("99", commandMetrics.GetTotalTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.GetTotalTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.GetTotalTimePercentile(100));
json.writeEndObject();
// property values for reporting what is actually seen by the command rather than what was set somewhere
var commandProperties = commandMetrics.Properties;
json.writeNumberField("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold.Get());
json.writeNumberField("propertyValue_circuitBreakerSleepWindowInMilliseconds", commandProperties.CircuitBreakerSleepWindowInMilliseconds.Get());
json.writeNumberField("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage.Get());
json.writeBooleanField("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen.Get());
json.writeBooleanField("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed.Get());
json.writeBooleanField("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled.Get());
//json.writeStringField("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy().get().name());
//json.writeNumberField("propertyValue_executionIsolationThreadTimeoutInMilliseconds", commandProperties.ExecutionTimeoutInMilliseconds().Get());
json.writeNumberField("propertyValue_executionTimeoutInMilliseconds", commandProperties.ExecutionTimeoutInMilliseconds.Get());
//json.writeBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", commandProperties.executionIsolationThreadInterruptOnTimeout().get());
//json.writeStringField("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.executionIsolationThreadPoolKeyOverride().get());
json.writeNumberField("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Get());
json.writeNumberField("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests.Get());
json.writeNumberField("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get());
//json.writeBooleanField("propertyValue_requestCacheEnabled", commandProperties.requestCacheEnabled().get());
//json.writeBooleanField("propertyValue_requestLogEnabled", commandProperties.requestLogEnabled().get());
json.writeNumberField("reportingHosts", 1); // this will get summed across all instances in a cluster
//json.writeStringField("threadPool", commandMetrics.getThreadPoolKey().name());
/*
* The following are commented out as these rarely change and are verbose for streaming for something people don't change.
* We could perhaps allow a property or request argument to include these.
*/
// json.put("propertyValue_metricsRollingPercentileEnabled", commandProperties.metricsRollingPercentileEnabled().get());
// json.put("propertyValue_metricsRollingPercentileBucketSize", commandProperties.metricsRollingPercentileBucketSize().get());
// json.put("propertyValue_metricsRollingPercentileWindow", commandProperties.metricsRollingPercentileWindowInMilliseconds().get());
// json.put("propertyValue_metricsRollingPercentileWindowBuckets", commandProperties.metricsRollingPercentileWindowBuckets().get());
// json.put("propertyValue_metricsRollingStatisticalWindowBuckets", commandProperties.metricsRollingStatisticalWindowBuckets().get());
json.writeEndObject();
return sb.ToString();
}
public void testMultipleTimeWindowRetriesBeforeClosingCircuit()
{
int sleepWindow = 200;
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
ICircuitBreaker cb = getCircuitBreaker("KEY_ONE", "OWNER_TWO", metrics, properties, clock);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.False(cb.AllowRequest);
Assert.True(cb.IsOpen());
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' fails so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep
metrics.MarkFailure(1000);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// the 'singleTest' fails again so it should go back to Sleep and not allow any requests again until another 'singleTest' after the Sleep
metrics.MarkFailure(1000);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
Assert.False(cb.AllowRequest);
// wait for sleepWindow to pass
clock.Increment(sleepWindow + 50);
// we should now allow 1 request
Assert.True(cb.AllowRequest);
// but the circuit should still be open
Assert.True(cb.IsOpen());
// and further requests are still blocked
Assert.False(cb.AllowRequest);
// now it finally succeeds
metrics.MarkSuccess(200);
cb.MarkSuccess();
// all requests should be open again
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
Assert.True(cb.AllowRequest);
// and the circuit should be closed again
Assert.False(cb.IsOpen());
}
private String GetCommandJson(CommandMetrics commandMetrics)
{
var circuitBreaker = this.circuitBreaker ?? CircuitBreakerFactory.GetInstance(commandMetrics.CommandName);
var sb = new StringBuilder(1024);
StringWriter sw = new StringWriter(sb, CultureInfo.InvariantCulture);
var json = new JsonWriter(sw);
json.writeStartObject();
json.writeStringField("type", "ServiceCommand");
json.writeStringField("name", commandMetrics.CommandName);
json.writeStringField("group", commandMetrics.CommandGroup);
json.writeNumberField("currentTime", DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond); // TODO check this
// circuit breaker
if (circuitBreaker == null)
{
// circuit breaker is disabled and thus never open
json.writeBooleanField("isCircuitBreakerOpen", false);
}
else
{
json.writeBooleanField("isCircuitBreakerOpen", circuitBreaker.IsOpen());
}
HealthCounts healthCounts = commandMetrics.GetHealthCounts();
json.writeNumberField("errorPercentage", healthCounts.ErrorPercentage);
json.writeNumberField("errorCount", healthCounts.ErrorCount);
json.writeNumberField("requestCount", healthCounts.TotalRequests);
// rolling counters
json.writeNumberField("rollingCountBadRequests", commandMetrics.GetRollingCount(RollingNumberEvent.BAD_REQUEST));
json.writeNumberField("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(RollingNumberEvent.EXCEPTION_THROWN));
json.writeNumberField("rollingCountFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FAILURE));
json.writeNumberField("rollingCountFallbackFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_FAILURE));
json.writeNumberField("rollingCountFallbackRejection", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_REJECTION));
json.writeNumberField("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_SUCCESS));
json.writeNumberField("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(RollingNumberEvent.RESPONSE_FROM_CACHE));
json.writeNumberField("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(RollingNumberEvent.SEMAPHORE_REJECTED));
json.writeNumberField("rollingCountShortCircuited", commandMetrics.GetRollingCount(RollingNumberEvent.SHORT_CIRCUITED));
json.writeNumberField("rollingCountSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.SUCCESS));
json.writeNumberField("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(RollingNumberEvent.THREAD_POOL_REJECTED));
json.writeNumberField("rollingCountTimeout", commandMetrics.GetRollingCount(RollingNumberEvent.TIMEOUT));
json.writeNumberField("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount);
json.writeNumberField("rollingMaxConcurrentExecutionCount", commandMetrics.GetRollingMaxConcurrentExecutions());
// latency percentiles
json.writeNumberField("latencyExecute_mean", commandMetrics.GetExecutionTimeMean());
json.writeObjectFieldStart("latencyExecute");
json.writeNumberField("0", commandMetrics.GetExecutionTimePercentile(0));
json.writeNumberField("25", commandMetrics.GetExecutionTimePercentile(25));
json.writeNumberField("50", commandMetrics.GetExecutionTimePercentile(50));
json.writeNumberField("75", commandMetrics.GetExecutionTimePercentile(75));
json.writeNumberField("90", commandMetrics.GetExecutionTimePercentile(90));
json.writeNumberField("95", commandMetrics.GetExecutionTimePercentile(95));
json.writeNumberField("99", commandMetrics.GetExecutionTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.GetExecutionTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.GetExecutionTimePercentile(100));
json.writeEndObject();
//
json.writeNumberField("latencyTotal_mean", commandMetrics.GetTotalTimeMean());
json.writeObjectFieldStart("latencyTotal");
json.writeNumberField("0", commandMetrics.GetTotalTimePercentile(0));
json.writeNumberField("25", commandMetrics.GetTotalTimePercentile(25));
json.writeNumberField("50", commandMetrics.GetTotalTimePercentile(50));
json.writeNumberField("75", commandMetrics.GetTotalTimePercentile(75));
json.writeNumberField("90", commandMetrics.GetTotalTimePercentile(90));
json.writeNumberField("95", commandMetrics.GetTotalTimePercentile(95));
json.writeNumberField("99", commandMetrics.GetTotalTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.GetTotalTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.GetTotalTimePercentile(100));
json.writeEndObject();
// property values for reporting what is actually seen by the command rather than what was set somewhere
var commandProperties = commandMetrics.Properties;
json.writeNumberField("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold.Value);
json.writeNumberField("propertyValue_circuitBreakerSleepWindowInMilliseconds", commandProperties.CircuitBreakerSleepWindowInMilliseconds.Value);
json.writeNumberField("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage.Value);
json.writeBooleanField("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen.Value);
json.writeBooleanField("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed.Value);
json.writeBooleanField("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled.Value);
json.writeStringField("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy.Value.ToString());
json.writeNumberField("propertyValue_executionIsolationThreadTimeoutInMilliseconds", commandProperties.ExecutionIsolationThreadTimeoutInMilliseconds.Value);
json.writeNumberField("propertyValue_executionTimeoutInMilliseconds", commandProperties.ExecutionIsolationThreadTimeoutInMilliseconds.Value);
//json.writeBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", commandProperties.executionIsolationThreadInterruptOnTimeout().get());
//json.writeStringField("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.executionIsolationThreadPoolKeyOverride().get());
json.writeNumberField("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Value);
json.writeNumberField("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests.Value);
json.writeNumberField("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Value);
json.writeBooleanField("propertyValue_requestCacheEnabled", commandProperties.RequestCacheEnabled.Value);
json.writeBooleanField("propertyValue_requestLogEnabled", commandProperties.RequestLogEnabled.Value);
json.writeNumberField("reportingHosts", 1); // this will get summed across all instances in a cluster
//json.writeStringField("threadPool", commandMetrics.getThreadPoolKey().name());
// Hystrix specific
json.writeNumberField("rollingCountCollapsedRequests", 0);
json.writeBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", false);
json.writeEndObject();
return(sb.ToString());
}
private static ICircuitBreaker getCircuitBreaker(string key, string commandGroup, CommandMetrics metrics, CommandPropertiesBuilder properties, IClock clock)
{
return(new DefaultCircuitBreaker(properties.Build(key), metrics, clock));
}
