public void testBadRequestsDoNotAffectErrorPercentage()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);;
metrics.MarkSuccess(100);
Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage);
metrics.MarkFailure(1000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
metrics.MarkBadRequest(1);
metrics.MarkBadRequest(2);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
metrics.MarkFailure(45);
metrics.MarkFailure(55);
Assert.Equal(75, metrics.GetHealthCounts().ErrorPercentage);
}
public void testErrorPercentage()
{
var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter();
var clock = new MockedClock();
CommandMetrics metrics = getMetrics(properties, clock);
metrics.MarkSuccess(100);
Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkFailure(1000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
Assert.Equal(25, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkTimeout(5000);
metrics.MarkTimeout(5000);
Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage);
clock.Increment(1);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
metrics.MarkSuccess(100);
// latent
clock.Increment(1);
metrics.MarkSuccess(5000);
// 6 success + 1 latent success + 1 failure + 2 timeout = 10 total
// latent success not considered error
// error percentage = 1 failure + 2 timeout / 10
Assert.Equal(30, metrics.GetHealthCounts().ErrorPercentage);
}
public bool IsOpen()
{
if (Interlocked.Read(ref _circuitOpen) == 1)
{
// if we're open we immediately return true and don't bother attempting to 'close' ourself as that is left to allowSingleTest and a subsequent successful test to close
return(true);
}
// we're closed, so let's see if errors have made us so we should trip the circuit open
HealthCounts health = _metrics.GetHealthCounts();
// check if we are past the statisticalWindowVolumeThreshold
if (health.TotalRequests < _properties.CircuitBreakerRequestVolumeThreshold.Value)
{
// we are not past the minimum volume threshold for the statisticalWindow so we'll return false immediately and not calculate anything
return(false);
}
if (health.ErrorPercentage < _properties.CircuitBreakerErrorThresholdPercentage.Value)
{
return(false);
}
else
{
// our failure rate is too high, trip the circuit
if (Interlocked.CompareExchange(ref _circuitOpen, 1L, 0L) == 0L)
{
// if the previousValue was false then we want to set the currentTime
Interlocked.Exchange(ref _circuitOpenedOrLastTestedTime, _clock.EllapsedTimeInMs);
return(true);
}
else
{
// How could previousValue be true? If another thread was going through this code at the same time a race-condition could have
// caused another thread to set it to true already even though we were in the process of doing the same
// In this case, we know the circuit is open, so let the other thread set the currentTime and report back that the circuit is open
return(true);
}
}
}
private String GetCommandJson(CommandMetrics commandMetrics)
{
var circuitBreaker = CircuitBreakerFactory.GetInstance(commandMetrics.CommandName);
var sb = new StringBuilder(1024);
StringWriter sw = new StringWriter(sb, CultureInfo.InvariantCulture);
var json = new JsonWriter(sw);
json.writeStartObject();
json.writeStringField("type", "HystrixCommand");
json.writeStringField("name", commandMetrics.CommandName);
// json.writeStringField("group", commandMetrics.CommandName);
json.writeNumberField("currentTime", DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond); // TODO check this
// circuit breaker
if (circuitBreaker == null)
{
// circuit breaker is disabled and thus never open
json.writeBooleanField("isCircuitBreakerOpen", false);
}
else
{
json.writeBooleanField("isCircuitBreakerOpen", circuitBreaker.IsOpen());
}
HealthCounts healthCounts = commandMetrics.GetHealthCounts();
json.writeNumberField("errorPercentage", healthCounts.ErrorPercentage);
json.writeNumberField("errorCount", healthCounts.ErrorCount);
json.writeNumberField("requestCount", healthCounts.TotalRequests);
// rolling counters
json.writeNumberField("rollingCountBadRequests", commandMetrics.GetRollingCount(RollingNumberEvent.BAD_REQUEST));
// json.writeNumberField("rollingCountCollapsedRequests", commandMetrics.GetRollingCount(RollingNumberEvent.COLLAPSED));
//json.writeNumberField("rollingCountEmit", commandMetrics.GetRollingCount(RollingNumberEvent.EMIT));
json.writeNumberField("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(RollingNumberEvent.EXCEPTION_THROWN));
json.writeNumberField("rollingCountFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FAILURE));
//json.writeNumberField("rollingCountEmit", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_EMIT));
json.writeNumberField("rollingCountFallbackFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_FAILURE));
json.writeNumberField("rollingCountFallbackRejection", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_REJECTION));
json.writeNumberField("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_SUCCESS));
json.writeNumberField("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(RollingNumberEvent.RESPONSE_FROM_CACHE));
json.writeNumberField("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(RollingNumberEvent.SEMAPHORE_REJECTED));
json.writeNumberField("rollingCountShortCircuited", commandMetrics.GetRollingCount(RollingNumberEvent.SHORT_CIRCUITED));
json.writeNumberField("rollingCountSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.SUCCESS));
json.writeNumberField("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(RollingNumberEvent.THREAD_POOL_REJECTED));
json.writeNumberField("rollingCountTimeout", commandMetrics.GetRollingCount(RollingNumberEvent.TIMEOUT));
json.writeNumberField("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount);
json.writeNumberField("rollingMaxConcurrentExecutionCount", commandMetrics.GetRollingMaxConcurrentExecutions());
// latency percentiles
json.writeNumberField("latencyExecute_mean", commandMetrics.GetExecutionTimeMean());
json.writeObjectFieldStart("latencyExecute");
json.writeNumberField("0", commandMetrics.GetExecutionTimePercentile(0));
json.writeNumberField("25", commandMetrics.GetExecutionTimePercentile(25));
json.writeNumberField("50", commandMetrics.GetExecutionTimePercentile(50));
json.writeNumberField("75", commandMetrics.GetExecutionTimePercentile(75));
json.writeNumberField("90", commandMetrics.GetExecutionTimePercentile(90));
json.writeNumberField("95", commandMetrics.GetExecutionTimePercentile(95));
json.writeNumberField("99", commandMetrics.GetExecutionTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.GetExecutionTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.GetExecutionTimePercentile(100));
json.writeEndObject();
//
json.writeNumberField("latencyTotal_mean", commandMetrics.GetTotalTimeMean());
json.writeObjectFieldStart("latencyTotal");
json.writeNumberField("0", commandMetrics.GetTotalTimePercentile(0));
json.writeNumberField("25", commandMetrics.GetTotalTimePercentile(25));
json.writeNumberField("50", commandMetrics.GetTotalTimePercentile(50));
json.writeNumberField("75", commandMetrics.GetTotalTimePercentile(75));
json.writeNumberField("90", commandMetrics.GetTotalTimePercentile(90));
json.writeNumberField("95", commandMetrics.GetTotalTimePercentile(95));
json.writeNumberField("99", commandMetrics.GetTotalTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.GetTotalTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.GetTotalTimePercentile(100));
json.writeEndObject();
// property values for reporting what is actually seen by the command rather than what was set somewhere
var commandProperties = commandMetrics.Properties;
json.writeNumberField("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold.Get());
json.writeNumberField("propertyValue_circuitBreakerSleepWindowInMilliseconds", commandProperties.CircuitBreakerSleepWindowInMilliseconds.Get());
json.writeNumberField("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage.Get());
json.writeBooleanField("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen.Get());
json.writeBooleanField("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed.Get());
json.writeBooleanField("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled.Get());
//json.writeStringField("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy().get().name());
//json.writeNumberField("propertyValue_executionIsolationThreadTimeoutInMilliseconds", commandProperties.ExecutionTimeoutInMilliseconds().Get());
json.writeNumberField("propertyValue_executionTimeoutInMilliseconds", commandProperties.ExecutionTimeoutInMilliseconds.Get());
//json.writeBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", commandProperties.executionIsolationThreadInterruptOnTimeout().get());
//json.writeStringField("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.executionIsolationThreadPoolKeyOverride().get());
json.writeNumberField("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Get());
json.writeNumberField("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests.Get());
json.writeNumberField("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get());
//json.writeBooleanField("propertyValue_requestCacheEnabled", commandProperties.requestCacheEnabled().get());
//json.writeBooleanField("propertyValue_requestLogEnabled", commandProperties.requestLogEnabled().get());
json.writeNumberField("reportingHosts", 1); // this will get summed across all instances in a cluster
//json.writeStringField("threadPool", commandMetrics.getThreadPoolKey().name());
/*
* The following are commented out as these rarely change and are verbose for streaming for something people don't change.
* We could perhaps allow a property or request argument to include these.
*/
// json.put("propertyValue_metricsRollingPercentileEnabled", commandProperties.metricsRollingPercentileEnabled().get());
// json.put("propertyValue_metricsRollingPercentileBucketSize", commandProperties.metricsRollingPercentileBucketSize().get());
// json.put("propertyValue_metricsRollingPercentileWindow", commandProperties.metricsRollingPercentileWindowInMilliseconds().get());
// json.put("propertyValue_metricsRollingPercentileWindowBuckets", commandProperties.metricsRollingPercentileWindowBuckets().get());
// json.put("propertyValue_metricsRollingStatisticalWindowBuckets", commandProperties.metricsRollingStatisticalWindowBuckets().get());
json.writeEndObject();
return sb.ToString();
}
private String GetCommandJson(CommandMetrics commandMetrics)
{
var circuitBreaker = this.circuitBreaker ?? CircuitBreakerFactory.GetInstance(commandMetrics.CommandName);
var sb = new StringBuilder(1024);
StringWriter sw = new StringWriter(sb, CultureInfo.InvariantCulture);
var json = new JsonWriter(sw);
json.writeStartObject();
json.writeStringField("type", "ServiceCommand");
json.writeStringField("name", commandMetrics.CommandName);
json.writeStringField("group", commandMetrics.CommandGroup);
json.writeNumberField("currentTime", DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond); // TODO check this
// circuit breaker
if (circuitBreaker == null)
{
// circuit breaker is disabled and thus never open
json.writeBooleanField("isCircuitBreakerOpen", false);
}
else
{
json.writeBooleanField("isCircuitBreakerOpen", circuitBreaker.IsOpen());
}
HealthCounts healthCounts = commandMetrics.GetHealthCounts();
json.writeNumberField("errorPercentage", healthCounts.ErrorPercentage);
json.writeNumberField("errorCount", healthCounts.ErrorCount);
json.writeNumberField("requestCount", healthCounts.TotalRequests);
// rolling counters
json.writeNumberField("rollingCountBadRequests", commandMetrics.GetRollingCount(RollingNumberEvent.BAD_REQUEST));
json.writeNumberField("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(RollingNumberEvent.EXCEPTION_THROWN));
json.writeNumberField("rollingCountFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FAILURE));
json.writeNumberField("rollingCountFallbackFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_FAILURE));
json.writeNumberField("rollingCountFallbackRejection", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_REJECTION));
json.writeNumberField("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_SUCCESS));
json.writeNumberField("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(RollingNumberEvent.RESPONSE_FROM_CACHE));
json.writeNumberField("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(RollingNumberEvent.SEMAPHORE_REJECTED));
json.writeNumberField("rollingCountShortCircuited", commandMetrics.GetRollingCount(RollingNumberEvent.SHORT_CIRCUITED));
json.writeNumberField("rollingCountSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.SUCCESS));
json.writeNumberField("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(RollingNumberEvent.THREAD_POOL_REJECTED));
json.writeNumberField("rollingCountTimeout", commandMetrics.GetRollingCount(RollingNumberEvent.TIMEOUT));
json.writeNumberField("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount);
json.writeNumberField("rollingMaxConcurrentExecutionCount", commandMetrics.GetRollingMaxConcurrentExecutions());
// latency percentiles
json.writeNumberField("latencyExecute_mean", commandMetrics.GetExecutionTimeMean());
json.writeObjectFieldStart("latencyExecute");
json.writeNumberField("0", commandMetrics.GetExecutionTimePercentile(0));
json.writeNumberField("25", commandMetrics.GetExecutionTimePercentile(25));
json.writeNumberField("50", commandMetrics.GetExecutionTimePercentile(50));
json.writeNumberField("75", commandMetrics.GetExecutionTimePercentile(75));
json.writeNumberField("90", commandMetrics.GetExecutionTimePercentile(90));
json.writeNumberField("95", commandMetrics.GetExecutionTimePercentile(95));
json.writeNumberField("99", commandMetrics.GetExecutionTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.GetExecutionTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.GetExecutionTimePercentile(100));
json.writeEndObject();
//
json.writeNumberField("latencyTotal_mean", commandMetrics.GetTotalTimeMean());
json.writeObjectFieldStart("latencyTotal");
json.writeNumberField("0", commandMetrics.GetTotalTimePercentile(0));
json.writeNumberField("25", commandMetrics.GetTotalTimePercentile(25));
json.writeNumberField("50", commandMetrics.GetTotalTimePercentile(50));
json.writeNumberField("75", commandMetrics.GetTotalTimePercentile(75));
json.writeNumberField("90", commandMetrics.GetTotalTimePercentile(90));
json.writeNumberField("95", commandMetrics.GetTotalTimePercentile(95));
json.writeNumberField("99", commandMetrics.GetTotalTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.GetTotalTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.GetTotalTimePercentile(100));
json.writeEndObject();
// property values for reporting what is actually seen by the command rather than what was set somewhere
var commandProperties = commandMetrics.Properties;
json.writeNumberField("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold.Value);
json.writeNumberField("propertyValue_circuitBreakerSleepWindowInMilliseconds", commandProperties.CircuitBreakerSleepWindowInMilliseconds.Value);
json.writeNumberField("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage.Value);
json.writeBooleanField("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen.Value);
json.writeBooleanField("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed.Value);
json.writeBooleanField("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled.Value);
json.writeStringField("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy.Value.ToString());
json.writeNumberField("propertyValue_executionIsolationThreadTimeoutInMilliseconds", commandProperties.ExecutionIsolationThreadTimeoutInMilliseconds.Value);
json.writeNumberField("propertyValue_executionTimeoutInMilliseconds", commandProperties.ExecutionIsolationThreadTimeoutInMilliseconds.Value);
//json.writeBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", commandProperties.executionIsolationThreadInterruptOnTimeout().get());
//json.writeStringField("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.executionIsolationThreadPoolKeyOverride().get());
json.writeNumberField("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Value);
json.writeNumberField("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests.Value);
json.writeNumberField("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Value);
json.writeBooleanField("propertyValue_requestCacheEnabled", commandProperties.RequestCacheEnabled.Value);
json.writeBooleanField("propertyValue_requestLogEnabled", commandProperties.RequestLogEnabled.Value);
json.writeNumberField("reportingHosts", 1); // this will get summed across all instances in a cluster
//json.writeStringField("threadPool", commandMetrics.getThreadPoolKey().name());
// Hystrix specific
json.writeNumberField("rollingCountCollapsedRequests", 0);
json.writeBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", false);
json.writeEndObject();
return(sb.ToString());
}