public void testBadRequestsDoNotAffectErrorPercentage() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock);; metrics.MarkSuccess(100); Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage); metrics.MarkFailure(1000); Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage); metrics.MarkBadRequest(1); metrics.MarkBadRequest(2); Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage); metrics.MarkFailure(45); metrics.MarkFailure(55); Assert.Equal(75, metrics.GetHealthCounts().ErrorPercentage); }
public void testErrorPercentage() { var properties = CommandPropertiesTest.GetUnitTestPropertiesSetter(); var clock = new MockedClock(); CommandMetrics metrics = getMetrics(properties, clock); metrics.MarkSuccess(100); Assert.Equal(0, metrics.GetHealthCounts().ErrorPercentage); clock.Increment(1); metrics.MarkFailure(1000); Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage); clock.Increment(1); metrics.MarkSuccess(100); metrics.MarkSuccess(100); Assert.Equal(25, metrics.GetHealthCounts().ErrorPercentage); clock.Increment(1); metrics.MarkTimeout(5000); metrics.MarkTimeout(5000); Assert.Equal(50, metrics.GetHealthCounts().ErrorPercentage); clock.Increment(1); metrics.MarkSuccess(100); metrics.MarkSuccess(100); metrics.MarkSuccess(100); // latent clock.Increment(1); metrics.MarkSuccess(5000); // 6 success + 1 latent success + 1 failure + 2 timeout = 10 total // latent success not considered error // error percentage = 1 failure + 2 timeout / 10 Assert.Equal(30, metrics.GetHealthCounts().ErrorPercentage); }
public bool IsOpen() { if (Interlocked.Read(ref _circuitOpen) == 1) { // if we're open we immediately return true and don't bother attempting to 'close' ourself as that is left to allowSingleTest and a subsequent successful test to close return(true); } // we're closed, so let's see if errors have made us so we should trip the circuit open HealthCounts health = _metrics.GetHealthCounts(); // check if we are past the statisticalWindowVolumeThreshold if (health.TotalRequests < _properties.CircuitBreakerRequestVolumeThreshold.Value) { // we are not past the minimum volume threshold for the statisticalWindow so we'll return false immediately and not calculate anything return(false); } if (health.ErrorPercentage < _properties.CircuitBreakerErrorThresholdPercentage.Value) { return(false); } else { // our failure rate is too high, trip the circuit if (Interlocked.CompareExchange(ref _circuitOpen, 1L, 0L) == 0L) { // if the previousValue was false then we want to set the currentTime Interlocked.Exchange(ref _circuitOpenedOrLastTestedTime, _clock.EllapsedTimeInMs); return(true); } else { // How could previousValue be true? If another thread was going through this code at the same time a race-condition could have // caused another thread to set it to true already even though we were in the process of doing the same // In this case, we know the circuit is open, so let the other thread set the currentTime and report back that the circuit is open return(true); } } }
private String GetCommandJson(CommandMetrics commandMetrics) { var circuitBreaker = CircuitBreakerFactory.GetInstance(commandMetrics.CommandName); var sb = new StringBuilder(1024); StringWriter sw = new StringWriter(sb, CultureInfo.InvariantCulture); var json = new JsonWriter(sw); json.writeStartObject(); json.writeStringField("type", "HystrixCommand"); json.writeStringField("name", commandMetrics.CommandName); // json.writeStringField("group", commandMetrics.CommandName); json.writeNumberField("currentTime", DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond); // TODO check this // circuit breaker if (circuitBreaker == null) { // circuit breaker is disabled and thus never open json.writeBooleanField("isCircuitBreakerOpen", false); } else { json.writeBooleanField("isCircuitBreakerOpen", circuitBreaker.IsOpen()); } HealthCounts healthCounts = commandMetrics.GetHealthCounts(); json.writeNumberField("errorPercentage", healthCounts.ErrorPercentage); json.writeNumberField("errorCount", healthCounts.ErrorCount); json.writeNumberField("requestCount", healthCounts.TotalRequests); // rolling counters json.writeNumberField("rollingCountBadRequests", commandMetrics.GetRollingCount(RollingNumberEvent.BAD_REQUEST)); // json.writeNumberField("rollingCountCollapsedRequests", commandMetrics.GetRollingCount(RollingNumberEvent.COLLAPSED)); //json.writeNumberField("rollingCountEmit", commandMetrics.GetRollingCount(RollingNumberEvent.EMIT)); json.writeNumberField("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(RollingNumberEvent.EXCEPTION_THROWN)); json.writeNumberField("rollingCountFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FAILURE)); //json.writeNumberField("rollingCountEmit", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_EMIT)); json.writeNumberField("rollingCountFallbackFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_FAILURE)); json.writeNumberField("rollingCountFallbackRejection", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_REJECTION)); json.writeNumberField("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_SUCCESS)); json.writeNumberField("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(RollingNumberEvent.RESPONSE_FROM_CACHE)); json.writeNumberField("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(RollingNumberEvent.SEMAPHORE_REJECTED)); json.writeNumberField("rollingCountShortCircuited", commandMetrics.GetRollingCount(RollingNumberEvent.SHORT_CIRCUITED)); json.writeNumberField("rollingCountSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.SUCCESS)); json.writeNumberField("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(RollingNumberEvent.THREAD_POOL_REJECTED)); json.writeNumberField("rollingCountTimeout", commandMetrics.GetRollingCount(RollingNumberEvent.TIMEOUT)); json.writeNumberField("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount); json.writeNumberField("rollingMaxConcurrentExecutionCount", commandMetrics.GetRollingMaxConcurrentExecutions()); // latency percentiles json.writeNumberField("latencyExecute_mean", commandMetrics.GetExecutionTimeMean()); json.writeObjectFieldStart("latencyExecute"); json.writeNumberField("0", commandMetrics.GetExecutionTimePercentile(0)); json.writeNumberField("25", commandMetrics.GetExecutionTimePercentile(25)); json.writeNumberField("50", commandMetrics.GetExecutionTimePercentile(50)); json.writeNumberField("75", commandMetrics.GetExecutionTimePercentile(75)); json.writeNumberField("90", commandMetrics.GetExecutionTimePercentile(90)); json.writeNumberField("95", commandMetrics.GetExecutionTimePercentile(95)); json.writeNumberField("99", commandMetrics.GetExecutionTimePercentile(99)); json.writeNumberField("99.5", commandMetrics.GetExecutionTimePercentile(99.5)); json.writeNumberField("100", commandMetrics.GetExecutionTimePercentile(100)); json.writeEndObject(); // json.writeNumberField("latencyTotal_mean", commandMetrics.GetTotalTimeMean()); json.writeObjectFieldStart("latencyTotal"); json.writeNumberField("0", commandMetrics.GetTotalTimePercentile(0)); json.writeNumberField("25", commandMetrics.GetTotalTimePercentile(25)); json.writeNumberField("50", commandMetrics.GetTotalTimePercentile(50)); json.writeNumberField("75", commandMetrics.GetTotalTimePercentile(75)); json.writeNumberField("90", commandMetrics.GetTotalTimePercentile(90)); json.writeNumberField("95", commandMetrics.GetTotalTimePercentile(95)); json.writeNumberField("99", commandMetrics.GetTotalTimePercentile(99)); json.writeNumberField("99.5", commandMetrics.GetTotalTimePercentile(99.5)); json.writeNumberField("100", commandMetrics.GetTotalTimePercentile(100)); json.writeEndObject(); // property values for reporting what is actually seen by the command rather than what was set somewhere var commandProperties = commandMetrics.Properties; json.writeNumberField("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold.Get()); json.writeNumberField("propertyValue_circuitBreakerSleepWindowInMilliseconds", commandProperties.CircuitBreakerSleepWindowInMilliseconds.Get()); json.writeNumberField("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage.Get()); json.writeBooleanField("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen.Get()); json.writeBooleanField("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed.Get()); json.writeBooleanField("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled.Get()); //json.writeStringField("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy().get().name()); //json.writeNumberField("propertyValue_executionIsolationThreadTimeoutInMilliseconds", commandProperties.ExecutionTimeoutInMilliseconds().Get()); json.writeNumberField("propertyValue_executionTimeoutInMilliseconds", commandProperties.ExecutionTimeoutInMilliseconds.Get()); //json.writeBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", commandProperties.executionIsolationThreadInterruptOnTimeout().get()); //json.writeStringField("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.executionIsolationThreadPoolKeyOverride().get()); json.writeNumberField("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Get()); json.writeNumberField("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests.Get()); json.writeNumberField("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get()); //json.writeBooleanField("propertyValue_requestCacheEnabled", commandProperties.requestCacheEnabled().get()); //json.writeBooleanField("propertyValue_requestLogEnabled", commandProperties.requestLogEnabled().get()); json.writeNumberField("reportingHosts", 1); // this will get summed across all instances in a cluster //json.writeStringField("threadPool", commandMetrics.getThreadPoolKey().name()); /* * The following are commented out as these rarely change and are verbose for streaming for something people don't change. * We could perhaps allow a property or request argument to include these. */ // json.put("propertyValue_metricsRollingPercentileEnabled", commandProperties.metricsRollingPercentileEnabled().get()); // json.put("propertyValue_metricsRollingPercentileBucketSize", commandProperties.metricsRollingPercentileBucketSize().get()); // json.put("propertyValue_metricsRollingPercentileWindow", commandProperties.metricsRollingPercentileWindowInMilliseconds().get()); // json.put("propertyValue_metricsRollingPercentileWindowBuckets", commandProperties.metricsRollingPercentileWindowBuckets().get()); // json.put("propertyValue_metricsRollingStatisticalWindowBuckets", commandProperties.metricsRollingStatisticalWindowBuckets().get()); json.writeEndObject(); return sb.ToString(); }
private String GetCommandJson(CommandMetrics commandMetrics) { var circuitBreaker = this.circuitBreaker ?? CircuitBreakerFactory.GetInstance(commandMetrics.CommandName); var sb = new StringBuilder(1024); StringWriter sw = new StringWriter(sb, CultureInfo.InvariantCulture); var json = new JsonWriter(sw); json.writeStartObject(); json.writeStringField("type", "ServiceCommand"); json.writeStringField("name", commandMetrics.CommandName); json.writeStringField("group", commandMetrics.CommandGroup); json.writeNumberField("currentTime", DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond); // TODO check this // circuit breaker if (circuitBreaker == null) { // circuit breaker is disabled and thus never open json.writeBooleanField("isCircuitBreakerOpen", false); } else { json.writeBooleanField("isCircuitBreakerOpen", circuitBreaker.IsOpen()); } HealthCounts healthCounts = commandMetrics.GetHealthCounts(); json.writeNumberField("errorPercentage", healthCounts.ErrorPercentage); json.writeNumberField("errorCount", healthCounts.ErrorCount); json.writeNumberField("requestCount", healthCounts.TotalRequests); // rolling counters json.writeNumberField("rollingCountBadRequests", commandMetrics.GetRollingCount(RollingNumberEvent.BAD_REQUEST)); json.writeNumberField("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(RollingNumberEvent.EXCEPTION_THROWN)); json.writeNumberField("rollingCountFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FAILURE)); json.writeNumberField("rollingCountFallbackFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_FAILURE)); json.writeNumberField("rollingCountFallbackRejection", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_REJECTION)); json.writeNumberField("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_SUCCESS)); json.writeNumberField("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(RollingNumberEvent.RESPONSE_FROM_CACHE)); json.writeNumberField("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(RollingNumberEvent.SEMAPHORE_REJECTED)); json.writeNumberField("rollingCountShortCircuited", commandMetrics.GetRollingCount(RollingNumberEvent.SHORT_CIRCUITED)); json.writeNumberField("rollingCountSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.SUCCESS)); json.writeNumberField("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(RollingNumberEvent.THREAD_POOL_REJECTED)); json.writeNumberField("rollingCountTimeout", commandMetrics.GetRollingCount(RollingNumberEvent.TIMEOUT)); json.writeNumberField("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount); json.writeNumberField("rollingMaxConcurrentExecutionCount", commandMetrics.GetRollingMaxConcurrentExecutions()); // latency percentiles json.writeNumberField("latencyExecute_mean", commandMetrics.GetExecutionTimeMean()); json.writeObjectFieldStart("latencyExecute"); json.writeNumberField("0", commandMetrics.GetExecutionTimePercentile(0)); json.writeNumberField("25", commandMetrics.GetExecutionTimePercentile(25)); json.writeNumberField("50", commandMetrics.GetExecutionTimePercentile(50)); json.writeNumberField("75", commandMetrics.GetExecutionTimePercentile(75)); json.writeNumberField("90", commandMetrics.GetExecutionTimePercentile(90)); json.writeNumberField("95", commandMetrics.GetExecutionTimePercentile(95)); json.writeNumberField("99", commandMetrics.GetExecutionTimePercentile(99)); json.writeNumberField("99.5", commandMetrics.GetExecutionTimePercentile(99.5)); json.writeNumberField("100", commandMetrics.GetExecutionTimePercentile(100)); json.writeEndObject(); // json.writeNumberField("latencyTotal_mean", commandMetrics.GetTotalTimeMean()); json.writeObjectFieldStart("latencyTotal"); json.writeNumberField("0", commandMetrics.GetTotalTimePercentile(0)); json.writeNumberField("25", commandMetrics.GetTotalTimePercentile(25)); json.writeNumberField("50", commandMetrics.GetTotalTimePercentile(50)); json.writeNumberField("75", commandMetrics.GetTotalTimePercentile(75)); json.writeNumberField("90", commandMetrics.GetTotalTimePercentile(90)); json.writeNumberField("95", commandMetrics.GetTotalTimePercentile(95)); json.writeNumberField("99", commandMetrics.GetTotalTimePercentile(99)); json.writeNumberField("99.5", commandMetrics.GetTotalTimePercentile(99.5)); json.writeNumberField("100", commandMetrics.GetTotalTimePercentile(100)); json.writeEndObject(); // property values for reporting what is actually seen by the command rather than what was set somewhere var commandProperties = commandMetrics.Properties; json.writeNumberField("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold.Value); json.writeNumberField("propertyValue_circuitBreakerSleepWindowInMilliseconds", commandProperties.CircuitBreakerSleepWindowInMilliseconds.Value); json.writeNumberField("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage.Value); json.writeBooleanField("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen.Value); json.writeBooleanField("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed.Value); json.writeBooleanField("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled.Value); json.writeStringField("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy.Value.ToString()); json.writeNumberField("propertyValue_executionIsolationThreadTimeoutInMilliseconds", commandProperties.ExecutionIsolationThreadTimeoutInMilliseconds.Value); json.writeNumberField("propertyValue_executionTimeoutInMilliseconds", commandProperties.ExecutionIsolationThreadTimeoutInMilliseconds.Value); //json.writeBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", commandProperties.executionIsolationThreadInterruptOnTimeout().get()); //json.writeStringField("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.executionIsolationThreadPoolKeyOverride().get()); json.writeNumberField("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Value); json.writeNumberField("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests.Value); json.writeNumberField("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Value); json.writeBooleanField("propertyValue_requestCacheEnabled", commandProperties.RequestCacheEnabled.Value); json.writeBooleanField("propertyValue_requestLogEnabled", commandProperties.RequestLogEnabled.Value); json.writeNumberField("reportingHosts", 1); // this will get summed across all instances in a cluster //json.writeStringField("threadPool", commandMetrics.getThreadPoolKey().name()); // Hystrix specific json.writeNumberField("rollingCountCollapsedRequests", 0); json.writeBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", false); json.writeEndObject(); return(sb.ToString()); }