private static void WriteCommandMetrics(JsonTextWriter writer, HystrixCommandMetrics commandMetrics)
        {
            IHystrixCommandKey     key            = commandMetrics.CommandKey;
            IHystrixCircuitBreaker circuitBreaker = HystrixCircuitBreakerFactory.GetInstance(key);

            writer.WriteStartObject();
            writer.WriteStringField("type", "HystrixCommand");
            writer.WriteStringField("name", key.Name);
            writer.WriteStringField("group", commandMetrics.CommandGroup.Name);
            writer.WriteLongField("currentTime", Time.CurrentTimeMillisJava);

            // circuit breaker
            if (circuitBreaker == null)
            {
                // circuit breaker is disabled and thus never open
                writer.WriteBooleanField("isCircuitBreakerOpen", false);
            }
            else
            {
                writer.WriteBooleanField("isCircuitBreakerOpen", circuitBreaker.IsOpen);
            }

            HealthCounts healthCounts = commandMetrics.Healthcounts;

            writer.WriteIntegerField("errorPercentage", healthCounts.ErrorPercentage);
            writer.WriteLongField("errorCount", healthCounts.ErrorCount);
            writer.WriteLongField("requestCount", healthCounts.TotalRequests);

            // rolling counters
            writer.WriteLongField("rollingCountBadRequests", commandMetrics.GetRollingCount(HystrixEventType.BAD_REQUEST));
            writer.WriteLongField("rollingCountCollapsedRequests", commandMetrics.GetRollingCount(HystrixEventType.COLLAPSED));
            writer.WriteLongField("rollingCountEmit", commandMetrics.GetRollingCount(HystrixEventType.EMIT));
            writer.WriteLongField("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(HystrixEventType.EXCEPTION_THROWN));
            writer.WriteLongField("rollingCountFailure", commandMetrics.GetRollingCount(HystrixEventType.FAILURE));
            writer.WriteLongField("rollingCountFallbackEmit", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_EMIT));
            writer.WriteLongField("rollingCountFallbackFailure", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_FAILURE));
            writer.WriteLongField("rollingCountFallbackMissing", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_MISSING));
            writer.WriteLongField("rollingCountFallbackRejection", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_REJECTION));
            writer.WriteLongField("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_SUCCESS));
            writer.WriteLongField("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(HystrixEventType.RESPONSE_FROM_CACHE));
            writer.WriteLongField("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(HystrixEventType.SEMAPHORE_REJECTED));
            writer.WriteLongField("rollingCountShortCircuited", commandMetrics.GetRollingCount(HystrixEventType.SHORT_CIRCUITED));
            writer.WriteLongField("rollingCountSuccess", commandMetrics.GetRollingCount(HystrixEventType.SUCCESS));
            writer.WriteLongField("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(HystrixEventType.THREAD_POOL_REJECTED));
            writer.WriteLongField("rollingCountTimeout", commandMetrics.GetRollingCount(HystrixEventType.TIMEOUT));

            writer.WriteIntegerField("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount);
            writer.WriteLongField("rollingMaxConcurrentExecutionCount", commandMetrics.RollingMaxConcurrentExecutions);

            // latency percentiles
            writer.WriteIntegerField("latencyExecute_mean", commandMetrics.ExecutionTimeMean);
            writer.WriteObjectFieldStart("latencyExecute");
            writer.WriteIntegerField("0", commandMetrics.GetExecutionTimePercentile(0));
            writer.WriteIntegerField("25", commandMetrics.GetExecutionTimePercentile(25));
            writer.WriteIntegerField("50", commandMetrics.GetExecutionTimePercentile(50));
            writer.WriteIntegerField("75", commandMetrics.GetExecutionTimePercentile(75));
            writer.WriteIntegerField("90", commandMetrics.GetExecutionTimePercentile(90));
            writer.WriteIntegerField("95", commandMetrics.GetExecutionTimePercentile(95));
            writer.WriteIntegerField("99", commandMetrics.GetExecutionTimePercentile(99));
            writer.WriteIntegerField("99.5", commandMetrics.GetExecutionTimePercentile(99.5));
            writer.WriteIntegerField("100", commandMetrics.GetExecutionTimePercentile(100));
            writer.WriteEndObject();
            writer.WriteIntegerField("latencyTotal_mean", commandMetrics.TotalTimeMean);
            writer.WriteObjectFieldStart("latencyTotal");
            writer.WriteIntegerField("0", commandMetrics.GetTotalTimePercentile(0));
            writer.WriteIntegerField("25", commandMetrics.GetTotalTimePercentile(25));
            writer.WriteIntegerField("50", commandMetrics.GetTotalTimePercentile(50));
            writer.WriteIntegerField("75", commandMetrics.GetTotalTimePercentile(75));
            writer.WriteIntegerField("90", commandMetrics.GetTotalTimePercentile(90));
            writer.WriteIntegerField("95", commandMetrics.GetTotalTimePercentile(95));
            writer.WriteIntegerField("99", commandMetrics.GetTotalTimePercentile(99));
            writer.WriteIntegerField("99.5", commandMetrics.GetTotalTimePercentile(99.5));
            writer.WriteIntegerField("100", commandMetrics.GetTotalTimePercentile(100));
            writer.WriteEndObject();

            // property values for reporting what is actually seen by the command rather than what was set somewhere
            IHystrixCommandOptions commandProperties = commandMetrics.Properties;

            writer.WriteIntegerField("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold);
            writer.WriteIntegerField("propertyValue_circuitBreakerSleepWindowInMilliseconds", commandProperties.CircuitBreakerSleepWindowInMilliseconds);
            writer.WriteIntegerField("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage);
            writer.WriteBooleanField("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen);
            writer.WriteBooleanField("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed);
            writer.WriteBooleanField("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled);

            writer.WriteStringField("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy.ToString());
            writer.WriteIntegerField("propertyValue_executionIsolationThreadTimeoutInMilliseconds", commandProperties.ExecutionTimeoutInMilliseconds);
            writer.WriteIntegerField("propertyValue_executionTimeoutInMilliseconds", commandProperties.ExecutionTimeoutInMilliseconds);
            writer.WriteBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", false);
            writer.WriteStringField("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.ExecutionIsolationThreadPoolKeyOverride);
            writer.WriteIntegerField("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests);
            writer.WriteIntegerField("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests);
            writer.WriteIntegerField("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds);
            writer.WriteBooleanField("propertyValue_requestCacheEnabled", commandProperties.RequestCacheEnabled);
            writer.WriteBooleanField("propertyValue_requestLogEnabled", commandProperties.RequestLogEnabled);
            writer.WriteIntegerField("reportingHosts", 1); // this will get summed across all instances in a cluster
            writer.WriteStringField("threadPool", commandMetrics.ThreadPoolKey.Name);
            writer.WriteEndObject();
        }
Exemple #2
0
        /// <summary>
        /// Produces JSON formatted metrics data from an instance of <see cref="HystrixCommandMetrics"/>.
        /// </summary>
        /// <param name="commandMetrics">The metrics of a command.</param>
        /// <returns>JSON formatted metrics data.</returns>
        private static string CreateCommandSampleData(HystrixCommandMetrics commandMetrics)
        {
            IHystrixCircuitBreaker    circuitBreaker    = HystrixCircuitBreakerFactory.GetInstance(commandMetrics.CommandKey);
            HealthCounts              healthCounts      = commandMetrics.GetHealthCounts();
            IHystrixCommandProperties commandProperties = commandMetrics.Properties;

            JObject data = new JObject(
                new JProperty("type", "HystrixCommand"),
                new JProperty("name", commandMetrics.CommandKey.Name),
                new JProperty("group", commandMetrics.CommandGroup.Name),
                new JProperty("currentTime", GetCurrentTimeForJavascript()),
                circuitBreaker == null ? new JProperty("isCircuitBreakerOpen", false) : new JProperty("isCircuitBreakerOpen", circuitBreaker.IsOpen()),
                new JProperty("errorPercentage", healthCounts.ErrorPercentage), // health counts
                new JProperty("errorCount", healthCounts.ErrorCount),
                new JProperty("requestCount", healthCounts.TotalRequests),
                new JProperty("rollingCountCollapsedRequests", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Collapsed)), // rolling counters
                new JProperty("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown)),
                new JProperty("rollingCountFailure", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Failure)),
                new JProperty("rollingCountFallbackFailure", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure)),
                new JProperty("rollingCountFallbackRejection", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection)),
                new JProperty("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess)),
                new JProperty("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache)),
                new JProperty("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected)),
                new JProperty("rollingCountShortCircuited", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited)),
                new JProperty("rollingCountSuccess", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Success)),
                new JProperty("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected)),
                new JProperty("rollingCountTimeout", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Timeout)),
                new JProperty("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount),
                new JProperty("latencyExecute_mean", commandMetrics.GetExecutionTimeMean()), // latency percentiles
                new JProperty(
                    "latencyExecute",
                    new JObject(
                        new JProperty("0", commandMetrics.GetExecutionTimePercentile(0)),
                        new JProperty("25", commandMetrics.GetExecutionTimePercentile(25)),
                        new JProperty("50", commandMetrics.GetExecutionTimePercentile(50)),
                        new JProperty("75", commandMetrics.GetExecutionTimePercentile(75)),
                        new JProperty("90", commandMetrics.GetExecutionTimePercentile(90)),
                        new JProperty("95", commandMetrics.GetExecutionTimePercentile(95)),
                        new JProperty("99", commandMetrics.GetExecutionTimePercentile(99)),
                        new JProperty("99.5", commandMetrics.GetExecutionTimePercentile(99.5)),
                        new JProperty("100", commandMetrics.GetExecutionTimePercentile(100)))),
                new JProperty("latencyTotal_mean", commandMetrics.GetTotalTimeMean()),
                new JProperty(
                    "latencyTotal",
                    new JObject(
                        new JProperty("0", commandMetrics.GetTotalTimePercentile(0)),
                        new JProperty("25", commandMetrics.GetTotalTimePercentile(25)),
                        new JProperty("50", commandMetrics.GetTotalTimePercentile(50)),
                        new JProperty("75", commandMetrics.GetTotalTimePercentile(75)),
                        new JProperty("90", commandMetrics.GetTotalTimePercentile(90)),
                        new JProperty("95", commandMetrics.GetTotalTimePercentile(95)),
                        new JProperty("99", commandMetrics.GetTotalTimePercentile(99)),
                        new JProperty("99.5", commandMetrics.GetTotalTimePercentile(99.5)),
                        new JProperty("100", commandMetrics.GetTotalTimePercentile(100)))),
                new JProperty("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold.Get()), // property values for reporting what is actually seen by the command rather than what was set somewhere
                new JProperty("propertyValue_circuitBreakerSleepWindowInMilliseconds", (long)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds),
                new JProperty("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage.Get()),
                new JProperty("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen.Get()),
                new JProperty("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed.Get()),
                new JProperty("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled.Get()),
                new JProperty("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy.Get()),
                new JProperty("propertyValue_executionIsolationThreadTimeoutInMilliseconds", (long)commandProperties.ExecutionIsolationThreadTimeout.Get().TotalMilliseconds),
                new JProperty("propertyValue_executionIsolationThreadInterruptOnTimeout", commandProperties.ExecutionIsolationThreadInterruptOnTimeout.Get()),
                new JProperty("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.ExecutionIsolationThreadPoolKeyOverride.Get()),
                new JProperty("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Get()),
                new JProperty("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests.Get()),
                new JProperty("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get()),
                new JProperty("propertyValue_requestCacheEnabled", commandProperties.RequestCacheEnabled.Get()),
                new JProperty("propertyValue_requestLogEnabled", commandProperties.RequestLogEnabled.Get()),
                new JProperty("reportingHosts", 1));

            return(data.ToString(Formatting.None));
        }
        public static List <HystrixCommandInfo> GetHystrixCommandInfo(string servicePath)
        {
            var             result   = new List <HystrixCommandInfo>();
            ServiceMetadata metadata = EndpointHost.Config.MetadataMap[servicePath];

            foreach (Operation operation in metadata.Operations)
            {
                HystrixCommandMetrics     commandMetrics    = operation.HystrixCommand.Metrics;
                IHystrixCircuitBreaker    circuitBreaker    = operation.HystrixCommand.CircuitBreaker;
                HealthCounts              healthCounts      = commandMetrics.GetHealthCounts();
                IHystrixCommandProperties commandProperties = commandMetrics.Properties;

                var commandInfo = new HystrixCommandInfo
                {
                    Type                                     = "HystrixCommand",
                    Name                                     = commandMetrics.OperationName,
                    Group                                    = commandMetrics.FullServiceName,
                    CurrentTime                              = DateTime.Now,
                    IsCircuitBreakerOpen                     = (circuitBreaker == null ? false : circuitBreaker.IsOpen()),
                    ErrorPercentage                          = healthCounts.ErrorPercentage,
                    ErrorCount                               = healthCounts.TotalErrorCount,
                    RequestCount                             = healthCounts.TotalRequests,
                    RollingCountSuccess                      = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Success),
                    RollingCountShortCircuited               = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited),
                    RollingCountTimeout                      = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Timeout),
                    RollingCountThreadPoolRejected           = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected),
                    RollingCountFrameworkExceptionThrown     = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FrameworkExceptionThrown),
                    RollingCountServiceExceptionThrown       = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ServiceExceptionThrown),
                    RollingCountValidationExceptionThrown    = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ValidationExceptionThrown),
                    CumulativeCountSuccess                   = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.Success),
                    CumulativeCountShortCircuited            = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ShortCircuited),
                    CumulativeCountTimeout                   = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.Timeout),
                    CumulativeCountThreadPoolRejected        = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ThreadPoolRejected),
                    CumulativeCountFrameworkExcetpionThrown  = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.FrameworkExceptionThrown),
                    CumulativeCountServiceExceptionThrown    = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ServiceExceptionThrown),
                    CumulativeCountValidationExceptionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ValidationExceptionThrown),
                    CurrentConcurrentExecutionCount          = commandMetrics.CurrentConcurrentExecutionCount,
                    LatencyExecuteMean                       = commandMetrics.GetServiceExecutionTimeMean(),
                    LatencyExecute                           = new PercentileInfo
                    {
                        P0      = commandMetrics.GetServiceExecutionTimePercentile(0),
                        P25     = commandMetrics.GetServiceExecutionTimePercentile(25),
                        P50     = commandMetrics.GetServiceExecutionTimePercentile(50),
                        P75     = commandMetrics.GetServiceExecutionTimePercentile(75),
                        P90     = commandMetrics.GetServiceExecutionTimePercentile(90),
                        P95     = commandMetrics.GetServiceExecutionTimePercentile(95),
                        P99     = commandMetrics.GetServiceExecutionTimePercentile(99),
                        P99DOT5 = commandMetrics.GetServiceExecutionTimePercentile(99.5),
                        P100    = commandMetrics.GetServiceExecutionTimePercentile(100),
                    },
                    LatencyTotalMean = commandMetrics.GetTotalTimeMean(),
                    LatencyTotal     = new PercentileInfo
                    {
                        P0      = commandMetrics.GetTotalTimePercentile(0),
                        P25     = commandMetrics.GetTotalTimePercentile(25),
                        P50     = commandMetrics.GetTotalTimePercentile(50),
                        P75     = commandMetrics.GetTotalTimePercentile(75),
                        P90     = commandMetrics.GetTotalTimePercentile(90),
                        P95     = commandMetrics.GetTotalTimePercentile(95),
                        P99     = commandMetrics.GetTotalTimePercentile(99),
                        P99DOT5 = commandMetrics.GetTotalTimePercentile(99.5),
                        P100    = commandMetrics.GetTotalTimePercentile(100),
                    },
                    PropertyValue_CircuitBreakerRequestVolumeThreshold    = commandProperties.CircuitBreakerRequestVolumeThreshold.Get(),
                    PropertyValue_CircuitBreakerSleepWindowInMilliseconds = (long)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds,
                    PropertyValue_CircuitBreakerErrorThresholdPercentage  = commandProperties.CircuitBreakerErrorThresholdPercentage.Get(),
                    PropertyValue_CircuitBreakerForceOpen   = commandProperties.CircuitBreakerForceOpen.Get(),
                    PropertyValue_CircuitBreakerForceClosed = commandProperties.CircuitBreakerForceClosed.Get(),
                    PropertyValue_CircuitBreakerEnabled     = commandProperties.CircuitBreakerEnabled.Get(),
                    PropertyValue_ExecutionIsolationThreadTimeoutInMilliseconds = (long)operation.HystrixCommand.GetExecutionTimeout().TotalMilliseconds,
                    PropertyValue_MetricsRollingStatisticalWindowInMilliseconds = commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get(),
                    PropertyValue_RequestLogEnabled = commandProperties.RequestLogEnabled.Get(),
                    ReportingHosts = 1,
                };

                result.Add(commandInfo);
            }

            return(result);
        }
 protected IMonitor <long> getRollingMonitor(string name, HystrixEventType @event)
 {
     return(ServoMetricFactory.GaugeMetric <long>(this,
                                                  MonitorConfig.builder(name).withTag(DataSourceLevel.DEBUG).withTag(ServoTypeTag).withTag(ServoInstanceTag).build(),
                                                  () => metrics.GetRollingCount(@event.ToHystrixRollingNumberEvent())));
 }