public bool IsOpen()
{
if (Interlocked.Read(ref _circuitOpen) == 1)
{
// if we're open we immediately return true and don't bother attempting to 'close' ourself as that is left to allowSingleTest and a subsequent successful test to close
return(true);
}
// we're closed, so let's see if errors have made us so we should trip the circuit open
HealthCounts health = _metrics.GetHealthCounts();
// check if we are past the statisticalWindowVolumeThreshold
if (health.TotalRequests < _properties.CircuitBreakerRequestVolumeThreshold.Value)
{
// we are not past the minimum volume threshold for the statisticalWindow so we'll return false immediately and not calculate anything
return(false);
}
if (health.ErrorPercentage < _properties.CircuitBreakerErrorThresholdPercentage.Value)
{
return(false);
}
else
{
// our failure rate is too high, trip the circuit
if (Interlocked.CompareExchange(ref _circuitOpen, 1L, 0L) == 0L)
{
// if the previousValue was false then we want to set the currentTime
Interlocked.Exchange(ref _circuitOpenedOrLastTestedTime, _clock.EllapsedTimeInMs);
return(true);
}
else
{
// How could previousValue be true? If another thread was going through this code at the same time a race-condition could have
// caused another thread to set it to true already even though we were in the process of doing the same
// In this case, we know the circuit is open, so let the other thread set the currentTime and report back that the circuit is open
return(true);
}
}
}
/// <inheritdoc />
public bool IsOpen()
{
if (this.circuitOpen.Value)
{
// if we're open we immediately return true and don't bother attempting to 'close' ourself as that is left to allowSingleTest and a subsequent successful test to close
return(true);
}
// we're closed, so let's see if errors have made us so we should trip the circuit open
HealthCounts health = this.metrics.GetHealthCounts();
// check if we are past the statisticalWindowVolumeThreshold
if (health.TotalRequests < this.properties.CircuitBreakerRequestVolumeThreshold.Get())
{
// we are not past the minimum volume threshold for the statisticalWindow so we'll return false immediately and not calculate anything
return(false);
}
if (health.ErrorPercentage < this.properties.CircuitBreakerErrorThresholdPercentage.Get())
{
return(false);
}
else
{
// our failure rate is too high, trip the circuit
if (this.circuitOpen.CompareAndSet(false, true))
{
// if the previousValue was false then we want to set the currentTime
// How could previousValue be true? If another thread was going through this code at the same time a race-condition could have
// caused another thread to set it to true already even though we were in the process of doing the same
this.circuitOpenedOrLastTestedTime.Value = ActualTime.CurrentTimeInMillis;
}
return(true);
}
}
private static void WriteCommandMetrics(JsonTextWriter writer, HystrixCommandMetrics commandMetrics)
{
IHystrixCommandKey key = commandMetrics.CommandKey;
IHystrixCircuitBreaker circuitBreaker = HystrixCircuitBreakerFactory.GetInstance(key);
writer.WriteStartObject();
writer.WriteStringField("type", "HystrixCommand");
writer.WriteStringField("name", key.Name);
writer.WriteStringField("group", commandMetrics.CommandGroup.Name);
writer.WriteLongField("currentTime", Time.CurrentTimeMillisJava);
// circuit breaker
if (circuitBreaker == null)
{
// circuit breaker is disabled and thus never open
writer.WriteBooleanField("isCircuitBreakerOpen", false);
}
else
{
writer.WriteBooleanField("isCircuitBreakerOpen", circuitBreaker.IsOpen);
}
HealthCounts healthCounts = commandMetrics.Healthcounts;
writer.WriteIntegerField("errorPercentage", healthCounts.ErrorPercentage);
writer.WriteLongField("errorCount", healthCounts.ErrorCount);
writer.WriteLongField("requestCount", healthCounts.TotalRequests);
// rolling counters
writer.WriteLongField("rollingCountBadRequests", commandMetrics.GetRollingCount(HystrixEventType.BAD_REQUEST));
writer.WriteLongField("rollingCountCollapsedRequests", commandMetrics.GetRollingCount(HystrixEventType.COLLAPSED));
writer.WriteLongField("rollingCountEmit", commandMetrics.GetRollingCount(HystrixEventType.EMIT));
writer.WriteLongField("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(HystrixEventType.EXCEPTION_THROWN));
writer.WriteLongField("rollingCountFailure", commandMetrics.GetRollingCount(HystrixEventType.FAILURE));
writer.WriteLongField("rollingCountFallbackEmit", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_EMIT));
writer.WriteLongField("rollingCountFallbackFailure", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_FAILURE));
writer.WriteLongField("rollingCountFallbackMissing", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_MISSING));
writer.WriteLongField("rollingCountFallbackRejection", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_REJECTION));
writer.WriteLongField("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_SUCCESS));
writer.WriteLongField("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(HystrixEventType.RESPONSE_FROM_CACHE));
writer.WriteLongField("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(HystrixEventType.SEMAPHORE_REJECTED));
writer.WriteLongField("rollingCountShortCircuited", commandMetrics.GetRollingCount(HystrixEventType.SHORT_CIRCUITED));
writer.WriteLongField("rollingCountSuccess", commandMetrics.GetRollingCount(HystrixEventType.SUCCESS));
writer.WriteLongField("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(HystrixEventType.THREAD_POOL_REJECTED));
writer.WriteLongField("rollingCountTimeout", commandMetrics.GetRollingCount(HystrixEventType.TIMEOUT));
writer.WriteIntegerField("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount);
writer.WriteLongField("rollingMaxConcurrentExecutionCount", commandMetrics.RollingMaxConcurrentExecutions);
// latency percentiles
writer.WriteIntegerField("latencyExecute_mean", commandMetrics.ExecutionTimeMean);
writer.WriteObjectFieldStart("latencyExecute");
writer.WriteIntegerField("0", commandMetrics.GetExecutionTimePercentile(0));
writer.WriteIntegerField("25", commandMetrics.GetExecutionTimePercentile(25));
writer.WriteIntegerField("50", commandMetrics.GetExecutionTimePercentile(50));
writer.WriteIntegerField("75", commandMetrics.GetExecutionTimePercentile(75));
writer.WriteIntegerField("90", commandMetrics.GetExecutionTimePercentile(90));
writer.WriteIntegerField("95", commandMetrics.GetExecutionTimePercentile(95));
writer.WriteIntegerField("99", commandMetrics.GetExecutionTimePercentile(99));
writer.WriteIntegerField("99.5", commandMetrics.GetExecutionTimePercentile(99.5));
writer.WriteIntegerField("100", commandMetrics.GetExecutionTimePercentile(100));
writer.WriteEndObject();
writer.WriteIntegerField("latencyTotal_mean", commandMetrics.TotalTimeMean);
writer.WriteObjectFieldStart("latencyTotal");
writer.WriteIntegerField("0", commandMetrics.GetTotalTimePercentile(0));
writer.WriteIntegerField("25", commandMetrics.GetTotalTimePercentile(25));
writer.WriteIntegerField("50", commandMetrics.GetTotalTimePercentile(50));
writer.WriteIntegerField("75", commandMetrics.GetTotalTimePercentile(75));
writer.WriteIntegerField("90", commandMetrics.GetTotalTimePercentile(90));
writer.WriteIntegerField("95", commandMetrics.GetTotalTimePercentile(95));
writer.WriteIntegerField("99", commandMetrics.GetTotalTimePercentile(99));
writer.WriteIntegerField("99.5", commandMetrics.GetTotalTimePercentile(99.5));
writer.WriteIntegerField("100", commandMetrics.GetTotalTimePercentile(100));
writer.WriteEndObject();
// property values for reporting what is actually seen by the command rather than what was set somewhere
IHystrixCommandOptions commandProperties = commandMetrics.Properties;
writer.WriteIntegerField("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold);
writer.WriteIntegerField("propertyValue_circuitBreakerSleepWindowInMilliseconds", commandProperties.CircuitBreakerSleepWindowInMilliseconds);
writer.WriteIntegerField("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage);
writer.WriteBooleanField("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen);
writer.WriteBooleanField("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed);
writer.WriteBooleanField("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled);
writer.WriteStringField("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy.ToString());
writer.WriteIntegerField("propertyValue_executionIsolationThreadTimeoutInMilliseconds", commandProperties.ExecutionTimeoutInMilliseconds);
writer.WriteIntegerField("propertyValue_executionTimeoutInMilliseconds", commandProperties.ExecutionTimeoutInMilliseconds);
writer.WriteBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", false);
writer.WriteStringField("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.ExecutionIsolationThreadPoolKeyOverride);
writer.WriteIntegerField("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests);
writer.WriteIntegerField("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests);
writer.WriteIntegerField("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds);
writer.WriteBooleanField("propertyValue_requestCacheEnabled", commandProperties.RequestCacheEnabled);
writer.WriteBooleanField("propertyValue_requestLogEnabled", commandProperties.RequestLogEnabled);
writer.WriteIntegerField("reportingHosts", 1); // this will get summed across all instances in a cluster
writer.WriteStringField("threadPool", commandMetrics.ThreadPoolKey.Name);
writer.WriteEndObject();
}
private static List <GlobalStreamHystrixCommandInfo> GetHystrixCommandInfoList()
{
var result = new List <GlobalStreamHystrixCommandInfo>();
foreach (ServiceMetadata metadata in EndpointHost.Config.MetadataMap.Values)
{
string refinedServiceName = ServiceUtils.RefineServiceName(metadata.ServiceNamespace, metadata.ServiceName);
foreach (Operation operation in metadata.Operations)
{
HystrixCommandMetrics commandMetrics = operation.HystrixCommand.Metrics;
IHystrixCircuitBreaker circuitBreaker = operation.HystrixCommand.CircuitBreaker;
HealthCounts healthCounts = commandMetrics.GetHealthCounts();
IHystrixCommandProperties commandProperties = commandMetrics.Properties;
GlobalStreamHystrixCommandInfo hystrixCommandInfo = new GlobalStreamHystrixCommandInfo()
{
type = TurbineDataTypeHystrixCommand,
name = refinedServiceName + "." + operation.Name.ToLower(),
group = refinedServiceName,
currentTime = DateTime.Now.ToUnixTimeMs(),
isCircuitBreakerOpen = circuitBreaker.IsOpen(),
errorPercentage = healthCounts.ErrorPercentage,
errorCount = healthCounts.TotalErrorCount,
requestCount = healthCounts.TotalRequests,
rollingCountExceptionsThrown = healthCounts.TotalExceptionCount,
rollingCountFailure = healthCounts.TotalFailureCount,
rollingCountSemaphoreRejected = 0,
rollingCountShortCircuited = healthCounts.ShortCircuitedCount,
rollingCountSuccess = healthCounts.SuccessCount,
rollingCountThreadPoolRejected = 0,
rollingCountTimeout = healthCounts.TimeoutCount,
rollingCountFallbackFailure = 0,
rollingCountFallbackSuccess = 0,
rollingCountFallbackRejection = 0,
latencyExecute = new GlobalStreamPercentileInfo()
{
P0 = commandMetrics.GetServiceExecutionTimePercentile(0),
P25 = commandMetrics.GetServiceExecutionTimePercentile(25),
P50 = commandMetrics.GetServiceExecutionTimePercentile(50),
P75 = commandMetrics.GetServiceExecutionTimePercentile(75),
P90 = commandMetrics.GetServiceExecutionTimePercentile(90),
P95 = commandMetrics.GetServiceExecutionTimePercentile(95),
P99 = commandMetrics.GetServiceExecutionTimePercentile(99),
P99DOT5 = commandMetrics.GetServiceExecutionTimePercentile(99.5),
P100 = commandMetrics.GetServiceExecutionTimePercentile(100)
},
latencyExecute_mean = commandMetrics.GetServiceExecutionTimeMean(),
latencyTotal = new GlobalStreamPercentileInfo()
{
P0 = commandMetrics.GetTotalTimePercentile(0),
P25 = commandMetrics.GetTotalTimePercentile(25),
P50 = commandMetrics.GetTotalTimePercentile(50),
P75 = commandMetrics.GetTotalTimePercentile(75),
P90 = commandMetrics.GetTotalTimePercentile(90),
P95 = commandMetrics.GetTotalTimePercentile(95),
P99 = commandMetrics.GetTotalTimePercentile(99),
P99DOT5 = commandMetrics.GetTotalTimePercentile(99.5),
P100 = commandMetrics.GetTotalTimePercentile(100)
},
latencyTotal_mean = commandMetrics.GetTotalTimeMean(),
reportingHosts = 1,
propertyValue_circuitBreakerEnabled = commandProperties.CircuitBreakerEnabled.Get(),
propertyValue_circuitBreakerErrorThresholdPercentage = commandProperties.CircuitBreakerErrorThresholdPercentage.Get(),
propertyValue_circuitBreakerForceClosed = commandProperties.CircuitBreakerForceClosed.Get(),
propertyValue_circuitBreakerForceOpen = commandProperties.CircuitBreakerForceOpen.Get(),
propertyValue_circuitBreakerRequestVolumeThreshold = commandProperties.CircuitBreakerRequestVolumeThreshold.Get(),
propertyValue_circuitBreakerSleepWindowInMilliseconds = (int)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds,
propertyValue_executionIsolationSemaphoreMaxConcurrentRequests = 1000,
propertyValue_executionIsolationStrategy = TurbineStrategySemaphore,
propertyValue_executionIsolationThreadTimeoutInMilliseconds = (int)operation.HystrixCommand.GetExecutionTimeout().TotalMilliseconds,
propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests = 0,
propertyValue_metricsRollingStatisticalWindowInMilliseconds = commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get(),
currentConcurrentExecutionCount = commandMetrics.CurrentConcurrentExecutionCount
};
result.Add(hystrixCommandInfo);
}
}
return(result);
}
private String GetCommandJson(CommandMetrics commandMetrics)
{
var circuitBreaker = this.circuitBreaker ?? CircuitBreakerFactory.GetInstance(commandMetrics.CommandName);
var sb = new StringBuilder(1024);
StringWriter sw = new StringWriter(sb, CultureInfo.InvariantCulture);
var json = new JsonWriter(sw);
json.writeStartObject();
json.writeStringField("type", "ServiceCommand");
json.writeStringField("name", commandMetrics.CommandName);
json.writeStringField("group", commandMetrics.CommandGroup);
json.writeNumberField("currentTime", DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond); // TODO check this
// circuit breaker
if (circuitBreaker == null)
{
// circuit breaker is disabled and thus never open
json.writeBooleanField("isCircuitBreakerOpen", false);
}
else
{
json.writeBooleanField("isCircuitBreakerOpen", circuitBreaker.IsOpen());
}
HealthCounts healthCounts = commandMetrics.GetHealthCounts();
json.writeNumberField("errorPercentage", healthCounts.ErrorPercentage);
json.writeNumberField("errorCount", healthCounts.ErrorCount);
json.writeNumberField("requestCount", healthCounts.TotalRequests);
// rolling counters
json.writeNumberField("rollingCountBadRequests", commandMetrics.GetRollingCount(RollingNumberEvent.BAD_REQUEST));
json.writeNumberField("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(RollingNumberEvent.EXCEPTION_THROWN));
json.writeNumberField("rollingCountFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FAILURE));
json.writeNumberField("rollingCountFallbackFailure", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_FAILURE));
json.writeNumberField("rollingCountFallbackRejection", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_REJECTION));
json.writeNumberField("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.FALLBACK_SUCCESS));
json.writeNumberField("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(RollingNumberEvent.RESPONSE_FROM_CACHE));
json.writeNumberField("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(RollingNumberEvent.SEMAPHORE_REJECTED));
json.writeNumberField("rollingCountShortCircuited", commandMetrics.GetRollingCount(RollingNumberEvent.SHORT_CIRCUITED));
json.writeNumberField("rollingCountSuccess", commandMetrics.GetRollingCount(RollingNumberEvent.SUCCESS));
json.writeNumberField("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(RollingNumberEvent.THREAD_POOL_REJECTED));
json.writeNumberField("rollingCountTimeout", commandMetrics.GetRollingCount(RollingNumberEvent.TIMEOUT));
json.writeNumberField("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount);
json.writeNumberField("rollingMaxConcurrentExecutionCount", commandMetrics.GetRollingMaxConcurrentExecutions());
// latency percentiles
json.writeNumberField("latencyExecute_mean", commandMetrics.GetExecutionTimeMean());
json.writeObjectFieldStart("latencyExecute");
json.writeNumberField("0", commandMetrics.GetExecutionTimePercentile(0));
json.writeNumberField("25", commandMetrics.GetExecutionTimePercentile(25));
json.writeNumberField("50", commandMetrics.GetExecutionTimePercentile(50));
json.writeNumberField("75", commandMetrics.GetExecutionTimePercentile(75));
json.writeNumberField("90", commandMetrics.GetExecutionTimePercentile(90));
json.writeNumberField("95", commandMetrics.GetExecutionTimePercentile(95));
json.writeNumberField("99", commandMetrics.GetExecutionTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.GetExecutionTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.GetExecutionTimePercentile(100));
json.writeEndObject();
//
json.writeNumberField("latencyTotal_mean", commandMetrics.GetTotalTimeMean());
json.writeObjectFieldStart("latencyTotal");
json.writeNumberField("0", commandMetrics.GetTotalTimePercentile(0));
json.writeNumberField("25", commandMetrics.GetTotalTimePercentile(25));
json.writeNumberField("50", commandMetrics.GetTotalTimePercentile(50));
json.writeNumberField("75", commandMetrics.GetTotalTimePercentile(75));
json.writeNumberField("90", commandMetrics.GetTotalTimePercentile(90));
json.writeNumberField("95", commandMetrics.GetTotalTimePercentile(95));
json.writeNumberField("99", commandMetrics.GetTotalTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.GetTotalTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.GetTotalTimePercentile(100));
json.writeEndObject();
// property values for reporting what is actually seen by the command rather than what was set somewhere
var commandProperties = commandMetrics.Properties;
json.writeNumberField("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold.Value);
json.writeNumberField("propertyValue_circuitBreakerSleepWindowInMilliseconds", commandProperties.CircuitBreakerSleepWindowInMilliseconds.Value);
json.writeNumberField("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage.Value);
json.writeBooleanField("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen.Value);
json.writeBooleanField("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed.Value);
json.writeBooleanField("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled.Value);
json.writeStringField("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy.Value.ToString());
json.writeNumberField("propertyValue_executionIsolationThreadTimeoutInMilliseconds", commandProperties.ExecutionIsolationThreadTimeoutInMilliseconds.Value);
json.writeNumberField("propertyValue_executionTimeoutInMilliseconds", commandProperties.ExecutionIsolationThreadTimeoutInMilliseconds.Value);
//json.writeBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", commandProperties.executionIsolationThreadInterruptOnTimeout().get());
//json.writeStringField("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.executionIsolationThreadPoolKeyOverride().get());
json.writeNumberField("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Value);
json.writeNumberField("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests.Value);
json.writeNumberField("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Value);
json.writeBooleanField("propertyValue_requestCacheEnabled", commandProperties.RequestCacheEnabled.Value);
json.writeBooleanField("propertyValue_requestLogEnabled", commandProperties.RequestLogEnabled.Value);
json.writeNumberField("reportingHosts", 1); // this will get summed across all instances in a cluster
//json.writeStringField("threadPool", commandMetrics.getThreadPoolKey().name());
// Hystrix specific
json.writeNumberField("rollingCountCollapsedRequests", 0);
json.writeBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", false);
json.writeEndObject();
return(sb.ToString());
}
public static List <HystrixCommandInfo> GetHystrixCommandInfo(string servicePath)
{
var result = new List <HystrixCommandInfo>();
ServiceMetadata metadata = EndpointHost.Config.MetadataMap[servicePath];
foreach (Operation operation in metadata.Operations)
{
HystrixCommandMetrics commandMetrics = operation.HystrixCommand.Metrics;
IHystrixCircuitBreaker circuitBreaker = operation.HystrixCommand.CircuitBreaker;
HealthCounts healthCounts = commandMetrics.GetHealthCounts();
IHystrixCommandProperties commandProperties = commandMetrics.Properties;
var commandInfo = new HystrixCommandInfo
{
Type = "HystrixCommand",
Name = commandMetrics.OperationName,
Group = commandMetrics.FullServiceName,
CurrentTime = DateTime.Now,
IsCircuitBreakerOpen = (circuitBreaker == null ? false : circuitBreaker.IsOpen()),
ErrorPercentage = healthCounts.ErrorPercentage,
ErrorCount = healthCounts.TotalErrorCount,
RequestCount = healthCounts.TotalRequests,
RollingCountSuccess = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Success),
RollingCountShortCircuited = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited),
RollingCountTimeout = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Timeout),
RollingCountThreadPoolRejected = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected),
RollingCountFrameworkExceptionThrown = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FrameworkExceptionThrown),
RollingCountServiceExceptionThrown = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ServiceExceptionThrown),
RollingCountValidationExceptionThrown = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ValidationExceptionThrown),
CumulativeCountSuccess = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.Success),
CumulativeCountShortCircuited = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ShortCircuited),
CumulativeCountTimeout = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.Timeout),
CumulativeCountThreadPoolRejected = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ThreadPoolRejected),
CumulativeCountFrameworkExcetpionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.FrameworkExceptionThrown),
CumulativeCountServiceExceptionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ServiceExceptionThrown),
CumulativeCountValidationExceptionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ValidationExceptionThrown),
CurrentConcurrentExecutionCount = commandMetrics.CurrentConcurrentExecutionCount,
LatencyExecuteMean = commandMetrics.GetServiceExecutionTimeMean(),
LatencyExecute = new PercentileInfo
{
P0 = commandMetrics.GetServiceExecutionTimePercentile(0),
P25 = commandMetrics.GetServiceExecutionTimePercentile(25),
P50 = commandMetrics.GetServiceExecutionTimePercentile(50),
P75 = commandMetrics.GetServiceExecutionTimePercentile(75),
P90 = commandMetrics.GetServiceExecutionTimePercentile(90),
P95 = commandMetrics.GetServiceExecutionTimePercentile(95),
P99 = commandMetrics.GetServiceExecutionTimePercentile(99),
P99DOT5 = commandMetrics.GetServiceExecutionTimePercentile(99.5),
P100 = commandMetrics.GetServiceExecutionTimePercentile(100),
},
LatencyTotalMean = commandMetrics.GetTotalTimeMean(),
LatencyTotal = new PercentileInfo
{
P0 = commandMetrics.GetTotalTimePercentile(0),
P25 = commandMetrics.GetTotalTimePercentile(25),
P50 = commandMetrics.GetTotalTimePercentile(50),
P75 = commandMetrics.GetTotalTimePercentile(75),
P90 = commandMetrics.GetTotalTimePercentile(90),
P95 = commandMetrics.GetTotalTimePercentile(95),
P99 = commandMetrics.GetTotalTimePercentile(99),
P99DOT5 = commandMetrics.GetTotalTimePercentile(99.5),
P100 = commandMetrics.GetTotalTimePercentile(100),
},
PropertyValue_CircuitBreakerRequestVolumeThreshold = commandProperties.CircuitBreakerRequestVolumeThreshold.Get(),
PropertyValue_CircuitBreakerSleepWindowInMilliseconds = (long)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds,
PropertyValue_CircuitBreakerErrorThresholdPercentage = commandProperties.CircuitBreakerErrorThresholdPercentage.Get(),
PropertyValue_CircuitBreakerForceOpen = commandProperties.CircuitBreakerForceOpen.Get(),
PropertyValue_CircuitBreakerForceClosed = commandProperties.CircuitBreakerForceClosed.Get(),
PropertyValue_CircuitBreakerEnabled = commandProperties.CircuitBreakerEnabled.Get(),
PropertyValue_ExecutionIsolationThreadTimeoutInMilliseconds = (long)operation.HystrixCommand.GetExecutionTimeout().TotalMilliseconds,
PropertyValue_MetricsRollingStatisticalWindowInMilliseconds = commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get(),
PropertyValue_RequestLogEnabled = commandProperties.RequestLogEnabled.Get(),
ReportingHosts = 1,
};
result.Add(commandInfo);
}
return(result);
}
/// <summary>
/// Produces JSON formatted metrics data from an instance of <see cref="HystrixCommandMetrics"/>.
/// </summary>
/// <param name="commandMetrics">The metrics of a command.</param>
/// <returns>JSON formatted metrics data.</returns>
private static string CreateCommandSampleData(HystrixCommandMetrics commandMetrics)
{
IHystrixCircuitBreaker circuitBreaker = HystrixCircuitBreakerFactory.GetInstance(commandMetrics.CommandKey);
HealthCounts healthCounts = commandMetrics.GetHealthCounts();
IHystrixCommandProperties commandProperties = commandMetrics.Properties;
JObject data = new JObject(
new JProperty("type", "HystrixCommand"),
new JProperty("name", commandMetrics.CommandKey.Name),
new JProperty("group", commandMetrics.CommandGroup.Name),
new JProperty("currentTime", GetCurrentTimeForJavascript()),
circuitBreaker == null ? new JProperty("isCircuitBreakerOpen", false) : new JProperty("isCircuitBreakerOpen", circuitBreaker.IsOpen()),
new JProperty("errorPercentage", healthCounts.ErrorPercentage), // health counts
new JProperty("errorCount", healthCounts.ErrorCount),
new JProperty("requestCount", healthCounts.TotalRequests),
new JProperty("rollingCountCollapsedRequests", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Collapsed)), // rolling counters
new JProperty("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown)),
new JProperty("rollingCountFailure", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Failure)),
new JProperty("rollingCountFallbackFailure", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure)),
new JProperty("rollingCountFallbackRejection", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection)),
new JProperty("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess)),
new JProperty("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache)),
new JProperty("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected)),
new JProperty("rollingCountShortCircuited", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited)),
new JProperty("rollingCountSuccess", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Success)),
new JProperty("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected)),
new JProperty("rollingCountTimeout", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Timeout)),
new JProperty("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount),
new JProperty("latencyExecute_mean", commandMetrics.GetExecutionTimeMean()), // latency percentiles
new JProperty(
"latencyExecute",
new JObject(
new JProperty("0", commandMetrics.GetExecutionTimePercentile(0)),
new JProperty("25", commandMetrics.GetExecutionTimePercentile(25)),
new JProperty("50", commandMetrics.GetExecutionTimePercentile(50)),
new JProperty("75", commandMetrics.GetExecutionTimePercentile(75)),
new JProperty("90", commandMetrics.GetExecutionTimePercentile(90)),
new JProperty("95", commandMetrics.GetExecutionTimePercentile(95)),
new JProperty("99", commandMetrics.GetExecutionTimePercentile(99)),
new JProperty("99.5", commandMetrics.GetExecutionTimePercentile(99.5)),
new JProperty("100", commandMetrics.GetExecutionTimePercentile(100)))),
new JProperty("latencyTotal_mean", commandMetrics.GetTotalTimeMean()),
new JProperty(
"latencyTotal",
new JObject(
new JProperty("0", commandMetrics.GetTotalTimePercentile(0)),
new JProperty("25", commandMetrics.GetTotalTimePercentile(25)),
new JProperty("50", commandMetrics.GetTotalTimePercentile(50)),
new JProperty("75", commandMetrics.GetTotalTimePercentile(75)),
new JProperty("90", commandMetrics.GetTotalTimePercentile(90)),
new JProperty("95", commandMetrics.GetTotalTimePercentile(95)),
new JProperty("99", commandMetrics.GetTotalTimePercentile(99)),
new JProperty("99.5", commandMetrics.GetTotalTimePercentile(99.5)),
new JProperty("100", commandMetrics.GetTotalTimePercentile(100)))),
new JProperty("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold.Get()), // property values for reporting what is actually seen by the command rather than what was set somewhere
new JProperty("propertyValue_circuitBreakerSleepWindowInMilliseconds", (long)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds),
new JProperty("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage.Get()),
new JProperty("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen.Get()),
new JProperty("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed.Get()),
new JProperty("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled.Get()),
new JProperty("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy.Get()),
new JProperty("propertyValue_executionIsolationThreadTimeoutInMilliseconds", (long)commandProperties.ExecutionIsolationThreadTimeout.Get().TotalMilliseconds),
new JProperty("propertyValue_executionIsolationThreadInterruptOnTimeout", commandProperties.ExecutionIsolationThreadInterruptOnTimeout.Get()),
new JProperty("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.ExecutionIsolationThreadPoolKeyOverride.Get()),
new JProperty("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Get()),
new JProperty("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests.Get()),
new JProperty("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get()),
new JProperty("propertyValue_requestCacheEnabled", commandProperties.RequestCacheEnabled.Get()),
new JProperty("propertyValue_requestLogEnabled", commandProperties.RequestLogEnabled.Get()),
new JProperty("reportingHosts", 1));
return(data.ToString(Formatting.None));
}