/// <summary> /// Initializes a new instance of the <see cref="HystrixCircuitBreakerImpl"/> class. /// </summary> /// <param name="properties">The properties of the owner command.</param> /// <param name="metrics">The metrics of the owner command.</param> public HystrixCircuitBreakerImpl(IHystrixCommandProperties properties, HystrixCommandMetrics metrics) { if (properties == null) { throw new ArgumentNullException("properties"); } if (metrics == null) { throw new ArgumentNullException("metrics"); } this.properties = properties; this.metrics = metrics; }
public void TestGetErrorPercentage() { string key = "cmd-metrics-A"; HystrixCommand <bool> cmd1 = new SuccessCommand(key, 0); HystrixCommandMetrics metrics = cmd1._metrics; Assert.True(WaitForHealthCountToUpdate(key, 1000), "Health count stream took to long"); cmd1.Execute(); Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long"); output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(0, metrics.Healthcounts.ErrorPercentage); HystrixCommand <bool> cmd2 = new FailureCommand(key, 0); cmd2.Execute(); Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long"); output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(50, metrics.Healthcounts.ErrorPercentage); HystrixCommand <bool> cmd3 = new SuccessCommand(key, 0); HystrixCommand <bool> cmd4 = new SuccessCommand(key, 0); cmd3.Execute(); cmd4.Execute(); Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long"); output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(25, metrics.Healthcounts.ErrorPercentage); HystrixCommand <bool> cmd5 = new TimeoutCommand(key); HystrixCommand <bool> cmd6 = new TimeoutCommand(key); cmd5.Execute(); cmd6.Execute(); Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long"); output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(50, metrics.Healthcounts.ErrorPercentage); HystrixCommand <bool> cmd7 = new SuccessCommand(key, 0); HystrixCommand <bool> cmd8 = new SuccessCommand(key, 0); HystrixCommand <bool> cmd9 = new SuccessCommand(key, 0); cmd7.Execute(); cmd8.Execute(); cmd9.Execute(); output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); // latent HystrixCommand <bool> cmd10 = new SuccessCommand(key, 60); cmd10.Execute(); output.WriteLine("ReqLog" + "@ " + Time.CurrentTimeMillis + " : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); // 6 success + 1 latent success + 1 failure + 2 timeout = 10 total // latent success not considered error // error percentage = 1 failure + 2 timeout / 10 Assert.True(WaitForHealthCountToUpdate(key, 250), "Health count stream took to long"); Assert.Equal(30, metrics.Healthcounts.ErrorPercentage); }
protected internal HystrixCircuitBreakerImpl(IHystrixCommandKey key, IHystrixCommandGroupKey commandGroup, IHystrixCommandOptions options, HystrixCommandMetrics metrics) { this.options = options; this.metrics = metrics; }
public HystrixMetricsPublisherCommandDefault(HystrixCommandKey commandKey, HystrixCommandGroupKey commandGroupKey, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandProperties properties) { // do nothing by default }
private static List <GlobalStreamHystrixCommandInfo> GetHystrixCommandInfoList() { var result = new List <GlobalStreamHystrixCommandInfo>(); foreach (ServiceMetadata metadata in EndpointHost.Config.MetadataMap.Values) { string refinedServiceName = ServiceUtils.RefineServiceName(metadata.ServiceNamespace, metadata.ServiceName); foreach (Operation operation in metadata.Operations) { HystrixCommandMetrics commandMetrics = operation.HystrixCommand.Metrics; IHystrixCircuitBreaker circuitBreaker = operation.HystrixCommand.CircuitBreaker; HealthCounts healthCounts = commandMetrics.GetHealthCounts(); IHystrixCommandProperties commandProperties = commandMetrics.Properties; GlobalStreamHystrixCommandInfo hystrixCommandInfo = new GlobalStreamHystrixCommandInfo() { type = TurbineDataTypeHystrixCommand, name = refinedServiceName + "." + operation.Name.ToLower(), group = refinedServiceName, currentTime = DateTime.Now.ToUnixTimeMs(), isCircuitBreakerOpen = circuitBreaker.IsOpen(), errorPercentage = healthCounts.ErrorPercentage, errorCount = healthCounts.TotalErrorCount, requestCount = healthCounts.TotalRequests, rollingCountExceptionsThrown = healthCounts.TotalExceptionCount, rollingCountFailure = healthCounts.TotalFailureCount, rollingCountSemaphoreRejected = 0, rollingCountShortCircuited = healthCounts.ShortCircuitedCount, rollingCountSuccess = healthCounts.SuccessCount, rollingCountThreadPoolRejected = 0, rollingCountTimeout = healthCounts.TimeoutCount, rollingCountFallbackFailure = 0, rollingCountFallbackSuccess = 0, rollingCountFallbackRejection = 0, latencyExecute = new GlobalStreamPercentileInfo() { P0 = commandMetrics.GetServiceExecutionTimePercentile(0), P25 = commandMetrics.GetServiceExecutionTimePercentile(25), P50 = commandMetrics.GetServiceExecutionTimePercentile(50), P75 = commandMetrics.GetServiceExecutionTimePercentile(75), P90 = commandMetrics.GetServiceExecutionTimePercentile(90), P95 = commandMetrics.GetServiceExecutionTimePercentile(95), P99 = commandMetrics.GetServiceExecutionTimePercentile(99), P99DOT5 = commandMetrics.GetServiceExecutionTimePercentile(99.5), P100 = commandMetrics.GetServiceExecutionTimePercentile(100) }, latencyExecute_mean = commandMetrics.GetServiceExecutionTimeMean(), latencyTotal = new GlobalStreamPercentileInfo() { P0 = commandMetrics.GetTotalTimePercentile(0), P25 = commandMetrics.GetTotalTimePercentile(25), P50 = commandMetrics.GetTotalTimePercentile(50), P75 = commandMetrics.GetTotalTimePercentile(75), P90 = commandMetrics.GetTotalTimePercentile(90), P95 = commandMetrics.GetTotalTimePercentile(95), P99 = commandMetrics.GetTotalTimePercentile(99), P99DOT5 = commandMetrics.GetTotalTimePercentile(99.5), P100 = commandMetrics.GetTotalTimePercentile(100) }, latencyTotal_mean = commandMetrics.GetTotalTimeMean(), reportingHosts = 1, propertyValue_circuitBreakerEnabled = commandProperties.CircuitBreakerEnabled.Get(), propertyValue_circuitBreakerErrorThresholdPercentage = commandProperties.CircuitBreakerErrorThresholdPercentage.Get(), propertyValue_circuitBreakerForceClosed = commandProperties.CircuitBreakerForceClosed.Get(), propertyValue_circuitBreakerForceOpen = commandProperties.CircuitBreakerForceOpen.Get(), propertyValue_circuitBreakerRequestVolumeThreshold = commandProperties.CircuitBreakerRequestVolumeThreshold.Get(), propertyValue_circuitBreakerSleepWindowInMilliseconds = (int)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds, propertyValue_executionIsolationSemaphoreMaxConcurrentRequests = 1000, propertyValue_executionIsolationStrategy = TurbineStrategySemaphore, propertyValue_executionIsolationThreadTimeoutInMilliseconds = (int)operation.HystrixCommand.GetExecutionTimeout().TotalMilliseconds, propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests = 0, propertyValue_metricsRollingStatisticalWindowInMilliseconds = commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get(), currentConcurrentExecutionCount = commandMetrics.CurrentConcurrentExecutionCount }; result.Add(hystrixCommandInfo); } } return(result); }
public TestCircuitBreaker(IHystrixCommandKey commandKey) { metrics = GetMetrics(commandKey, HystrixCommandOptionsTest.GetUnitTestOptions()); forceShortCircuit = false; }
public TestCircuitBreaker() { this.metrics = HystrixCircuitBreakerTest.GetMetrics(HystrixCommandOptionsTest.GetUnitTestOptions()); forceShortCircuit = false; }
public TestCommandBuilder SetMetrics(HystrixCommandMetrics metrics) { this.Metrics = metrics; return(this); }
public TestCircuitBreaker() { this.metrics = HystrixCircuitBreakerTest.GetMetrics(UnitTestSetterFactory.GetCommandPropertiesSetter()); forceShortCircuit = false; }
// send metrics to backend central logging system void sendMetrics(object sender, ElapsedEventArgs arg) { try { if (EndpointHost.MetadataMap == null) { return; } foreach (ServiceMetadata metadata in EndpointHost.MetadataMap.Values) { try { if (metadata.ServiceName == ServiceMetadata.AnonymousServiceName) { continue; } var tagMap = new Dictionary <string, string>(); tagMap["webservice"] = (ServiceUtils.ConvertNamespaceToMetricPrefix(metadata.ServiceNamespace) + "." + metadata.ServiceName) .ToLower().Replace("soa.ant.com.", string.Empty); tagMap["frameworkversion"] = string.Format("SS-{0} CG-{1}", metadata.AntServiceStackVersion, metadata.AntCodeGenVersion); var now = DateTime.Now; Action <HystrixCommandMetrics, long, string, string> logEventDistribution = (m, c, tn, tv) => { if (c <= 0) { return; } tagMap[tn] = tv; //metricLog.log(m.MetricNameEventDistribution, c, tagMap, now); }; Action <HystrixCommandMetrics, long, long?, string, string> logLatencyDistribution = (m, s, e, tn, tv) => { int count = m.GetServiceExecutionCountInTotalTimeRange(s, e); if (count <= 0) { return; } tagMap[tn] = tv; // metricLog.log(m.MetricNameLatencyDistribution, count, tagMap, now); }; Action <HystrixCommandMetrics, double, string, string> logLatencyPencentile = (m, p, tn, tv) => { long pencentile = m.GetTotalTimePercentile(p); if (pencentile <= 0) { return; } tagMap[tn] = tv; // metricLog.log(m.MetricNameLatencyPercentile, pencentile, tagMap, now); }; foreach (Operation operation in metadata.Operations) { HystrixCommandMetrics commandMetrics = operation.HystrixCommand.Metrics; tagMap["operation"] = tagMap["webservice"] + "." + commandMetrics.OperationName.ToLower(); long successCount = commandMetrics.GetSuccessCount(); long frameworkErrorCount = commandMetrics.GetFrameworkErrorCount(); long serviceErrorCount = commandMetrics.GetServiceErrorCount(); long validationErrorCount = commandMetrics.GetValidationErrorCount(); long shortCircuitedCount = commandMetrics.GetShortCircuitCount(); long timeoutCount = commandMetrics.GetTimeoutCount(); long threadPoolRejectedCount = commandMetrics.GetThreadPoolRejectedCount(); commandMetrics.ResetMetricsCounters(); long totalCount = successCount + frameworkErrorCount + serviceErrorCount + validationErrorCount + shortCircuitedCount + timeoutCount + threadPoolRejectedCount; //metricLog.log(commandMetrics.MetricNameRequestCount, totalCount, tagMap, now); // if(operation.IsAsync) //metricLog.log(commandMetrics.MetricNameConcurrentExecutionCount, commandMetrics.CurrentConcurrentExecutionCount, tagMap, now); var tagName = "distribution"; logEventDistribution(commandMetrics, successCount, tagName, "Success"); logEventDistribution(commandMetrics, shortCircuitedCount, tagName, "Short Circuited"); logEventDistribution(commandMetrics, timeoutCount, tagName, "Timeout"); logEventDistribution(commandMetrics, threadPoolRejectedCount, tagName, "Threadpool Rejected"); logEventDistribution(commandMetrics, frameworkErrorCount, tagName, "Framework Exception"); logEventDistribution(commandMetrics, serviceErrorCount, tagName, "Service Exception"); logEventDistribution(commandMetrics, validationErrorCount, tagName, "Validation Exception"); if (tagMap.ContainsKey(tagName)) { tagMap.Remove(tagName); } int count; long sum, min, max; commandMetrics.GetTotalTimeMetricsData(out count, out sum, out min, out max); tagName = "SetFeatureType"; tagMap[tagName] = "count"; // metricLog.log(commandMetrics.MetricNameLatency, count, tagMap, now); tagMap[tagName] = "sum"; //metricLog.log(commandMetrics.MetricNameLatency, sum, tagMap, now); tagMap[tagName] = "min"; // metricLog.log(commandMetrics.MetricNameLatency, min, tagMap, now); // tagMap[tagName] = "max"; //metricLog.log(commandMetrics.MetricNameLatency, max, tagMap, now); tagMap.Remove(tagName); tagName = "distribution"; logLatencyDistribution(commandMetrics, 0, 10, tagName, "0 ~ 10ms"); logLatencyDistribution(commandMetrics, 10, 50, tagName, "10 ~ 50ms"); logLatencyDistribution(commandMetrics, 50, 200, tagName, "50 ~ 200ms"); logLatencyDistribution(commandMetrics, 200, 500, tagName, "200 ~ 500ms"); logLatencyDistribution(commandMetrics, 500, 1000, tagName, "500ms ~ 1s"); logLatencyDistribution(commandMetrics, 1000, 5 * 1000, tagName, "1 ~ 5s"); logLatencyDistribution(commandMetrics, 5 * 1000, 10 * 1000, tagName, "5 ~ 10s"); logLatencyDistribution(commandMetrics, 10 * 1000, 30 * 1000, tagName, "10 ~ 30s"); logLatencyDistribution(commandMetrics, 30 * 1000, 100 * 1000, tagName, "30 ~ 100s"); logLatencyDistribution(commandMetrics, 100 * 1000, null, tagName, ">= 100s"); if (tagMap.ContainsKey(tagName)) { tagMap.Remove(tagName); } tagName = "percentile"; logLatencyPencentile(commandMetrics, 0, tagName, "0"); logLatencyPencentile(commandMetrics, 25, tagName, "25"); logLatencyPencentile(commandMetrics, 50, tagName, "50"); logLatencyPencentile(commandMetrics, 75, tagName, "75"); logLatencyPencentile(commandMetrics, 90, tagName, "90"); logLatencyPencentile(commandMetrics, 95, tagName, "95"); logLatencyPencentile(commandMetrics, 99, tagName, "99"); logLatencyPencentile(commandMetrics, 99.5, tagName, "99.5"); logLatencyPencentile(commandMetrics, 100, tagName, "100"); if (tagMap.ContainsKey(tagName)) { tagMap.Remove(tagName); } } } catch (Exception ex) { log.Error("Fail to send metrics to backend!", ex, new Dictionary <string, string>() { { "ErrorCode", "FXD300009" } }); } } } catch (Exception ex) { log.Error("Fail to send metrics to backend!", ex, new Dictionary <string, string>() { { "ErrorCode", "FXD300009" } }); } }
public static List <HystrixCommandInfo> GetHystrixCommandInfo(string servicePath) { var result = new List <HystrixCommandInfo>(); ServiceMetadata metadata = EndpointHost.Config.MetadataMap[servicePath]; foreach (Operation operation in metadata.Operations) { HystrixCommandMetrics commandMetrics = operation.HystrixCommand.Metrics; IHystrixCircuitBreaker circuitBreaker = operation.HystrixCommand.CircuitBreaker; HealthCounts healthCounts = commandMetrics.GetHealthCounts(); IHystrixCommandProperties commandProperties = commandMetrics.Properties; var commandInfo = new HystrixCommandInfo { Type = "HystrixCommand", Name = commandMetrics.OperationName, Group = commandMetrics.FullServiceName, CurrentTime = DateTime.Now, IsCircuitBreakerOpen = (circuitBreaker == null ? false : circuitBreaker.IsOpen()), ErrorPercentage = healthCounts.ErrorPercentage, ErrorCount = healthCounts.TotalErrorCount, RequestCount = healthCounts.TotalRequests, RollingCountSuccess = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Success), RollingCountShortCircuited = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited), RollingCountTimeout = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Timeout), RollingCountThreadPoolRejected = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected), RollingCountFrameworkExceptionThrown = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FrameworkExceptionThrown), RollingCountServiceExceptionThrown = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ServiceExceptionThrown), RollingCountValidationExceptionThrown = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ValidationExceptionThrown), CumulativeCountSuccess = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.Success), CumulativeCountShortCircuited = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ShortCircuited), CumulativeCountTimeout = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.Timeout), CumulativeCountThreadPoolRejected = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ThreadPoolRejected), CumulativeCountFrameworkExcetpionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.FrameworkExceptionThrown), CumulativeCountServiceExceptionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ServiceExceptionThrown), CumulativeCountValidationExceptionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ValidationExceptionThrown), CurrentConcurrentExecutionCount = commandMetrics.CurrentConcurrentExecutionCount, LatencyExecuteMean = commandMetrics.GetServiceExecutionTimeMean(), LatencyExecute = new PercentileInfo { P0 = commandMetrics.GetServiceExecutionTimePercentile(0), P25 = commandMetrics.GetServiceExecutionTimePercentile(25), P50 = commandMetrics.GetServiceExecutionTimePercentile(50), P75 = commandMetrics.GetServiceExecutionTimePercentile(75), P90 = commandMetrics.GetServiceExecutionTimePercentile(90), P95 = commandMetrics.GetServiceExecutionTimePercentile(95), P99 = commandMetrics.GetServiceExecutionTimePercentile(99), P99DOT5 = commandMetrics.GetServiceExecutionTimePercentile(99.5), P100 = commandMetrics.GetServiceExecutionTimePercentile(100), }, LatencyTotalMean = commandMetrics.GetTotalTimeMean(), LatencyTotal = new PercentileInfo { P0 = commandMetrics.GetTotalTimePercentile(0), P25 = commandMetrics.GetTotalTimePercentile(25), P50 = commandMetrics.GetTotalTimePercentile(50), P75 = commandMetrics.GetTotalTimePercentile(75), P90 = commandMetrics.GetTotalTimePercentile(90), P95 = commandMetrics.GetTotalTimePercentile(95), P99 = commandMetrics.GetTotalTimePercentile(99), P99DOT5 = commandMetrics.GetTotalTimePercentile(99.5), P100 = commandMetrics.GetTotalTimePercentile(100), }, PropertyValue_CircuitBreakerRequestVolumeThreshold = commandProperties.CircuitBreakerRequestVolumeThreshold.Get(), PropertyValue_CircuitBreakerSleepWindowInMilliseconds = (long)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds, PropertyValue_CircuitBreakerErrorThresholdPercentage = commandProperties.CircuitBreakerErrorThresholdPercentage.Get(), PropertyValue_CircuitBreakerForceOpen = commandProperties.CircuitBreakerForceOpen.Get(), PropertyValue_CircuitBreakerForceClosed = commandProperties.CircuitBreakerForceClosed.Get(), PropertyValue_CircuitBreakerEnabled = commandProperties.CircuitBreakerEnabled.Get(), PropertyValue_ExecutionIsolationThreadTimeoutInMilliseconds = (long)operation.HystrixCommand.GetExecutionTimeout().TotalMilliseconds, PropertyValue_MetricsRollingStatisticalWindowInMilliseconds = commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get(), PropertyValue_RequestLogEnabled = commandProperties.RequestLogEnabled.Get(), ReportingHosts = 1, }; result.Add(commandInfo); } return(result); }
public IHystrixMetricsPublisherCommand GetPublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandProperties properties) { return this.commandPublishers.GetOrAdd(commandKey.Name, w => this.strategy.GetMetricsPublisherForCommand(commandKey, commandOwner, metrics, circuitBreaker, properties), w => w.Initialize()); }
public static IHystrixMetricsPublisherCommand CreateOrRetrievePublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandProperties properties) { return instance.GetPublisherForCommand(commandKey, commandOwner, metrics, circuitBreaker, properties); }
/// <summary> /// Produces JSON formatted metrics data from an instance of <see cref="HystrixCommandMetrics"/>. /// </summary> /// <param name="commandMetrics">The metrics of a command.</param> /// <returns>JSON formatted metrics data.</returns> private static string CreateCommandSampleData(HystrixCommandMetrics commandMetrics) { IHystrixCircuitBreaker circuitBreaker = HystrixCircuitBreakerFactory.GetInstance(commandMetrics.CommandKey); HealthCounts healthCounts = commandMetrics.GetHealthCounts(); IHystrixCommandProperties commandProperties = commandMetrics.Properties; JObject data = new JObject( new JProperty("type", "HystrixCommand"), new JProperty("name", commandMetrics.CommandKey.Name), new JProperty("group", commandMetrics.CommandGroup.Name), new JProperty("currentTime", GetCurrentTimeForJavascript()), circuitBreaker == null ? new JProperty("isCircuitBreakerOpen", false) : new JProperty("isCircuitBreakerOpen", circuitBreaker.IsOpen()), new JProperty("errorPercentage", healthCounts.ErrorPercentage), // health counts new JProperty("errorCount", healthCounts.ErrorCount), new JProperty("requestCount", healthCounts.TotalRequests), new JProperty("rollingCountCollapsedRequests", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Collapsed)), // rolling counters new JProperty("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown)), new JProperty("rollingCountFailure", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Failure)), new JProperty("rollingCountFallbackFailure", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure)), new JProperty("rollingCountFallbackRejection", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection)), new JProperty("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess)), new JProperty("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache)), new JProperty("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected)), new JProperty("rollingCountShortCircuited", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited)), new JProperty("rollingCountSuccess", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Success)), new JProperty("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected)), new JProperty("rollingCountTimeout", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Timeout)), new JProperty("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount), new JProperty("latencyExecute_mean", commandMetrics.GetExecutionTimeMean()), // latency percentiles new JProperty( "latencyExecute", new JObject( new JProperty("0", commandMetrics.GetExecutionTimePercentile(0)), new JProperty("25", commandMetrics.GetExecutionTimePercentile(25)), new JProperty("50", commandMetrics.GetExecutionTimePercentile(50)), new JProperty("75", commandMetrics.GetExecutionTimePercentile(75)), new JProperty("90", commandMetrics.GetExecutionTimePercentile(90)), new JProperty("95", commandMetrics.GetExecutionTimePercentile(95)), new JProperty("99", commandMetrics.GetExecutionTimePercentile(99)), new JProperty("99.5", commandMetrics.GetExecutionTimePercentile(99.5)), new JProperty("100", commandMetrics.GetExecutionTimePercentile(100)))), new JProperty("latencyTotal_mean", commandMetrics.GetTotalTimeMean()), new JProperty( "latencyTotal", new JObject( new JProperty("0", commandMetrics.GetTotalTimePercentile(0)), new JProperty("25", commandMetrics.GetTotalTimePercentile(25)), new JProperty("50", commandMetrics.GetTotalTimePercentile(50)), new JProperty("75", commandMetrics.GetTotalTimePercentile(75)), new JProperty("90", commandMetrics.GetTotalTimePercentile(90)), new JProperty("95", commandMetrics.GetTotalTimePercentile(95)), new JProperty("99", commandMetrics.GetTotalTimePercentile(99)), new JProperty("99.5", commandMetrics.GetTotalTimePercentile(99.5)), new JProperty("100", commandMetrics.GetTotalTimePercentile(100)))), new JProperty("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold.Get()), // property values for reporting what is actually seen by the command rather than what was set somewhere new JProperty("propertyValue_circuitBreakerSleepWindowInMilliseconds", (long)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds), new JProperty("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage.Get()), new JProperty("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen.Get()), new JProperty("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed.Get()), new JProperty("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled.Get()), new JProperty("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy.Get()), new JProperty("propertyValue_executionIsolationThreadTimeoutInMilliseconds", (long)commandProperties.ExecutionIsolationThreadTimeout.Get().TotalMilliseconds), new JProperty("propertyValue_executionIsolationThreadInterruptOnTimeout", commandProperties.ExecutionIsolationThreadInterruptOnTimeout.Get()), new JProperty("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.ExecutionIsolationThreadPoolKeyOverride.Get()), new JProperty("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Get()), new JProperty("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests.Get()), new JProperty("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get()), new JProperty("propertyValue_requestCacheEnabled", commandProperties.RequestCacheEnabled.Get()), new JProperty("propertyValue_requestLogEnabled", commandProperties.RequestLogEnabled.Get()), new JProperty("reportingHosts", 1)); return(data.ToString(Formatting.None)); }
private static IHystrixCircuitBreaker GetCircuitBreaker(HystrixCommandKey key, HystrixCommandGroupKey commandGroup, HystrixCommandMetrics metrics, HystrixCommandPropertiesSetter properties) { return(new HystrixCircuitBreakerImpl(new MockingHystrixCommandProperties(properties), metrics)); }
public override IHystrixMetricsPublisherCommand GetMetricsPublisherForCommand(IHystrixCommandKey commandKey, IHystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandOptions properties) { return(new MyHystrixMetricsPublisherCommand(commandCounter)); }
public void CircuitBreaker_MultipleTimeWindowRetriesBeforeClosingCircuit() { try { int sleepWindow = 200; HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); // the 'singleTest' fails so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep metrics.MarkFailure(1000); Assert.IsFalse(cb.AllowRequest()); Assert.IsFalse(cb.AllowRequest()); Assert.IsFalse(cb.AllowRequest()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); // the 'singleTest' fails again so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep metrics.MarkFailure(1000); Assert.IsFalse(cb.AllowRequest()); Assert.IsFalse(cb.AllowRequest()); Assert.IsFalse(cb.AllowRequest()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); // now it finally succeeds metrics.MarkSuccess(200); cb.MarkSuccess(); // all requests should be open again Assert.IsTrue(cb.AllowRequest()); Assert.IsTrue(cb.AllowRequest()); Assert.IsTrue(cb.AllowRequest()); // and the circuit should be closed again Assert.IsFalse(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
public IHystrixMetricsPublisherCommand GetMetricsPublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandGroupKey, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandProperties properties) { return(new HystrixServoMetricsPublisherCommand(commandKey, commandGroupKey, metrics, circuitBreaker, properties)); }
public virtual IHystrixMetricsPublisherCommand GetMetricsPublisherForCommand(IHystrixCommandKey commandKey, IHystrixCommandGroupKey commandGroupKey, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandOptions properties) { return(new HystrixMetricsPublisherCommandDefault(commandKey, commandGroupKey, metrics, circuitBreaker, properties)); }
public static IHystrixMetricsPublisherCommand CreateOrRetrievePublisherForCommand(IHystrixCommandKey commandKey, IHystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandOptions properties) { return(SINGLETON.GetPublisherForCommand(commandKey, commandOwner, metrics, circuitBreaker, properties)); }
internal static HystrixCommandMetrics GetMetrics(IHystrixCommandKey commandKey, HystrixCommandOptions properties) { return(HystrixCommandMetrics.GetInstance(commandKey, CommandOwnerForUnitTest.OWNER_ONE, ThreadPoolKeyForUnitTest.THREAD_POOL_ONE, properties)); }
internal IHystrixMetricsPublisherCommand GetPublisherForCommand(IHystrixCommandKey commandKey, IHystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandOptions properties) { return(commandPublishers.GetOrAddEx(commandKey.Name, (k) => { IHystrixMetricsPublisherCommand newPublisher = HystrixPlugins.MetricsPublisher.GetMetricsPublisherForCommand(commandKey, commandOwner, metrics, circuitBreaker, properties); newPublisher.Initialize(); return newPublisher; })); }
private static HystrixCommandUtilization SampleCommandUtilization(HystrixCommandMetrics commandMetrics) { return(HystrixCommandUtilization.Sample(commandMetrics)); }
public void TestGetErrorPercentage() { String key = "cmd-metrics-A"; HystrixCommand <bool> cmd1 = new SuccessCommand(key, 1); HystrixCommandMetrics metrics = cmd1.metrics; cmd1.Execute(); Time.Wait(200); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(0, metrics.Healthcounts.ErrorPercentage); HystrixCommand <bool> cmd2 = new FailureCommand(key, 1); cmd2.Execute(); Time.Wait(200); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(50, metrics.Healthcounts.ErrorPercentage); HystrixCommand <bool> cmd3 = new SuccessCommand(key, 1); HystrixCommand <bool> cmd4 = new SuccessCommand(key, 1); cmd3.Execute(); cmd4.Execute(); Time.Wait(200); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(25, metrics.Healthcounts.ErrorPercentage); HystrixCommand <bool> cmd5 = new TimeoutCommand(key); HystrixCommand <bool> cmd6 = new TimeoutCommand(key); cmd5.Execute(); cmd6.Execute(); Time.Wait(200); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); Assert.Equal(50, metrics.Healthcounts.ErrorPercentage); HystrixCommand <bool> cmd7 = new SuccessCommand(key, 1); HystrixCommand <bool> cmd8 = new SuccessCommand(key, 1); HystrixCommand <bool> cmd9 = new SuccessCommand(key, 1); cmd7.Execute(); cmd8.Execute(); cmd9.Execute(); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); // latent HystrixCommand <bool> cmd10 = new SuccessCommand(key, 60); cmd10.Execute(); output.WriteLine("ReqLog : " + HystrixRequestLog.CurrentRequestLog.GetExecutedCommandsAsString()); // 6 success + 1 latent success + 1 failure + 2 timeout = 10 total // latent success not considered error // error percentage = 1 failure + 2 timeout / 10 Time.Wait(200); Assert.Equal(30, metrics.Healthcounts.ErrorPercentage); }
public virtual IHystrixMetricsPublisherCommand GetMetricsPublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandGroupKey, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandProperties properties) { return new HystrixMetricsPublisherCommandDefault(commandKey, commandGroupKey, metrics, circuitBreaker, properties); }
public static IHystrixMetricsPublisherCommand CreateOrRetrievePublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandProperties properties) { return(instance.GetPublisherForCommand(commandKey, commandOwner, metrics, circuitBreaker, properties)); }
private static IHystrixCircuitBreaker GetCircuitBreaker(HystrixCommandKey key, HystrixCommandGroupKey commandGroup, HystrixCommandMetrics metrics, HystrixCommandPropertiesSetter properties) { return new HystrixCircuitBreakerImpl(new MockingHystrixCommandProperties(properties), metrics); }
public IHystrixMetricsPublisherCommand GetPublisherForCommand(HystrixCommandKey commandKey, HystrixCommandGroupKey commandOwner, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandProperties properties) { return(this.commandPublishers.GetOrAdd(commandKey.Name, w => this.strategy.GetMetricsPublisherForCommand(commandKey, commandOwner, metrics, circuitBreaker, properties), w => w.Initialize())); }
private static void WriteCommandMetrics(JsonTextWriter writer, HystrixCommandMetrics commandMetrics) { IHystrixCommandKey key = commandMetrics.CommandKey; IHystrixCircuitBreaker circuitBreaker = HystrixCircuitBreakerFactory.GetInstance(key); writer.WriteStartObject(); writer.WriteStringField("type", "HystrixCommand"); writer.WriteStringField("name", key.Name); writer.WriteStringField("group", commandMetrics.CommandGroup.Name); writer.WriteLongField("currentTime", Time.CurrentTimeMillisJava); // circuit breaker if (circuitBreaker == null) { // circuit breaker is disabled and thus never open writer.WriteBooleanField("isCircuitBreakerOpen", false); } else { writer.WriteBooleanField("isCircuitBreakerOpen", circuitBreaker.IsOpen); } HealthCounts healthCounts = commandMetrics.Healthcounts; writer.WriteIntegerField("errorPercentage", healthCounts.ErrorPercentage); writer.WriteLongField("errorCount", healthCounts.ErrorCount); writer.WriteLongField("requestCount", healthCounts.TotalRequests); // rolling counters writer.WriteLongField("rollingCountBadRequests", commandMetrics.GetRollingCount(HystrixEventType.BAD_REQUEST)); writer.WriteLongField("rollingCountCollapsedRequests", commandMetrics.GetRollingCount(HystrixEventType.COLLAPSED)); writer.WriteLongField("rollingCountEmit", commandMetrics.GetRollingCount(HystrixEventType.EMIT)); writer.WriteLongField("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(HystrixEventType.EXCEPTION_THROWN)); writer.WriteLongField("rollingCountFailure", commandMetrics.GetRollingCount(HystrixEventType.FAILURE)); writer.WriteLongField("rollingCountFallbackEmit", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_EMIT)); writer.WriteLongField("rollingCountFallbackFailure", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_FAILURE)); writer.WriteLongField("rollingCountFallbackMissing", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_MISSING)); writer.WriteLongField("rollingCountFallbackRejection", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_REJECTION)); writer.WriteLongField("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(HystrixEventType.FALLBACK_SUCCESS)); writer.WriteLongField("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(HystrixEventType.RESPONSE_FROM_CACHE)); writer.WriteLongField("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(HystrixEventType.SEMAPHORE_REJECTED)); writer.WriteLongField("rollingCountShortCircuited", commandMetrics.GetRollingCount(HystrixEventType.SHORT_CIRCUITED)); writer.WriteLongField("rollingCountSuccess", commandMetrics.GetRollingCount(HystrixEventType.SUCCESS)); writer.WriteLongField("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(HystrixEventType.THREAD_POOL_REJECTED)); writer.WriteLongField("rollingCountTimeout", commandMetrics.GetRollingCount(HystrixEventType.TIMEOUT)); writer.WriteIntegerField("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount); writer.WriteLongField("rollingMaxConcurrentExecutionCount", commandMetrics.RollingMaxConcurrentExecutions); // latency percentiles writer.WriteIntegerField("latencyExecute_mean", commandMetrics.ExecutionTimeMean); writer.WriteObjectFieldStart("latencyExecute"); writer.WriteIntegerField("0", commandMetrics.GetExecutionTimePercentile(0)); writer.WriteIntegerField("25", commandMetrics.GetExecutionTimePercentile(25)); writer.WriteIntegerField("50", commandMetrics.GetExecutionTimePercentile(50)); writer.WriteIntegerField("75", commandMetrics.GetExecutionTimePercentile(75)); writer.WriteIntegerField("90", commandMetrics.GetExecutionTimePercentile(90)); writer.WriteIntegerField("95", commandMetrics.GetExecutionTimePercentile(95)); writer.WriteIntegerField("99", commandMetrics.GetExecutionTimePercentile(99)); writer.WriteIntegerField("99.5", commandMetrics.GetExecutionTimePercentile(99.5)); writer.WriteIntegerField("100", commandMetrics.GetExecutionTimePercentile(100)); writer.WriteEndObject(); writer.WriteIntegerField("latencyTotal_mean", commandMetrics.TotalTimeMean); writer.WriteObjectFieldStart("latencyTotal"); writer.WriteIntegerField("0", commandMetrics.GetTotalTimePercentile(0)); writer.WriteIntegerField("25", commandMetrics.GetTotalTimePercentile(25)); writer.WriteIntegerField("50", commandMetrics.GetTotalTimePercentile(50)); writer.WriteIntegerField("75", commandMetrics.GetTotalTimePercentile(75)); writer.WriteIntegerField("90", commandMetrics.GetTotalTimePercentile(90)); writer.WriteIntegerField("95", commandMetrics.GetTotalTimePercentile(95)); writer.WriteIntegerField("99", commandMetrics.GetTotalTimePercentile(99)); writer.WriteIntegerField("99.5", commandMetrics.GetTotalTimePercentile(99.5)); writer.WriteIntegerField("100", commandMetrics.GetTotalTimePercentile(100)); writer.WriteEndObject(); // property values for reporting what is actually seen by the command rather than what was set somewhere IHystrixCommandOptions commandProperties = commandMetrics.Properties; writer.WriteIntegerField("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold); writer.WriteIntegerField("propertyValue_circuitBreakerSleepWindowInMilliseconds", commandProperties.CircuitBreakerSleepWindowInMilliseconds); writer.WriteIntegerField("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage); writer.WriteBooleanField("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen); writer.WriteBooleanField("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed); writer.WriteBooleanField("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled); writer.WriteStringField("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy.ToString()); writer.WriteIntegerField("propertyValue_executionIsolationThreadTimeoutInMilliseconds", commandProperties.ExecutionTimeoutInMilliseconds); writer.WriteIntegerField("propertyValue_executionTimeoutInMilliseconds", commandProperties.ExecutionTimeoutInMilliseconds); writer.WriteBooleanField("propertyValue_executionIsolationThreadInterruptOnTimeout", false); writer.WriteStringField("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.ExecutionIsolationThreadPoolKeyOverride); writer.WriteIntegerField("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests); writer.WriteIntegerField("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests); writer.WriteIntegerField("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds); writer.WriteBooleanField("propertyValue_requestCacheEnabled", commandProperties.RequestCacheEnabled); writer.WriteBooleanField("propertyValue_requestLogEnabled", commandProperties.RequestLogEnabled); writer.WriteIntegerField("reportingHosts", 1); // this will get summed across all instances in a cluster writer.WriteStringField("threadPool", commandMetrics.ThreadPoolKey.Name); writer.WriteEndObject(); }
public static ICircuitBreaker GetInstance(IHystrixCommandKey key, IHystrixCommandGroupKey group, IHystrixCommandOptions options, HystrixCommandMetrics metrics) { return(CircuitBreakersByCommand.GetOrAddEx(key.Name, (k) => new HystrixCircuitBreakerImpl(key, group, options, metrics))); }
public static HystrixCommandUtilization Sample(HystrixCommandMetrics commandMetrics) { return(new HystrixCommandUtilization(commandMetrics.CurrentConcurrentExecutionCount)); }
/// <summary> /// Gets the <see cref="IHystrixCircuitBreaker"/> instance for a given <see cref="HystrixCommandKey"/>. /// If no circuit breaker exists for the specified command key, a new one will be created using the properties and metrics parameters. /// If a circuit breaker already exists, those parameters will be ignored. /// </summary> /// <param name="commandKey">Command key of command instance requesting the circuit breaker.</param> /// <param name="properties">The properties of the specified command.</param> /// <param name="metrics">The metrics of the specified command.</param> /// <returns>A new or an existing circuit breaker instance.</returns> public static IHystrixCircuitBreaker GetInstance(HystrixCommandKey commandKey, IHystrixCommandProperties properties, HystrixCommandMetrics metrics) { return(Instances.GetOrAdd(commandKey, w => new HystrixCircuitBreakerImpl(properties, metrics))); }
public override IHystrixMetricsPublisherCommand GetMetricsPublisherForCommand(IHystrixCommandKey commandKey, IHystrixCommandGroupKey commandGroupKey, HystrixCommandMetrics metrics, IHystrixCircuitBreaker circuitBreaker, IHystrixCommandOptions properties) { return(commandToReturn); }