public void CircuitBreaker_TripCircuit() { try { HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter(); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); metrics.MarkSuccess(1000); metrics.MarkSuccess(1000); metrics.MarkSuccess(1000); metrics.MarkSuccess(1000); // this should still allow requests as everything has been successful Assert.IsTrue(cb.AllowRequest()); Assert.IsFalse(cb.IsOpen()); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
public void CircuitBreaker_TripCircuitOnFailuresAboveThreshold() { try { HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter(); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // this should start as allowing requests Assert.IsTrue(cb.AllowRequest()); Assert.IsFalse(cb.IsOpen()); // success with high latency metrics.MarkSuccess(400); metrics.MarkSuccess(400); metrics.MarkFailure(10); metrics.MarkSuccess(400); metrics.MarkFailure(10); metrics.MarkFailure(10); metrics.MarkSuccess(400); metrics.MarkFailure(10); metrics.MarkFailure(10); // this should trip the circuit as the error percentage is above the threshold Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
public void CircuitBreaker_SingleTestOnOpenCircuitAfterTimeWindow() { try { int sleepWindow = 200; HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
public void CircuitBreaker_TripCircuitOnTimeouts() { try { HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter(); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // this should start as allowing requests Assert.IsTrue(cb.AllowRequest()); Assert.IsFalse(cb.IsOpen()); // timeouts metrics.MarkTimeout(2000); metrics.MarkTimeout(2000); metrics.MarkTimeout(2000); metrics.MarkTimeout(2000); // everything has been a timeout so we should not allow any requests Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
public void CircuitBreaker_LowVolumeDoesNotTripCircuit() { try { int sleepWindow = 200; int lowVolume = 5; HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithCircuitBreakerRequestVolumeThreshold(lowVolume); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // even though it has all failed we won't trip the circuit because the volume is low Assert.IsTrue(cb.AllowRequest()); Assert.IsFalse(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
public void CircuitBreaker_CircuitClosedAfterSuccessAndClearsStatisticalWindow() { try { int statisticalWindow = 200; int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); // the 'singleTest' succeeds so should cause the circuit to be closed metrics.MarkSuccess(500); cb.MarkSuccess(); // all requests should be open again Assert.IsTrue(cb.AllowRequest()); Assert.IsTrue(cb.AllowRequest()); Assert.IsTrue(cb.AllowRequest()); // and the circuit should be closed again Assert.IsFalse(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }
private static List <GlobalStreamHystrixCommandInfo> GetHystrixCommandInfoList() { var result = new List <GlobalStreamHystrixCommandInfo>(); foreach (ServiceMetadata metadata in EndpointHost.Config.MetadataMap.Values) { string refinedServiceName = ServiceUtils.RefineServiceName(metadata.ServiceNamespace, metadata.ServiceName); foreach (Operation operation in metadata.Operations) { HystrixCommandMetrics commandMetrics = operation.HystrixCommand.Metrics; IHystrixCircuitBreaker circuitBreaker = operation.HystrixCommand.CircuitBreaker; HealthCounts healthCounts = commandMetrics.GetHealthCounts(); IHystrixCommandProperties commandProperties = commandMetrics.Properties; GlobalStreamHystrixCommandInfo hystrixCommandInfo = new GlobalStreamHystrixCommandInfo() { type = TurbineDataTypeHystrixCommand, name = refinedServiceName + "." + operation.Name.ToLower(), group = refinedServiceName, currentTime = DateTime.Now.ToUnixTimeMs(), isCircuitBreakerOpen = circuitBreaker.IsOpen(), errorPercentage = healthCounts.ErrorPercentage, errorCount = healthCounts.TotalErrorCount, requestCount = healthCounts.TotalRequests, rollingCountExceptionsThrown = healthCounts.TotalExceptionCount, rollingCountFailure = healthCounts.TotalFailureCount, rollingCountSemaphoreRejected = 0, rollingCountShortCircuited = healthCounts.ShortCircuitedCount, rollingCountSuccess = healthCounts.SuccessCount, rollingCountThreadPoolRejected = 0, rollingCountTimeout = healthCounts.TimeoutCount, rollingCountFallbackFailure = 0, rollingCountFallbackSuccess = 0, rollingCountFallbackRejection = 0, latencyExecute = new GlobalStreamPercentileInfo() { P0 = commandMetrics.GetServiceExecutionTimePercentile(0), P25 = commandMetrics.GetServiceExecutionTimePercentile(25), P50 = commandMetrics.GetServiceExecutionTimePercentile(50), P75 = commandMetrics.GetServiceExecutionTimePercentile(75), P90 = commandMetrics.GetServiceExecutionTimePercentile(90), P95 = commandMetrics.GetServiceExecutionTimePercentile(95), P99 = commandMetrics.GetServiceExecutionTimePercentile(99), P99DOT5 = commandMetrics.GetServiceExecutionTimePercentile(99.5), P100 = commandMetrics.GetServiceExecutionTimePercentile(100) }, latencyExecute_mean = commandMetrics.GetServiceExecutionTimeMean(), latencyTotal = new GlobalStreamPercentileInfo() { P0 = commandMetrics.GetTotalTimePercentile(0), P25 = commandMetrics.GetTotalTimePercentile(25), P50 = commandMetrics.GetTotalTimePercentile(50), P75 = commandMetrics.GetTotalTimePercentile(75), P90 = commandMetrics.GetTotalTimePercentile(90), P95 = commandMetrics.GetTotalTimePercentile(95), P99 = commandMetrics.GetTotalTimePercentile(99), P99DOT5 = commandMetrics.GetTotalTimePercentile(99.5), P100 = commandMetrics.GetTotalTimePercentile(100) }, latencyTotal_mean = commandMetrics.GetTotalTimeMean(), reportingHosts = 1, propertyValue_circuitBreakerEnabled = commandProperties.CircuitBreakerEnabled.Get(), propertyValue_circuitBreakerErrorThresholdPercentage = commandProperties.CircuitBreakerErrorThresholdPercentage.Get(), propertyValue_circuitBreakerForceClosed = commandProperties.CircuitBreakerForceClosed.Get(), propertyValue_circuitBreakerForceOpen = commandProperties.CircuitBreakerForceOpen.Get(), propertyValue_circuitBreakerRequestVolumeThreshold = commandProperties.CircuitBreakerRequestVolumeThreshold.Get(), propertyValue_circuitBreakerSleepWindowInMilliseconds = (int)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds, propertyValue_executionIsolationSemaphoreMaxConcurrentRequests = 1000, propertyValue_executionIsolationStrategy = TurbineStrategySemaphore, propertyValue_executionIsolationThreadTimeoutInMilliseconds = (int)operation.HystrixCommand.GetExecutionTimeout().TotalMilliseconds, propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests = 0, propertyValue_metricsRollingStatisticalWindowInMilliseconds = commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get(), currentConcurrentExecutionCount = commandMetrics.CurrentConcurrentExecutionCount }; result.Add(hystrixCommandInfo); } } return(result); }
public static List <HystrixCommandInfo> GetHystrixCommandInfo(string servicePath) { var result = new List <HystrixCommandInfo>(); ServiceMetadata metadata = EndpointHost.Config.MetadataMap[servicePath]; foreach (Operation operation in metadata.Operations) { HystrixCommandMetrics commandMetrics = operation.HystrixCommand.Metrics; IHystrixCircuitBreaker circuitBreaker = operation.HystrixCommand.CircuitBreaker; HealthCounts healthCounts = commandMetrics.GetHealthCounts(); IHystrixCommandProperties commandProperties = commandMetrics.Properties; var commandInfo = new HystrixCommandInfo { Type = "HystrixCommand", Name = commandMetrics.OperationName, Group = commandMetrics.FullServiceName, CurrentTime = DateTime.Now, IsCircuitBreakerOpen = (circuitBreaker == null ? false : circuitBreaker.IsOpen()), ErrorPercentage = healthCounts.ErrorPercentage, ErrorCount = healthCounts.TotalErrorCount, RequestCount = healthCounts.TotalRequests, RollingCountSuccess = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Success), RollingCountShortCircuited = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited), RollingCountTimeout = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Timeout), RollingCountThreadPoolRejected = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected), RollingCountFrameworkExceptionThrown = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FrameworkExceptionThrown), RollingCountServiceExceptionThrown = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ServiceExceptionThrown), RollingCountValidationExceptionThrown = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ValidationExceptionThrown), CumulativeCountSuccess = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.Success), CumulativeCountShortCircuited = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ShortCircuited), CumulativeCountTimeout = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.Timeout), CumulativeCountThreadPoolRejected = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ThreadPoolRejected), CumulativeCountFrameworkExcetpionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.FrameworkExceptionThrown), CumulativeCountServiceExceptionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ServiceExceptionThrown), CumulativeCountValidationExceptionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ValidationExceptionThrown), CurrentConcurrentExecutionCount = commandMetrics.CurrentConcurrentExecutionCount, LatencyExecuteMean = commandMetrics.GetServiceExecutionTimeMean(), LatencyExecute = new PercentileInfo { P0 = commandMetrics.GetServiceExecutionTimePercentile(0), P25 = commandMetrics.GetServiceExecutionTimePercentile(25), P50 = commandMetrics.GetServiceExecutionTimePercentile(50), P75 = commandMetrics.GetServiceExecutionTimePercentile(75), P90 = commandMetrics.GetServiceExecutionTimePercentile(90), P95 = commandMetrics.GetServiceExecutionTimePercentile(95), P99 = commandMetrics.GetServiceExecutionTimePercentile(99), P99DOT5 = commandMetrics.GetServiceExecutionTimePercentile(99.5), P100 = commandMetrics.GetServiceExecutionTimePercentile(100), }, LatencyTotalMean = commandMetrics.GetTotalTimeMean(), LatencyTotal = new PercentileInfo { P0 = commandMetrics.GetTotalTimePercentile(0), P25 = commandMetrics.GetTotalTimePercentile(25), P50 = commandMetrics.GetTotalTimePercentile(50), P75 = commandMetrics.GetTotalTimePercentile(75), P90 = commandMetrics.GetTotalTimePercentile(90), P95 = commandMetrics.GetTotalTimePercentile(95), P99 = commandMetrics.GetTotalTimePercentile(99), P99DOT5 = commandMetrics.GetTotalTimePercentile(99.5), P100 = commandMetrics.GetTotalTimePercentile(100), }, PropertyValue_CircuitBreakerRequestVolumeThreshold = commandProperties.CircuitBreakerRequestVolumeThreshold.Get(), PropertyValue_CircuitBreakerSleepWindowInMilliseconds = (long)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds, PropertyValue_CircuitBreakerErrorThresholdPercentage = commandProperties.CircuitBreakerErrorThresholdPercentage.Get(), PropertyValue_CircuitBreakerForceOpen = commandProperties.CircuitBreakerForceOpen.Get(), PropertyValue_CircuitBreakerForceClosed = commandProperties.CircuitBreakerForceClosed.Get(), PropertyValue_CircuitBreakerEnabled = commandProperties.CircuitBreakerEnabled.Get(), PropertyValue_ExecutionIsolationThreadTimeoutInMilliseconds = (long)operation.HystrixCommand.GetExecutionTimeout().TotalMilliseconds, PropertyValue_MetricsRollingStatisticalWindowInMilliseconds = commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get(), PropertyValue_RequestLogEnabled = commandProperties.RequestLogEnabled.Get(), ReportingHosts = 1, }; result.Add(commandInfo); } return(result); }
/// <summary> /// Produces JSON formatted metrics data from an instance of <see cref="HystrixCommandMetrics"/>. /// </summary> /// <param name="commandMetrics">The metrics of a command.</param> /// <returns>JSON formatted metrics data.</returns> private static string CreateCommandSampleData(HystrixCommandMetrics commandMetrics) { IHystrixCircuitBreaker circuitBreaker = HystrixCircuitBreakerFactory.GetInstance(commandMetrics.CommandKey); HealthCounts healthCounts = commandMetrics.GetHealthCounts(); IHystrixCommandProperties commandProperties = commandMetrics.Properties; JObject data = new JObject( new JProperty("type", "HystrixCommand"), new JProperty("name", commandMetrics.CommandKey.Name), new JProperty("group", commandMetrics.CommandGroup.Name), new JProperty("currentTime", GetCurrentTimeForJavascript()), circuitBreaker == null ? new JProperty("isCircuitBreakerOpen", false) : new JProperty("isCircuitBreakerOpen", circuitBreaker.IsOpen()), new JProperty("errorPercentage", healthCounts.ErrorPercentage), // health counts new JProperty("errorCount", healthCounts.ErrorCount), new JProperty("requestCount", healthCounts.TotalRequests), new JProperty("rollingCountCollapsedRequests", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Collapsed)), // rolling counters new JProperty("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown)), new JProperty("rollingCountFailure", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Failure)), new JProperty("rollingCountFallbackFailure", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure)), new JProperty("rollingCountFallbackRejection", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection)), new JProperty("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess)), new JProperty("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache)), new JProperty("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected)), new JProperty("rollingCountShortCircuited", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited)), new JProperty("rollingCountSuccess", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Success)), new JProperty("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected)), new JProperty("rollingCountTimeout", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Timeout)), new JProperty("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount), new JProperty("latencyExecute_mean", commandMetrics.GetExecutionTimeMean()), // latency percentiles new JProperty( "latencyExecute", new JObject( new JProperty("0", commandMetrics.GetExecutionTimePercentile(0)), new JProperty("25", commandMetrics.GetExecutionTimePercentile(25)), new JProperty("50", commandMetrics.GetExecutionTimePercentile(50)), new JProperty("75", commandMetrics.GetExecutionTimePercentile(75)), new JProperty("90", commandMetrics.GetExecutionTimePercentile(90)), new JProperty("95", commandMetrics.GetExecutionTimePercentile(95)), new JProperty("99", commandMetrics.GetExecutionTimePercentile(99)), new JProperty("99.5", commandMetrics.GetExecutionTimePercentile(99.5)), new JProperty("100", commandMetrics.GetExecutionTimePercentile(100)))), new JProperty("latencyTotal_mean", commandMetrics.GetTotalTimeMean()), new JProperty( "latencyTotal", new JObject( new JProperty("0", commandMetrics.GetTotalTimePercentile(0)), new JProperty("25", commandMetrics.GetTotalTimePercentile(25)), new JProperty("50", commandMetrics.GetTotalTimePercentile(50)), new JProperty("75", commandMetrics.GetTotalTimePercentile(75)), new JProperty("90", commandMetrics.GetTotalTimePercentile(90)), new JProperty("95", commandMetrics.GetTotalTimePercentile(95)), new JProperty("99", commandMetrics.GetTotalTimePercentile(99)), new JProperty("99.5", commandMetrics.GetTotalTimePercentile(99.5)), new JProperty("100", commandMetrics.GetTotalTimePercentile(100)))), new JProperty("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold.Get()), // property values for reporting what is actually seen by the command rather than what was set somewhere new JProperty("propertyValue_circuitBreakerSleepWindowInMilliseconds", (long)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds), new JProperty("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage.Get()), new JProperty("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen.Get()), new JProperty("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed.Get()), new JProperty("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled.Get()), new JProperty("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy.Get()), new JProperty("propertyValue_executionIsolationThreadTimeoutInMilliseconds", (long)commandProperties.ExecutionIsolationThreadTimeout.Get().TotalMilliseconds), new JProperty("propertyValue_executionIsolationThreadInterruptOnTimeout", commandProperties.ExecutionIsolationThreadInterruptOnTimeout.Get()), new JProperty("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.ExecutionIsolationThreadPoolKeyOverride.Get()), new JProperty("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Get()), new JProperty("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests.Get()), new JProperty("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get()), new JProperty("propertyValue_requestCacheEnabled", commandProperties.RequestCacheEnabled.Get()), new JProperty("propertyValue_requestLogEnabled", commandProperties.RequestLogEnabled.Get()), new JProperty("reportingHosts", 1)); return(data.ToString(Formatting.None)); }
/** * Servo will flatten metric names as: getServoTypeTag()_getServoInstanceTag()_monitorName */ private List <IMonitor> getServoMonitors() { List <IMonitor> monitors = new List <IMonitor>(); monitors.Add(ServoMetricFactory.InformationalMetric <bool>(this, MonitorConfig.builder("isCircuitBreakerOpen").build(), () => circuitBreaker.IsOpen())); // allow Servo and monitor to know exactly at what point in time these stats are for so they can be plotted accurately monitors.Add(getCurrentValueMonitor("currentTime", () => DateTime.UtcNow.ToUnixTimestamp(), DataSourceLevel.DEBUG)); // cumulative counts //monitors.Add(getCumulativeMonitor("countBadRequests", HystrixEventType.BAD_REQUEST)); // monitors.add(getCumulativeMonitor("countCollapsedRequests", HystrixEventType.COLLAPSED)); // monitors.add(getCumulativeMonitor("countEmit", HystrixEventType.EMIT)); monitors.Add(getCumulativeMonitor("countExceptionsThrown", HystrixEventType.ExceptionThrown)); monitors.Add(getCumulativeMonitor("countFailure", HystrixEventType.Failure)); // monitors.add(getCumulativeMonitor("countFallbackEmit", HystrixEventType.FALLBACK_EMIT)); monitors.Add(getCumulativeMonitor("countFallbackFailure", HystrixEventType.FallbackFailure)); // monitors.add(getCumulativeMonitor("countFallbackMissing", HystrixEventType.FALLBACK_MISSING)); monitors.Add(getCumulativeMonitor("countFallbackRejection", HystrixEventType.FallbackRejection)); monitors.Add(getCumulativeMonitor("countFallbackSuccess", HystrixEventType.FallbackSuccess)); monitors.Add(getCumulativeMonitor("countResponsesFromCache", HystrixEventType.ResponseFromCache)); monitors.Add(getCumulativeMonitor("countSemaphoreRejected", HystrixEventType.SemaphoreRejected)); monitors.Add(getCumulativeMonitor("countShortCircuited", HystrixEventType.ShortCircuited)); monitors.Add(getCumulativeMonitor("countSuccess", HystrixEventType.Success)); monitors.Add(getCumulativeMonitor("countThreadPoolRejected", HystrixEventType.ThreadPoolRejected)); monitors.Add(getCumulativeMonitor("countTimeout", HystrixEventType.Timeout)); // rolling counts // monitors.add(getRollingMonitor("rollingCountBadRequests", HystrixEventType.BAD_REQUEST)); // monitors.add(getRollingMonitor("rollingCountCollapsedRequests", HystrixEventType.COLLAPSED)); // monitors.add(getRollingMonitor("rollingCountEmit", HystrixEventType.EMIT)); monitors.Add(getRollingMonitor("rollingCountExceptionsThrown", HystrixEventType.ExceptionThrown)); monitors.Add(getRollingMonitor("rollingCountFailure", HystrixEventType.Failure)); //monitors.Add(getRollingMonitor("rollingCountFallbackEmit", HystrixEventType.FALLBACK_EMIT)); monitors.Add(getRollingMonitor("rollingCountFallbackFailure", HystrixEventType.FallbackFailure)); //monitors.Add(getRollingMonitor("rollingCountFallbackMissing", HystrixEventType.FALLBACK_MISSING)); monitors.Add(getRollingMonitor("rollingCountFallbackRejection", HystrixEventType.FallbackRejection)); monitors.Add(getRollingMonitor("rollingCountFallbackSuccess", HystrixEventType.FallbackSuccess)); monitors.Add(getRollingMonitor("rollingCountResponsesFromCache", HystrixEventType.ResponseFromCache)); monitors.Add(getRollingMonitor("rollingCountSemaphoreRejected", HystrixEventType.SemaphoreRejected)); monitors.Add(getRollingMonitor("rollingCountShortCircuited", HystrixEventType.ShortCircuited)); monitors.Add(getRollingMonitor("rollingCountSuccess", HystrixEventType.Success)); monitors.Add(getRollingMonitor("rollingCountThreadPoolRejected", HystrixEventType.ThreadPoolRejected)); monitors.Add(getRollingMonitor("rollingCountTimeout", HystrixEventType.Timeout)); // the number of executionSemaphorePermits in use right now monitors.Add(getCurrentValueMonitor("executionSemaphorePermitsInUse", () => metrics.CurrentConcurrentExecutionCount)); // error percentage derived from current metrics monitors.Add(getCurrentValueMonitor("errorPercentage", () => metrics.GetHealthCounts().ErrorPercentage)); // execution latency metrics monitors.Add(getExecutionLatencyMeanMonitor("latencyExecute_mean")); monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_5", 5)); monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_25", 25)); monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_50", 50)); monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_75", 75)); monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_90", 90)); monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_99", 99)); monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_995", 99.5)); // // total latency metrics monitors.Add(getTotalLatencyMeanMonitor("latencyTotal_mean")); monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_5", 5)); monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_25", 25)); monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_50", 50)); monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_75", 75)); monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_90", 90)); monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_99", 99)); monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_995", 995)); // group //monitors.Add(ServoMetricFactory.InformationalMetric<string>(this, // MonitorConfig.builder("commandGroup").build(), // () => commandGroupKey != null ? commandGroupKey.Name : null)); // properties (so the values can be inspected and monitored) monitors.Add(ServoMetricFactory.InformationalMetric <int>(this, MonitorConfig.builder("propertyValue_rollingStatisticalWindowInMilliseconds").build(), () => properties.MetricsRollingStatisticalWindowInMilliseconds.Get())); monitors.Add(ServoMetricFactory.InformationalMetric <int>(this, MonitorConfig.builder("propertyValue_circuitBreakerRequestVolumeThreshold").build(), () => properties.CircuitBreakerRequestVolumeThreshold.Get())); monitors.Add(ServoMetricFactory.InformationalMetric <long>(this, MonitorConfig.builder("propertyValue_circuitBreakerSleepWindowInMilliseconds").build(), () => properties.CircuitBreakerSleepWindow.Get().Milliseconds)); monitors.Add(ServoMetricFactory.InformationalMetric <int>(this, MonitorConfig.builder("propertyValue_circuitBreakerErrorThresholdPercentage").build(), () => properties.CircuitBreakerErrorThresholdPercentage.Get())); monitors.Add(ServoMetricFactory.InformationalMetric <bool>(this, MonitorConfig.builder("propertyValue_circuitBreakerForceOpen").build(), () => properties.CircuitBreakerForceOpen.Get())); monitors.Add(ServoMetricFactory.InformationalMetric <bool>(this, MonitorConfig.builder("propertyValue_circuitBreakerForceClosed").build(), () => properties.CircuitBreakerForceClosed.Get())); monitors.Add(ServoMetricFactory.InformationalMetric <long>(this, MonitorConfig.builder("propertyValue_executionIsolationThreadTimeoutInMilliseconds").build(), () => properties.ExecutionIsolationThreadTimeout.Get().Milliseconds)); //monitors.Add(ServoMetricFactory.InformationalMetric<TimeSpan>(this, // MonitorConfig.builder("propertyValue_executionTimeoutInMilliseconds").build(), // () => properties.executionTimeoutInMilliseconds.Get())); monitors.Add(ServoMetricFactory.InformationalMetric <ExecutionIsolationStrategy>(this, MonitorConfig.builder("propertyValue_executionIsolationStrategy").build(), () => properties.ExecutionIsolationStrategy.Get())); monitors.Add(ServoMetricFactory.InformationalMetric <bool>(this, MonitorConfig.builder("propertyValue_metricsRollingPercentileEnabled").build(), () => properties.MetricsRollingPercentileEnabled.Get())); monitors.Add(ServoMetricFactory.InformationalMetric <bool>(this, MonitorConfig.builder("propertyValue_requestCacheEnabled").build(), () => properties.RequestCacheEnabled.Get())); monitors.Add(ServoMetricFactory.InformationalMetric <bool>(this, MonitorConfig.builder("propertyValue_requestLogEnabled").build(), () => properties.RequestLogEnabled.Get())); monitors.Add(ServoMetricFactory.InformationalMetric <int>(this, MonitorConfig.builder("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests").build(), () => properties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Get())); monitors.Add(ServoMetricFactory.InformationalMetric <int>(this, MonitorConfig.builder("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests").build(), () => properties.FallbackIsolationSemaphoreMaxConcurrentRequests.Get())); return(monitors); }
public void CircuitBreaker_MultipleTimeWindowRetriesBeforeClosingCircuit() { try { int sleepWindow = 200; HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow); HystrixCommandMetrics metrics = GetMetrics(properties); IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties); // fail metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); metrics.MarkFailure(1000); // everything has failed in the test window so we should return false now Assert.IsFalse(cb.AllowRequest()); Assert.IsTrue(cb.IsOpen()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); // the 'singleTest' fails so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep metrics.MarkFailure(1000); Assert.IsFalse(cb.AllowRequest()); Assert.IsFalse(cb.AllowRequest()); Assert.IsFalse(cb.AllowRequest()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); // the 'singleTest' fails again so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep metrics.MarkFailure(1000); Assert.IsFalse(cb.AllowRequest()); Assert.IsFalse(cb.AllowRequest()); Assert.IsFalse(cb.AllowRequest()); // wait for sleepWindow to pass Thread.Sleep(sleepWindow + 50); // we should now allow 1 request Assert.IsTrue(cb.AllowRequest()); // but the circuit should still be open Assert.IsTrue(cb.IsOpen()); // and further requests are still blocked Assert.IsFalse(cb.AllowRequest()); // now it finally succeeds metrics.MarkSuccess(200); cb.MarkSuccess(); // all requests should be open again Assert.IsTrue(cb.AllowRequest()); Assert.IsTrue(cb.AllowRequest()); Assert.IsTrue(cb.AllowRequest()); // and the circuit should be closed again Assert.IsFalse(cb.IsOpen()); } catch (Exception e) { Console.WriteLine(e.ToString()); Assert.Fail("Error occurred: " + e.Message); } }