public bool IsOpen() { if (!this.ConfigSet.CircuitBreakerEnabled) { return(false); } if (!this.ConfigSet.CircuitBreakerForceOpen) { if (this.OpenFlag != null && this.OpenFlag == true) { return(true); } CommandExecutionHealthSnapshot executionHealthSnapshot = this.Metrics.GetExecutionHealthSnapshot(); if (executionHealthSnapshot.TotalCount < this.ConfigSet.CircuitBreakerRequestCountThreshold) { return(false); } if (executionHealthSnapshot.ErrorPercentage < this.ConfigSet.CircuitBreakerErrorThresholdPercentage) { return(false); } if (this.OpenFlag.CompareAndSet(false, true)) { this.CircuitOpenedOrLastTestedTime.Value = CommonUtils.CurrentTimeInMiliseconds; Dictionary <string, string> tagData = new Dictionary <string, string>(); tagData.Add("CircuitBreaker", "Open"); CommonUtils.Log.Log(LogLevelEnum.Fatal, "Circuit Breaker is open after lots of fail or timeout happen.", tagData.AddLogTagData("FXD303010")); } } return(true); }
public void Reset() { this.ExecutionEventBuffer = new CounterBuffer <CommandExecutionEventEnum>(this.ConfigSet.MetricsRollingStatisticalWindowInMilliseconds, this.ConfigSet.MetricsRollingStatisticalWindowBuckets); this.ExecutionLatencyBuffer = new IntegerPercentileBuffer(this.ConfigSet.MetricsRollingPercentileWindowInMilliseconds, this.ConfigSet.MetricsRollingPercentileWindowBuckets, this.ConfigSet.MetricsRollingPercentileBucketSize); this.TotalExecutionLatencyBuffer = new IntegerPercentileBuffer(this.ConfigSet.MetricsRollingPercentileWindowInMilliseconds, this.ConfigSet.MetricsRollingPercentileWindowBuckets, this.ConfigSet.MetricsRollingPercentileBucketSize); this._lastUpdateExecutionEventSnapshotTimeInMilliseconds = 0L; this._executionEventDistributionSnapshot = new Dictionary <CommandExecutionEventEnum, int>(); this._executionEventHealthSnapshot = new CommandExecutionHealthSnapshot(0, 0); this._lastGetLatencyBufferSnapshotTimeInMilliseconds = 0L; this._latencyBufferSnapshot = new List <long>(); this._lastGetTotalLatencyBufferSnapshotTimeInMilliseconds = 0L; this._totalLatencyBufferSnapshot = new List <long>(); }
private void UpdateExecutionEventSnapshot() { long currentTimeInMiliseconds = CommonUtils.CurrentTimeInMiliseconds; if ((this._lastUpdateExecutionEventSnapshotTimeInMilliseconds == 0L) || ((this._lastUpdateExecutionEventSnapshotTimeInMilliseconds + this.ConfigSet.MetricsHealthSnapshotIntervalInMilliseconds) <= currentTimeInMiliseconds)) { Dictionary <CommandExecutionEventEnum, int> executionEventDistribution = new Dictionary <CommandExecutionEventEnum, int>(); foreach (CommandExecutionEventEnum enum2 in CommonUtils.CommandExecutionEvents) { executionEventDistribution[enum2] = this.ExecutionEventBuffer.GetCount(enum2); } this._executionEventHealthSnapshot = executionEventDistribution.GetHealthSnapshot(); this._executionEventDistributionSnapshot = executionEventDistribution; this._lastUpdateExecutionEventSnapshotTimeInMilliseconds = currentTimeInMiliseconds; } }
public bool IsOpen() { if (!ConfigSet.CircuitBreakerEnabled) { return(false); } if (ConfigSet.CircuitBreakerForceOpen) { return(true); } if (OpenFlag) { return(true); } CommandExecutionHealthSnapshot healthSnapshot = Metrics.GetExecutionHealthSnapshot(); if (healthSnapshot.TotalCount < ConfigSet.CircuitBreakerRequestCountThreshold) { return(false); } if (healthSnapshot.ErrorPercentage < ConfigSet.CircuitBreakerErrorThresholdPercentage) { return(false); } // our failure rate is too high, trip the circuit if (this.OpenFlag.CompareAndSet(false, true)) { // if the previousValue was false then we want to set the currentTime // How could previousValue be true? If another thread was going through this code at the same time a race-condition could have // caused another thread to set it to true already even though we were in the process of doing the same this.CircuitOpenedOrLastTestedTime.Value = CommonUtils.CurrentTimeInMiliseconds; CommonUtils.Log.Log( LogLevelEnum.Fatal, "Circuit Breaker is open after lots of fail or timeout happen.", new Dictionary <string, string>() { { "CircuitBreaker", "Open" } }.AddLogTagData("FXD303010")); } return(true); }
private void UpdateExecutionEventSnapshot() { long currentTimeInMilliseconds = CommonUtils.CurrentTimeInMiliseconds; if (_lastUpdateExecutionEventSnapshotTimeInMilliseconds == 0 || _lastUpdateExecutionEventSnapshotTimeInMilliseconds + ConfigSet.MetricsHealthSnapshotIntervalInMilliseconds <= currentTimeInMilliseconds) { Dictionary <CommandExecutionEventEnum, int> executionEventDistribution = new Dictionary <CommandExecutionEventEnum, int>(); foreach (CommandExecutionEventEnum @event in CommonUtils.CommandExecutionEvents) { executionEventDistribution[@event] = ExecutionEventBuffer.GetCount(@event); } _executionEventHealthSnapshot = executionEventDistribution.GetHealthSnapshot(); _executionEventDistributionSnapshot = executionEventDistribution; _lastUpdateExecutionEventSnapshotTimeInMilliseconds = currentTimeInMilliseconds; } }
private static void Report() { var tagMap = new Dictionary <string, string>(); tagMap["app"] = HystrixCommandBase.HystrixAppName.ToLower(); tagMap["version"] = HystrixCommandBase.HystrixVersion; string tag = string.Empty; var now = DateTime.Now; ; // log metrics Dictionary <string, CommandComponents> commandComponentsCollection = HystrixCommandBase.CommandComponentsCollection.ToDictionary(p => p.Key, p => p.Value); foreach (CommandComponents item in commandComponentsCollection.Values) { tagMap["instancekey"] = item.CommandInfo.InstanceKey; tagMap["commandkey"] = item.CommandInfo.CommandKey; tagMap["groupkey"] = item.CommandInfo.GroupKey; tagMap["domain"] = item.CommandInfo.Domain; tagMap["isolationmode"] = item.CommandInfo.Type; tag = "event"; Dictionary <CommandExecutionEventEnum, int> eventDistribution = item.Metrics.GetExecutionEventDistribution(); foreach (KeyValuePair <CommandExecutionEventEnum, int> item2 in eventDistribution) { if (item2.Value <= 0) { continue; } if (item2.Key == CommandExecutionEventEnum.ExceptionThrown) { continue; } tagMap[tag] = item2.Key.ToString(); // log metrics } if (tagMap.ContainsKey(tag)) { tagMap.Remove(tag); } CommandExecutionHealthSnapshot healthSnapshot = item.Metrics.GetExecutionHealthSnapshot(); if (healthSnapshot.ErrorPercentage > 0) { ; // log metrics } tag = "percentile"; foreach (KeyValuePair <double, string> item2 in LatencyPercentileValues) { tagMap[tag] = item2.Value; long latencyPencentile = item.Metrics.GetTotalExecutionLatencyPencentile(item2.Key); if (latencyPencentile > 0) { ; // log metrics } if (item.IsolationMode == IsolationModeEnum.ThreadIsolation) { latencyPencentile = item.Metrics.GetExecutionLatencyPencentile(item2.Key); if (latencyPencentile > 0) { ; // log metrics } } } if (tagMap.ContainsKey(tag)) { tagMap.Remove(tag); } int currentConcurrentExecutionCount = item.Metrics.CurrentConcurrentExecutionCount; if (currentConcurrentExecutionCount > 0) { ; // log metrics } int utilization = 0; int resourceLimit = item.ConfigSet.CommandMaxConcurrentCount; if (resourceLimit > 0) { utilization = (int)((double)currentConcurrentExecutionCount / resourceLimit * 100); } else { utilization = item.Metrics.CurrentConcurrentExecutionCount * 100; } if (utilization > 0) { ; // log metrics } } }
public static List <HystrixCommandInfo> GetHystrixCommandInfoList() { List <HystrixCommandInfo> list = new List <HystrixCommandInfo>(); foreach (CommandComponents components in HystrixCommandBase.CommandComponentsCollection.Values.ToArray <CommandComponents>()) { CommandExecutionHealthSnapshot executionHealthSnapshot = components.Metrics.GetExecutionHealthSnapshot(); Dictionary <CommandExecutionEventEnum, int> executionEventDistribution = components.Metrics.GetExecutionEventDistribution(); HystrixCommandInfo info2 = new HystrixCommandInfo { type = "HystrixCommand", name = components.CommandInfo.Key, group = (components.CommandInfo.InstanceKey == null) ? components.CommandInfo.GroupKey : components.CommandInfo.CommandKey, currentTime = CommonUtils.CurrentUnixTimeInMilliseconds, isCircuitBreakerOpen = components.CircuitBreaker.IsOpen(), errorPercentage = executionHealthSnapshot.ErrorPercentage, errorCount = ((IEnumerable <int>)(from p in executionEventDistribution where CommonUtils.CoreFailedCommandExecutionEvents.Contains <CommandExecutionEventEnum>(p.Key) select p.Value)).Sum(), requestCount = ((IEnumerable <int>)(from p in executionEventDistribution where CommonUtils.CoreCommandExecutionEvents.Contains <CommandExecutionEventEnum>(p.Key) select p.Value)).Sum(), rollingCountExceptionsThrown = (long)executionEventDistribution[CommandExecutionEventEnum.ExceptionThrown], rollingCountFailure = (long)executionEventDistribution[CommandExecutionEventEnum.Failed], rollingCountSemaphoreRejected = (components.IsolationMode == IsolationModeEnum.SemaphoreIsolation) ? ((long)executionEventDistribution[CommandExecutionEventEnum.Rejected]) : ((long)0), rollingCountShortCircuited = (long)executionEventDistribution[CommandExecutionEventEnum.ShortCircuited], rollingCountSuccess = (long)executionEventDistribution[CommandExecutionEventEnum.Success], rollingCountThreadPoolRejected = (components.IsolationMode == IsolationModeEnum.ThreadIsolation) ? ((long)executionEventDistribution[CommandExecutionEventEnum.Rejected]) : ((long)0), rollingCountTimeout = (long)executionEventDistribution[CommandExecutionEventEnum.Timeout], rollingCountFallbackFailure = (long)executionEventDistribution[CommandExecutionEventEnum.FallbackFailed], rollingCountFallbackSuccess = (long)executionEventDistribution[CommandExecutionEventEnum.FallbackSuccess], rollingCountFallbackRejection = (long)executionEventDistribution[CommandExecutionEventEnum.FallbackRejected] }; PercentileInfo info3 = new PercentileInfo { P0 = components.Metrics.GetExecutionLatencyPencentile(0.0), P25 = components.Metrics.GetExecutionLatencyPencentile(25.0), P50 = components.Metrics.GetExecutionLatencyPencentile(50.0), P75 = components.Metrics.GetExecutionLatencyPencentile(75.0), P90 = components.Metrics.GetExecutionLatencyPencentile(90.0), P95 = components.Metrics.GetExecutionLatencyPencentile(95.0), P99 = components.Metrics.GetExecutionLatencyPencentile(99.0), P99DOT5 = components.Metrics.GetExecutionLatencyPencentile(99.5), P100 = components.Metrics.GetExecutionLatencyPencentile(100.0) }; info2.latencyExecute = info3; info2.latencyExecute_mean = components.Metrics.GetAverageExecutionLatency(); PercentileInfo info4 = new PercentileInfo { P0 = components.Metrics.GetTotalExecutionLatencyPencentile(0.0), P25 = components.Metrics.GetTotalExecutionLatencyPencentile(25.0), P50 = components.Metrics.GetTotalExecutionLatencyPencentile(50.0), P75 = components.Metrics.GetTotalExecutionLatencyPencentile(75.0), P90 = components.Metrics.GetTotalExecutionLatencyPencentile(90.0), P95 = components.Metrics.GetTotalExecutionLatencyPencentile(95.0), P99 = components.Metrics.GetTotalExecutionLatencyPencentile(99.0), P99DOT5 = components.Metrics.GetTotalExecutionLatencyPencentile(99.5), P100 = components.Metrics.GetTotalExecutionLatencyPencentile(100.0) }; info2.latencyTotal = info4; info2.latencyTotal_mean = components.Metrics.GetAverageTotalExecutionLatency(); info2.reportingHosts = 1; info2.propertyValue_circuitBreakerEnabled = components.ConfigSet.CircuitBreakerEnabled; info2.propertyValue_circuitBreakerErrorThresholdPercentage = components.ConfigSet.CircuitBreakerErrorThresholdPercentage; info2.propertyValue_circuitBreakerForceClosed = components.ConfigSet.CircuitBreakerForceClosed; info2.propertyValue_circuitBreakerForceOpen = components.ConfigSet.CircuitBreakerForceOpen; info2.propertyValue_circuitBreakerRequestVolumeThreshold = components.ConfigSet.CircuitBreakerRequestCountThreshold; info2.propertyValue_circuitBreakerSleepWindowInMilliseconds = components.ConfigSet.CircuitBreakerSleepWindowInMilliseconds; info2.propertyValue_executionIsolationSemaphoreMaxConcurrentRequests = components.ConfigSet.CommandMaxConcurrentCount; info2.propertyValue_executionIsolationStrategy = (components.IsolationMode == IsolationModeEnum.SemaphoreIsolation) ? "SEMAPHORE" : "THREAD"; info2.propertyValue_executionIsolationThreadTimeoutInMilliseconds = components.ConfigSet.CommandTimeoutInMilliseconds; info2.propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests = components.ConfigSet.FallbackMaxConcurrentCount; info2.propertyValue_metricsRollingStatisticalWindowInMilliseconds = components.ConfigSet.MetricsRollingStatisticalWindowInMilliseconds; info2.currentConcurrentExecutionCount = components.Metrics.CurrentConcurrentExecutionCount; HystrixCommandInfo item = info2; list.Add(item); } return(list); }
public static List <HystrixCommandInfo> GetHystrixCommandInfoList() { List <HystrixCommandInfo> hystrixCommandInfoList = new List <HystrixCommandInfo>(); CommandComponents[] commandComponentsList = HystrixCommandBase.CommandComponentsCollection.Values.ToArray(); foreach (CommandComponents item in commandComponentsList) { CommandExecutionHealthSnapshot healthSnapshot = item.Metrics.GetExecutionHealthSnapshot(); Dictionary <CommandExecutionEventEnum, int> commandExecutionEventDistribution = item.Metrics.GetExecutionEventDistribution(); HystrixCommandInfo hystrixCommandInfo = new HystrixCommandInfo() { type = TurbineDataTypeHystrixCommand, name = item.CommandInfo.Key, group = item.CommandInfo.InstanceKey == null ? item.CommandInfo.GroupKey : item.CommandInfo.CommandKey, currentTime = CommonUtils.CurrentUnixTimeInMilliseconds, isCircuitBreakerOpen = item.CircuitBreaker.IsOpen(), errorPercentage = healthSnapshot.ErrorPercentage, errorCount = commandExecutionEventDistribution.Where(p => CommonUtils.CoreFailedCommandExecutionEvents.Contains(p.Key)).Select(p => p.Value).Sum(), requestCount = commandExecutionEventDistribution.Where(p => CommonUtils.CoreCommandExecutionEvents.Contains(p.Key)).Select(p => p.Value).Sum(), rollingCountExceptionsThrown = commandExecutionEventDistribution[CommandExecutionEventEnum.ExceptionThrown], rollingCountFailure = commandExecutionEventDistribution[CommandExecutionEventEnum.Failed], rollingCountSemaphoreRejected = item.IsolationMode == IsolationModeEnum.SemaphoreIsolation ? commandExecutionEventDistribution[CommandExecutionEventEnum.Rejected] : 0, rollingCountShortCircuited = commandExecutionEventDistribution[CommandExecutionEventEnum.ShortCircuited], rollingCountSuccess = commandExecutionEventDistribution[CommandExecutionEventEnum.Success], rollingCountThreadPoolRejected = item.IsolationMode == IsolationModeEnum.ThreadIsolation ? commandExecutionEventDistribution[CommandExecutionEventEnum.Rejected] : 0, rollingCountTimeout = commandExecutionEventDistribution[CommandExecutionEventEnum.Timeout], rollingCountFallbackFailure = commandExecutionEventDistribution[CommandExecutionEventEnum.FallbackFailed], rollingCountFallbackSuccess = commandExecutionEventDistribution[CommandExecutionEventEnum.FallbackSuccess], rollingCountFallbackRejection = commandExecutionEventDistribution[CommandExecutionEventEnum.FallbackRejected], latencyExecute = new PercentileInfo() { P0 = item.Metrics.GetExecutionLatencyPencentile(0), P25 = item.Metrics.GetExecutionLatencyPencentile(25), P50 = item.Metrics.GetExecutionLatencyPencentile(50), P75 = item.Metrics.GetExecutionLatencyPencentile(75), P90 = item.Metrics.GetExecutionLatencyPencentile(90), P95 = item.Metrics.GetExecutionLatencyPencentile(95), P99 = item.Metrics.GetExecutionLatencyPencentile(99), P99DOT5 = item.Metrics.GetExecutionLatencyPencentile(99.5), P100 = item.Metrics.GetExecutionLatencyPencentile(100) }, latencyExecute_mean = item.Metrics.GetAverageExecutionLatency(), latencyTotal = new PercentileInfo() { P0 = item.Metrics.GetTotalExecutionLatencyPencentile(0), P25 = item.Metrics.GetTotalExecutionLatencyPencentile(25), P50 = item.Metrics.GetTotalExecutionLatencyPencentile(50), P75 = item.Metrics.GetTotalExecutionLatencyPencentile(75), P90 = item.Metrics.GetTotalExecutionLatencyPencentile(90), P95 = item.Metrics.GetTotalExecutionLatencyPencentile(95), P99 = item.Metrics.GetTotalExecutionLatencyPencentile(99), P99DOT5 = item.Metrics.GetTotalExecutionLatencyPencentile(99.5), P100 = item.Metrics.GetTotalExecutionLatencyPencentile(100) }, latencyTotal_mean = item.Metrics.GetAverageTotalExecutionLatency(), reportingHosts = 1, propertyValue_circuitBreakerEnabled = item.ConfigSet.CircuitBreakerEnabled, propertyValue_circuitBreakerErrorThresholdPercentage = item.ConfigSet.CircuitBreakerErrorThresholdPercentage, propertyValue_circuitBreakerForceClosed = item.ConfigSet.CircuitBreakerForceClosed, propertyValue_circuitBreakerForceOpen = item.ConfigSet.CircuitBreakerForceOpen, propertyValue_circuitBreakerRequestVolumeThreshold = item.ConfigSet.CircuitBreakerRequestCountThreshold, propertyValue_circuitBreakerSleepWindowInMilliseconds = item.ConfigSet.CircuitBreakerSleepWindowInMilliseconds, propertyValue_executionIsolationSemaphoreMaxConcurrentRequests = item.ConfigSet.CommandMaxConcurrentCount, propertyValue_executionIsolationStrategy = item.IsolationMode == IsolationModeEnum.SemaphoreIsolation ? TurbineStrategySemaphore : TurbineStrategyThread, propertyValue_executionIsolationThreadTimeoutInMilliseconds = item.ConfigSet.CommandTimeoutInMilliseconds, propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests = item.ConfigSet.FallbackMaxConcurrentCount, propertyValue_metricsRollingStatisticalWindowInMilliseconds = item.ConfigSet.MetricsRollingStatisticalWindowInMilliseconds, currentConcurrentExecutionCount = item.Metrics.CurrentConcurrentExecutionCount, }; hystrixCommandInfoList.Add(hystrixCommandInfo); } return(hystrixCommandInfoList); }
private static void Report() { Dictionary <string, string> tags = new Dictionary <string, string>(); tags["app"] = HystrixCommandBase.HystrixAppName.ToLower(); tags["version"] = HystrixCommandBase.HystrixVersion; string key = string.Empty; DateTime now = DateTime.Now; //Metrics.log("chystrix.app.instance", (long) 1L, tags, now); foreach (CommandComponents components in HystrixCommandBase.CommandComponentsCollection.ToDictionary <KeyValuePair <string, CommandComponents>, string, CommandComponents>(p => p.Key, p => p.Value).Values) { tags["instancekey"] = components.CommandInfo.InstanceKey; tags["commandkey"] = components.CommandInfo.CommandKey; tags["groupkey"] = components.CommandInfo.GroupKey; tags["domain"] = components.CommandInfo.Domain; tags["isolationmode"] = components.CommandInfo.Type; key = "event"; foreach (KeyValuePair <CommandExecutionEventEnum, int> pair in components.Metrics.GetExecutionEventDistribution()) { if ((pair.Value > 0) && (((CommandExecutionEventEnum)pair.Key) != CommandExecutionEventEnum.ExceptionThrown)) { tags[key] = pair.Key.ToString(); //Metrics.log("chystrix.execution.event.distribution", (long) pair.Value, tags, now); } } if (tags.ContainsKey(key)) { tags.Remove(key); } CommandExecutionHealthSnapshot executionHealthSnapshot = components.Metrics.GetExecutionHealthSnapshot(); if (executionHealthSnapshot.ErrorPercentage > 0) { //Metrics.log("chystrix.error.percentage", (long) executionHealthSnapshot.ErrorPercentage, tags, now); } key = "percentile"; foreach (KeyValuePair <double, string> pair2 in LatencyPercentileValues) { tags[key] = pair2.Value; long totalExecutionLatencyPencentile = components.Metrics.GetTotalExecutionLatencyPencentile(pair2.Key); if (totalExecutionLatencyPencentile > 0L) { //Metrics.log("chystrix.execution.total_latency.percentile", totalExecutionLatencyPencentile, tags, now); } if (components.IsolationMode == IsolationModeEnum.ThreadIsolation) { totalExecutionLatencyPencentile = components.Metrics.GetExecutionLatencyPencentile(pair2.Key); if (totalExecutionLatencyPencentile > 0L) { //Metrics.log("chystrix.execution.latency.percentile", totalExecutionLatencyPencentile, tags, now); } } } if (tags.ContainsKey(key)) { tags.Remove(key); } int currentConcurrentExecutionCount = components.Metrics.CurrentConcurrentExecutionCount; if (currentConcurrentExecutionCount > 0) { //Metrics.log("chystrix.execution.concurrent_count", (long) currentConcurrentExecutionCount, tags, now); } int num3 = 0; int commandMaxConcurrentCount = components.ConfigSet.CommandMaxConcurrentCount; if (commandMaxConcurrentCount > 0) { num3 = (int)((((double)currentConcurrentExecutionCount) / ((double)commandMaxConcurrentCount)) * 100.0); } else { num3 = components.Metrics.CurrentConcurrentExecutionCount * 100; } if (num3 > 0) { //Metrics.log("chystrix.resource.utilization", (long) num3, tags, now); } } }