public bool IsOpen()
{
if (!this.ConfigSet.CircuitBreakerEnabled)
{
return(false);
}
if (!this.ConfigSet.CircuitBreakerForceOpen)
{
if (this.OpenFlag != null && this.OpenFlag == true)
{
return(true);
}
CommandExecutionHealthSnapshot executionHealthSnapshot = this.Metrics.GetExecutionHealthSnapshot();
if (executionHealthSnapshot.TotalCount < this.ConfigSet.CircuitBreakerRequestCountThreshold)
{
return(false);
}
if (executionHealthSnapshot.ErrorPercentage < this.ConfigSet.CircuitBreakerErrorThresholdPercentage)
{
return(false);
}
if (this.OpenFlag.CompareAndSet(false, true))
{
this.CircuitOpenedOrLastTestedTime.Value = CommonUtils.CurrentTimeInMiliseconds;
Dictionary <string, string> tagData = new Dictionary <string, string>();
tagData.Add("CircuitBreaker", "Open");
CommonUtils.Log.Log(LogLevelEnum.Fatal, "Circuit Breaker is open after lots of fail or timeout happen.", tagData.AddLogTagData("FXD303010"));
}
}
return(true);
}
public void Reset()
{
this.ExecutionEventBuffer = new CounterBuffer <CommandExecutionEventEnum>(this.ConfigSet.MetricsRollingStatisticalWindowInMilliseconds, this.ConfigSet.MetricsRollingStatisticalWindowBuckets);
this.ExecutionLatencyBuffer = new IntegerPercentileBuffer(this.ConfigSet.MetricsRollingPercentileWindowInMilliseconds, this.ConfigSet.MetricsRollingPercentileWindowBuckets, this.ConfigSet.MetricsRollingPercentileBucketSize);
this.TotalExecutionLatencyBuffer = new IntegerPercentileBuffer(this.ConfigSet.MetricsRollingPercentileWindowInMilliseconds, this.ConfigSet.MetricsRollingPercentileWindowBuckets, this.ConfigSet.MetricsRollingPercentileBucketSize);
this._lastUpdateExecutionEventSnapshotTimeInMilliseconds = 0L;
this._executionEventDistributionSnapshot = new Dictionary <CommandExecutionEventEnum, int>();
this._executionEventHealthSnapshot = new CommandExecutionHealthSnapshot(0, 0);
this._lastGetLatencyBufferSnapshotTimeInMilliseconds = 0L;
this._latencyBufferSnapshot = new List <long>();
this._lastGetTotalLatencyBufferSnapshotTimeInMilliseconds = 0L;
this._totalLatencyBufferSnapshot = new List <long>();
}
private void UpdateExecutionEventSnapshot()
{
long currentTimeInMiliseconds = CommonUtils.CurrentTimeInMiliseconds;
if ((this._lastUpdateExecutionEventSnapshotTimeInMilliseconds == 0L) || ((this._lastUpdateExecutionEventSnapshotTimeInMilliseconds + this.ConfigSet.MetricsHealthSnapshotIntervalInMilliseconds) <= currentTimeInMiliseconds))
{
Dictionary <CommandExecutionEventEnum, int> executionEventDistribution = new Dictionary <CommandExecutionEventEnum, int>();
foreach (CommandExecutionEventEnum enum2 in CommonUtils.CommandExecutionEvents)
{
executionEventDistribution[enum2] = this.ExecutionEventBuffer.GetCount(enum2);
}
this._executionEventHealthSnapshot = executionEventDistribution.GetHealthSnapshot();
this._executionEventDistributionSnapshot = executionEventDistribution;
this._lastUpdateExecutionEventSnapshotTimeInMilliseconds = currentTimeInMiliseconds;
}
}
public bool IsOpen()
{
if (!ConfigSet.CircuitBreakerEnabled)
{
return(false);
}
if (ConfigSet.CircuitBreakerForceOpen)
{
return(true);
}
if (OpenFlag)
{
return(true);
}
CommandExecutionHealthSnapshot healthSnapshot = Metrics.GetExecutionHealthSnapshot();
if (healthSnapshot.TotalCount < ConfigSet.CircuitBreakerRequestCountThreshold)
{
return(false);
}
if (healthSnapshot.ErrorPercentage < ConfigSet.CircuitBreakerErrorThresholdPercentage)
{
return(false);
}
// our failure rate is too high, trip the circuit
if (this.OpenFlag.CompareAndSet(false, true))
{
// if the previousValue was false then we want to set the currentTime
// How could previousValue be true? If another thread was going through this code at the same time a race-condition could have
// caused another thread to set it to true already even though we were in the process of doing the same
this.CircuitOpenedOrLastTestedTime.Value = CommonUtils.CurrentTimeInMiliseconds;
CommonUtils.Log.Log(
LogLevelEnum.Fatal,
"Circuit Breaker is open after lots of fail or timeout happen.",
new Dictionary <string, string>()
{
{ "CircuitBreaker", "Open" }
}.AddLogTagData("FXD303010"));
}
return(true);
}
private void UpdateExecutionEventSnapshot()
{
long currentTimeInMilliseconds = CommonUtils.CurrentTimeInMiliseconds;
if (_lastUpdateExecutionEventSnapshotTimeInMilliseconds == 0 ||
_lastUpdateExecutionEventSnapshotTimeInMilliseconds + ConfigSet.MetricsHealthSnapshotIntervalInMilliseconds <= currentTimeInMilliseconds)
{
Dictionary <CommandExecutionEventEnum, int> executionEventDistribution = new Dictionary <CommandExecutionEventEnum, int>();
foreach (CommandExecutionEventEnum @event in CommonUtils.CommandExecutionEvents)
{
executionEventDistribution[@event] = ExecutionEventBuffer.GetCount(@event);
}
_executionEventHealthSnapshot = executionEventDistribution.GetHealthSnapshot();
_executionEventDistributionSnapshot = executionEventDistribution;
_lastUpdateExecutionEventSnapshotTimeInMilliseconds = currentTimeInMilliseconds;
}
}
private static void Report()
{
var tagMap = new Dictionary <string, string>();
tagMap["app"] = HystrixCommandBase.HystrixAppName.ToLower();
tagMap["version"] = HystrixCommandBase.HystrixVersion;
string tag = string.Empty;
var now = DateTime.Now;
; // log metrics
Dictionary <string, CommandComponents> commandComponentsCollection = HystrixCommandBase.CommandComponentsCollection.ToDictionary(p => p.Key, p => p.Value);
foreach (CommandComponents item in commandComponentsCollection.Values)
{
tagMap["instancekey"] = item.CommandInfo.InstanceKey;
tagMap["commandkey"] = item.CommandInfo.CommandKey;
tagMap["groupkey"] = item.CommandInfo.GroupKey;
tagMap["domain"] = item.CommandInfo.Domain;
tagMap["isolationmode"] = item.CommandInfo.Type;
tag = "event";
Dictionary <CommandExecutionEventEnum, int> eventDistribution = item.Metrics.GetExecutionEventDistribution();
foreach (KeyValuePair <CommandExecutionEventEnum, int> item2 in eventDistribution)
{
if (item2.Value <= 0)
{
continue;
}
if (item2.Key == CommandExecutionEventEnum.ExceptionThrown)
{
continue;
}
tagMap[tag] = item2.Key.ToString();
// log metrics
}
if (tagMap.ContainsKey(tag))
{
tagMap.Remove(tag);
}
CommandExecutionHealthSnapshot healthSnapshot = item.Metrics.GetExecutionHealthSnapshot();
if (healthSnapshot.ErrorPercentage > 0)
{
; // log metrics
}
tag = "percentile";
foreach (KeyValuePair <double, string> item2 in LatencyPercentileValues)
{
tagMap[tag] = item2.Value;
long latencyPencentile = item.Metrics.GetTotalExecutionLatencyPencentile(item2.Key);
if (latencyPencentile > 0)
{
; // log metrics
}
if (item.IsolationMode == IsolationModeEnum.ThreadIsolation)
{
latencyPencentile = item.Metrics.GetExecutionLatencyPencentile(item2.Key);
if (latencyPencentile > 0)
{
; // log metrics
}
}
}
if (tagMap.ContainsKey(tag))
{
tagMap.Remove(tag);
}
int currentConcurrentExecutionCount = item.Metrics.CurrentConcurrentExecutionCount;
if (currentConcurrentExecutionCount > 0)
{
; // log metrics
}
int utilization = 0;
int resourceLimit = item.ConfigSet.CommandMaxConcurrentCount;
if (resourceLimit > 0)
{
utilization = (int)((double)currentConcurrentExecutionCount / resourceLimit * 100);
}
else
{
utilization = item.Metrics.CurrentConcurrentExecutionCount * 100;
}
if (utilization > 0)
{
; // log metrics
}
}
}
public static List <HystrixCommandInfo> GetHystrixCommandInfoList()
{
List <HystrixCommandInfo> list = new List <HystrixCommandInfo>();
foreach (CommandComponents components in HystrixCommandBase.CommandComponentsCollection.Values.ToArray <CommandComponents>())
{
CommandExecutionHealthSnapshot executionHealthSnapshot = components.Metrics.GetExecutionHealthSnapshot();
Dictionary <CommandExecutionEventEnum, int> executionEventDistribution = components.Metrics.GetExecutionEventDistribution();
HystrixCommandInfo info2 = new HystrixCommandInfo {
type = "HystrixCommand",
name = components.CommandInfo.Key,
group = (components.CommandInfo.InstanceKey == null) ? components.CommandInfo.GroupKey : components.CommandInfo.CommandKey,
currentTime = CommonUtils.CurrentUnixTimeInMilliseconds,
isCircuitBreakerOpen = components.CircuitBreaker.IsOpen(),
errorPercentage = executionHealthSnapshot.ErrorPercentage,
errorCount = ((IEnumerable <int>)(from p in executionEventDistribution
where CommonUtils.CoreFailedCommandExecutionEvents.Contains <CommandExecutionEventEnum>(p.Key)
select p.Value)).Sum(),
requestCount = ((IEnumerable <int>)(from p in executionEventDistribution
where CommonUtils.CoreCommandExecutionEvents.Contains <CommandExecutionEventEnum>(p.Key)
select p.Value)).Sum(),
rollingCountExceptionsThrown = (long)executionEventDistribution[CommandExecutionEventEnum.ExceptionThrown],
rollingCountFailure = (long)executionEventDistribution[CommandExecutionEventEnum.Failed],
rollingCountSemaphoreRejected = (components.IsolationMode == IsolationModeEnum.SemaphoreIsolation) ? ((long)executionEventDistribution[CommandExecutionEventEnum.Rejected]) : ((long)0),
rollingCountShortCircuited = (long)executionEventDistribution[CommandExecutionEventEnum.ShortCircuited],
rollingCountSuccess = (long)executionEventDistribution[CommandExecutionEventEnum.Success],
rollingCountThreadPoolRejected = (components.IsolationMode == IsolationModeEnum.ThreadIsolation) ? ((long)executionEventDistribution[CommandExecutionEventEnum.Rejected]) : ((long)0),
rollingCountTimeout = (long)executionEventDistribution[CommandExecutionEventEnum.Timeout],
rollingCountFallbackFailure = (long)executionEventDistribution[CommandExecutionEventEnum.FallbackFailed],
rollingCountFallbackSuccess = (long)executionEventDistribution[CommandExecutionEventEnum.FallbackSuccess],
rollingCountFallbackRejection = (long)executionEventDistribution[CommandExecutionEventEnum.FallbackRejected]
};
PercentileInfo info3 = new PercentileInfo {
P0 = components.Metrics.GetExecutionLatencyPencentile(0.0),
P25 = components.Metrics.GetExecutionLatencyPencentile(25.0),
P50 = components.Metrics.GetExecutionLatencyPencentile(50.0),
P75 = components.Metrics.GetExecutionLatencyPencentile(75.0),
P90 = components.Metrics.GetExecutionLatencyPencentile(90.0),
P95 = components.Metrics.GetExecutionLatencyPencentile(95.0),
P99 = components.Metrics.GetExecutionLatencyPencentile(99.0),
P99DOT5 = components.Metrics.GetExecutionLatencyPencentile(99.5),
P100 = components.Metrics.GetExecutionLatencyPencentile(100.0)
};
info2.latencyExecute = info3;
info2.latencyExecute_mean = components.Metrics.GetAverageExecutionLatency();
PercentileInfo info4 = new PercentileInfo {
P0 = components.Metrics.GetTotalExecutionLatencyPencentile(0.0),
P25 = components.Metrics.GetTotalExecutionLatencyPencentile(25.0),
P50 = components.Metrics.GetTotalExecutionLatencyPencentile(50.0),
P75 = components.Metrics.GetTotalExecutionLatencyPencentile(75.0),
P90 = components.Metrics.GetTotalExecutionLatencyPencentile(90.0),
P95 = components.Metrics.GetTotalExecutionLatencyPencentile(95.0),
P99 = components.Metrics.GetTotalExecutionLatencyPencentile(99.0),
P99DOT5 = components.Metrics.GetTotalExecutionLatencyPencentile(99.5),
P100 = components.Metrics.GetTotalExecutionLatencyPencentile(100.0)
};
info2.latencyTotal = info4;
info2.latencyTotal_mean = components.Metrics.GetAverageTotalExecutionLatency();
info2.reportingHosts = 1;
info2.propertyValue_circuitBreakerEnabled = components.ConfigSet.CircuitBreakerEnabled;
info2.propertyValue_circuitBreakerErrorThresholdPercentage = components.ConfigSet.CircuitBreakerErrorThresholdPercentage;
info2.propertyValue_circuitBreakerForceClosed = components.ConfigSet.CircuitBreakerForceClosed;
info2.propertyValue_circuitBreakerForceOpen = components.ConfigSet.CircuitBreakerForceOpen;
info2.propertyValue_circuitBreakerRequestVolumeThreshold = components.ConfigSet.CircuitBreakerRequestCountThreshold;
info2.propertyValue_circuitBreakerSleepWindowInMilliseconds = components.ConfigSet.CircuitBreakerSleepWindowInMilliseconds;
info2.propertyValue_executionIsolationSemaphoreMaxConcurrentRequests = components.ConfigSet.CommandMaxConcurrentCount;
info2.propertyValue_executionIsolationStrategy = (components.IsolationMode == IsolationModeEnum.SemaphoreIsolation) ? "SEMAPHORE" : "THREAD";
info2.propertyValue_executionIsolationThreadTimeoutInMilliseconds = components.ConfigSet.CommandTimeoutInMilliseconds;
info2.propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests = components.ConfigSet.FallbackMaxConcurrentCount;
info2.propertyValue_metricsRollingStatisticalWindowInMilliseconds = components.ConfigSet.MetricsRollingStatisticalWindowInMilliseconds;
info2.currentConcurrentExecutionCount = components.Metrics.CurrentConcurrentExecutionCount;
HystrixCommandInfo item = info2;
list.Add(item);
}
return(list);
}
public static List <HystrixCommandInfo> GetHystrixCommandInfoList()
{
List <HystrixCommandInfo> hystrixCommandInfoList = new List <HystrixCommandInfo>();
CommandComponents[] commandComponentsList = HystrixCommandBase.CommandComponentsCollection.Values.ToArray();
foreach (CommandComponents item in commandComponentsList)
{
CommandExecutionHealthSnapshot healthSnapshot = item.Metrics.GetExecutionHealthSnapshot();
Dictionary <CommandExecutionEventEnum, int> commandExecutionEventDistribution = item.Metrics.GetExecutionEventDistribution();
HystrixCommandInfo hystrixCommandInfo = new HystrixCommandInfo()
{
type = TurbineDataTypeHystrixCommand,
name = item.CommandInfo.Key,
group = item.CommandInfo.InstanceKey == null ? item.CommandInfo.GroupKey : item.CommandInfo.CommandKey,
currentTime = CommonUtils.CurrentUnixTimeInMilliseconds,
isCircuitBreakerOpen = item.CircuitBreaker.IsOpen(),
errorPercentage = healthSnapshot.ErrorPercentage,
errorCount = commandExecutionEventDistribution.Where(p => CommonUtils.CoreFailedCommandExecutionEvents.Contains(p.Key)).Select(p => p.Value).Sum(),
requestCount = commandExecutionEventDistribution.Where(p => CommonUtils.CoreCommandExecutionEvents.Contains(p.Key)).Select(p => p.Value).Sum(),
rollingCountExceptionsThrown = commandExecutionEventDistribution[CommandExecutionEventEnum.ExceptionThrown],
rollingCountFailure = commandExecutionEventDistribution[CommandExecutionEventEnum.Failed],
rollingCountSemaphoreRejected = item.IsolationMode == IsolationModeEnum.SemaphoreIsolation ? commandExecutionEventDistribution[CommandExecutionEventEnum.Rejected] : 0,
rollingCountShortCircuited = commandExecutionEventDistribution[CommandExecutionEventEnum.ShortCircuited],
rollingCountSuccess = commandExecutionEventDistribution[CommandExecutionEventEnum.Success],
rollingCountThreadPoolRejected = item.IsolationMode == IsolationModeEnum.ThreadIsolation ? commandExecutionEventDistribution[CommandExecutionEventEnum.Rejected] : 0,
rollingCountTimeout = commandExecutionEventDistribution[CommandExecutionEventEnum.Timeout],
rollingCountFallbackFailure = commandExecutionEventDistribution[CommandExecutionEventEnum.FallbackFailed],
rollingCountFallbackSuccess = commandExecutionEventDistribution[CommandExecutionEventEnum.FallbackSuccess],
rollingCountFallbackRejection = commandExecutionEventDistribution[CommandExecutionEventEnum.FallbackRejected],
latencyExecute = new PercentileInfo()
{
P0 = item.Metrics.GetExecutionLatencyPencentile(0),
P25 = item.Metrics.GetExecutionLatencyPencentile(25),
P50 = item.Metrics.GetExecutionLatencyPencentile(50),
P75 = item.Metrics.GetExecutionLatencyPencentile(75),
P90 = item.Metrics.GetExecutionLatencyPencentile(90),
P95 = item.Metrics.GetExecutionLatencyPencentile(95),
P99 = item.Metrics.GetExecutionLatencyPencentile(99),
P99DOT5 = item.Metrics.GetExecutionLatencyPencentile(99.5),
P100 = item.Metrics.GetExecutionLatencyPencentile(100)
},
latencyExecute_mean = item.Metrics.GetAverageExecutionLatency(),
latencyTotal = new PercentileInfo()
{
P0 = item.Metrics.GetTotalExecutionLatencyPencentile(0),
P25 = item.Metrics.GetTotalExecutionLatencyPencentile(25),
P50 = item.Metrics.GetTotalExecutionLatencyPencentile(50),
P75 = item.Metrics.GetTotalExecutionLatencyPencentile(75),
P90 = item.Metrics.GetTotalExecutionLatencyPencentile(90),
P95 = item.Metrics.GetTotalExecutionLatencyPencentile(95),
P99 = item.Metrics.GetTotalExecutionLatencyPencentile(99),
P99DOT5 = item.Metrics.GetTotalExecutionLatencyPencentile(99.5),
P100 = item.Metrics.GetTotalExecutionLatencyPencentile(100)
},
latencyTotal_mean = item.Metrics.GetAverageTotalExecutionLatency(),
reportingHosts = 1,
propertyValue_circuitBreakerEnabled = item.ConfigSet.CircuitBreakerEnabled,
propertyValue_circuitBreakerErrorThresholdPercentage = item.ConfigSet.CircuitBreakerErrorThresholdPercentage,
propertyValue_circuitBreakerForceClosed = item.ConfigSet.CircuitBreakerForceClosed,
propertyValue_circuitBreakerForceOpen = item.ConfigSet.CircuitBreakerForceOpen,
propertyValue_circuitBreakerRequestVolumeThreshold = item.ConfigSet.CircuitBreakerRequestCountThreshold,
propertyValue_circuitBreakerSleepWindowInMilliseconds = item.ConfigSet.CircuitBreakerSleepWindowInMilliseconds,
propertyValue_executionIsolationSemaphoreMaxConcurrentRequests = item.ConfigSet.CommandMaxConcurrentCount,
propertyValue_executionIsolationStrategy = item.IsolationMode == IsolationModeEnum.SemaphoreIsolation ? TurbineStrategySemaphore : TurbineStrategyThread,
propertyValue_executionIsolationThreadTimeoutInMilliseconds = item.ConfigSet.CommandTimeoutInMilliseconds,
propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests = item.ConfigSet.FallbackMaxConcurrentCount,
propertyValue_metricsRollingStatisticalWindowInMilliseconds = item.ConfigSet.MetricsRollingStatisticalWindowInMilliseconds,
currentConcurrentExecutionCount = item.Metrics.CurrentConcurrentExecutionCount,
};
hystrixCommandInfoList.Add(hystrixCommandInfo);
}
return(hystrixCommandInfoList);
}
private static void Report()
{
Dictionary <string, string> tags = new Dictionary <string, string>();
tags["app"] = HystrixCommandBase.HystrixAppName.ToLower();
tags["version"] = HystrixCommandBase.HystrixVersion;
string key = string.Empty;
DateTime now = DateTime.Now;
//Metrics.log("chystrix.app.instance", (long) 1L, tags, now);
foreach (CommandComponents components in HystrixCommandBase.CommandComponentsCollection.ToDictionary <KeyValuePair <string, CommandComponents>, string, CommandComponents>(p => p.Key, p => p.Value).Values)
{
tags["instancekey"] = components.CommandInfo.InstanceKey;
tags["commandkey"] = components.CommandInfo.CommandKey;
tags["groupkey"] = components.CommandInfo.GroupKey;
tags["domain"] = components.CommandInfo.Domain;
tags["isolationmode"] = components.CommandInfo.Type;
key = "event";
foreach (KeyValuePair <CommandExecutionEventEnum, int> pair in components.Metrics.GetExecutionEventDistribution())
{
if ((pair.Value > 0) && (((CommandExecutionEventEnum)pair.Key) != CommandExecutionEventEnum.ExceptionThrown))
{
tags[key] = pair.Key.ToString();
//Metrics.log("chystrix.execution.event.distribution", (long) pair.Value, tags, now);
}
}
if (tags.ContainsKey(key))
{
tags.Remove(key);
}
CommandExecutionHealthSnapshot executionHealthSnapshot = components.Metrics.GetExecutionHealthSnapshot();
if (executionHealthSnapshot.ErrorPercentage > 0)
{
//Metrics.log("chystrix.error.percentage", (long) executionHealthSnapshot.ErrorPercentage, tags, now);
}
key = "percentile";
foreach (KeyValuePair <double, string> pair2 in LatencyPercentileValues)
{
tags[key] = pair2.Value;
long totalExecutionLatencyPencentile = components.Metrics.GetTotalExecutionLatencyPencentile(pair2.Key);
if (totalExecutionLatencyPencentile > 0L)
{
//Metrics.log("chystrix.execution.total_latency.percentile", totalExecutionLatencyPencentile, tags, now);
}
if (components.IsolationMode == IsolationModeEnum.ThreadIsolation)
{
totalExecutionLatencyPencentile = components.Metrics.GetExecutionLatencyPencentile(pair2.Key);
if (totalExecutionLatencyPencentile > 0L)
{
//Metrics.log("chystrix.execution.latency.percentile", totalExecutionLatencyPencentile, tags, now);
}
}
}
if (tags.ContainsKey(key))
{
tags.Remove(key);
}
int currentConcurrentExecutionCount = components.Metrics.CurrentConcurrentExecutionCount;
if (currentConcurrentExecutionCount > 0)
{
//Metrics.log("chystrix.execution.concurrent_count", (long) currentConcurrentExecutionCount, tags, now);
}
int num3 = 0;
int commandMaxConcurrentCount = components.ConfigSet.CommandMaxConcurrentCount;
if (commandMaxConcurrentCount > 0)
{
num3 = (int)((((double)currentConcurrentExecutionCount) / ((double)commandMaxConcurrentCount)) * 100.0);
}
else
{
num3 = components.Metrics.CurrentConcurrentExecutionCount * 100;
}
if (num3 > 0)
{
//Metrics.log("chystrix.resource.utilization", (long) num3, tags, now);
}
}
}
