public void CircuitBreaker_TripCircuit()
{
try
{
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter();
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
metrics.MarkSuccess(1000);
// this should still allow requests as everything has been successful
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void CircuitBreaker_TripCircuitOnFailuresAboveThreshold()
{
try
{
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter();
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// this should start as allowing requests
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
// success with high latency
metrics.MarkSuccess(400);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkFailure(10);
metrics.MarkSuccess(400);
metrics.MarkFailure(10);
metrics.MarkFailure(10);
// this should trip the circuit as the error percentage is above the threshold
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void CircuitBreaker_SingleTestOnOpenCircuitAfterTimeWindow()
{
try
{
int sleepWindow = 200;
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void CircuitBreaker_TripCircuitOnTimeouts()
{
try
{
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter();
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// this should start as allowing requests
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
// timeouts
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
metrics.MarkTimeout(2000);
// everything has been a timeout so we should not allow any requests
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void CircuitBreaker_LowVolumeDoesNotTripCircuit()
{
try
{
int sleepWindow = 200;
int lowVolume = 5;
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithCircuitBreakerRequestVolumeThreshold(lowVolume);
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// even though it has all failed we won't trip the circuit because the volume is low
Assert.IsTrue(cb.AllowRequest());
Assert.IsFalse(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
public void CircuitBreaker_CircuitClosedAfterSuccessAndClearsStatisticalWindow()
{
try
{
int statisticalWindow = 200;
int sleepWindow = 10; // this is set very low so that returning from a retry still ends up having data in the buckets for the statisticalWindow
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow).WithMetricsRollingStatisticalWindowInMilliseconds(statisticalWindow);
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// the 'singleTest' succeeds so should cause the circuit to be closed
metrics.MarkSuccess(500);
cb.MarkSuccess();
// all requests should be open again
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
// and the circuit should be closed again
Assert.IsFalse(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
private static List <GlobalStreamHystrixCommandInfo> GetHystrixCommandInfoList()
{
var result = new List <GlobalStreamHystrixCommandInfo>();
foreach (ServiceMetadata metadata in EndpointHost.Config.MetadataMap.Values)
{
string refinedServiceName = ServiceUtils.RefineServiceName(metadata.ServiceNamespace, metadata.ServiceName);
foreach (Operation operation in metadata.Operations)
{
HystrixCommandMetrics commandMetrics = operation.HystrixCommand.Metrics;
IHystrixCircuitBreaker circuitBreaker = operation.HystrixCommand.CircuitBreaker;
HealthCounts healthCounts = commandMetrics.GetHealthCounts();
IHystrixCommandProperties commandProperties = commandMetrics.Properties;
GlobalStreamHystrixCommandInfo hystrixCommandInfo = new GlobalStreamHystrixCommandInfo()
{
type = TurbineDataTypeHystrixCommand,
name = refinedServiceName + "." + operation.Name.ToLower(),
group = refinedServiceName,
currentTime = DateTime.Now.ToUnixTimeMs(),
isCircuitBreakerOpen = circuitBreaker.IsOpen(),
errorPercentage = healthCounts.ErrorPercentage,
errorCount = healthCounts.TotalErrorCount,
requestCount = healthCounts.TotalRequests,
rollingCountExceptionsThrown = healthCounts.TotalExceptionCount,
rollingCountFailure = healthCounts.TotalFailureCount,
rollingCountSemaphoreRejected = 0,
rollingCountShortCircuited = healthCounts.ShortCircuitedCount,
rollingCountSuccess = healthCounts.SuccessCount,
rollingCountThreadPoolRejected = 0,
rollingCountTimeout = healthCounts.TimeoutCount,
rollingCountFallbackFailure = 0,
rollingCountFallbackSuccess = 0,
rollingCountFallbackRejection = 0,
latencyExecute = new GlobalStreamPercentileInfo()
{
P0 = commandMetrics.GetServiceExecutionTimePercentile(0),
P25 = commandMetrics.GetServiceExecutionTimePercentile(25),
P50 = commandMetrics.GetServiceExecutionTimePercentile(50),
P75 = commandMetrics.GetServiceExecutionTimePercentile(75),
P90 = commandMetrics.GetServiceExecutionTimePercentile(90),
P95 = commandMetrics.GetServiceExecutionTimePercentile(95),
P99 = commandMetrics.GetServiceExecutionTimePercentile(99),
P99DOT5 = commandMetrics.GetServiceExecutionTimePercentile(99.5),
P100 = commandMetrics.GetServiceExecutionTimePercentile(100)
},
latencyExecute_mean = commandMetrics.GetServiceExecutionTimeMean(),
latencyTotal = new GlobalStreamPercentileInfo()
{
P0 = commandMetrics.GetTotalTimePercentile(0),
P25 = commandMetrics.GetTotalTimePercentile(25),
P50 = commandMetrics.GetTotalTimePercentile(50),
P75 = commandMetrics.GetTotalTimePercentile(75),
P90 = commandMetrics.GetTotalTimePercentile(90),
P95 = commandMetrics.GetTotalTimePercentile(95),
P99 = commandMetrics.GetTotalTimePercentile(99),
P99DOT5 = commandMetrics.GetTotalTimePercentile(99.5),
P100 = commandMetrics.GetTotalTimePercentile(100)
},
latencyTotal_mean = commandMetrics.GetTotalTimeMean(),
reportingHosts = 1,
propertyValue_circuitBreakerEnabled = commandProperties.CircuitBreakerEnabled.Get(),
propertyValue_circuitBreakerErrorThresholdPercentage = commandProperties.CircuitBreakerErrorThresholdPercentage.Get(),
propertyValue_circuitBreakerForceClosed = commandProperties.CircuitBreakerForceClosed.Get(),
propertyValue_circuitBreakerForceOpen = commandProperties.CircuitBreakerForceOpen.Get(),
propertyValue_circuitBreakerRequestVolumeThreshold = commandProperties.CircuitBreakerRequestVolumeThreshold.Get(),
propertyValue_circuitBreakerSleepWindowInMilliseconds = (int)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds,
propertyValue_executionIsolationSemaphoreMaxConcurrentRequests = 1000,
propertyValue_executionIsolationStrategy = TurbineStrategySemaphore,
propertyValue_executionIsolationThreadTimeoutInMilliseconds = (int)operation.HystrixCommand.GetExecutionTimeout().TotalMilliseconds,
propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests = 0,
propertyValue_metricsRollingStatisticalWindowInMilliseconds = commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get(),
currentConcurrentExecutionCount = commandMetrics.CurrentConcurrentExecutionCount
};
result.Add(hystrixCommandInfo);
}
}
return(result);
}
public static List <HystrixCommandInfo> GetHystrixCommandInfo(string servicePath)
{
var result = new List <HystrixCommandInfo>();
ServiceMetadata metadata = EndpointHost.Config.MetadataMap[servicePath];
foreach (Operation operation in metadata.Operations)
{
HystrixCommandMetrics commandMetrics = operation.HystrixCommand.Metrics;
IHystrixCircuitBreaker circuitBreaker = operation.HystrixCommand.CircuitBreaker;
HealthCounts healthCounts = commandMetrics.GetHealthCounts();
IHystrixCommandProperties commandProperties = commandMetrics.Properties;
var commandInfo = new HystrixCommandInfo
{
Type = "HystrixCommand",
Name = commandMetrics.OperationName,
Group = commandMetrics.FullServiceName,
CurrentTime = DateTime.Now,
IsCircuitBreakerOpen = (circuitBreaker == null ? false : circuitBreaker.IsOpen()),
ErrorPercentage = healthCounts.ErrorPercentage,
ErrorCount = healthCounts.TotalErrorCount,
RequestCount = healthCounts.TotalRequests,
RollingCountSuccess = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Success),
RollingCountShortCircuited = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited),
RollingCountTimeout = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Timeout),
RollingCountThreadPoolRejected = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected),
RollingCountFrameworkExceptionThrown = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FrameworkExceptionThrown),
RollingCountServiceExceptionThrown = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ServiceExceptionThrown),
RollingCountValidationExceptionThrown = commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ValidationExceptionThrown),
CumulativeCountSuccess = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.Success),
CumulativeCountShortCircuited = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ShortCircuited),
CumulativeCountTimeout = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.Timeout),
CumulativeCountThreadPoolRejected = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ThreadPoolRejected),
CumulativeCountFrameworkExcetpionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.FrameworkExceptionThrown),
CumulativeCountServiceExceptionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ServiceExceptionThrown),
CumulativeCountValidationExceptionThrown = commandMetrics.GetCumulativeCount(HystrixRollingNumberEvent.ValidationExceptionThrown),
CurrentConcurrentExecutionCount = commandMetrics.CurrentConcurrentExecutionCount,
LatencyExecuteMean = commandMetrics.GetServiceExecutionTimeMean(),
LatencyExecute = new PercentileInfo
{
P0 = commandMetrics.GetServiceExecutionTimePercentile(0),
P25 = commandMetrics.GetServiceExecutionTimePercentile(25),
P50 = commandMetrics.GetServiceExecutionTimePercentile(50),
P75 = commandMetrics.GetServiceExecutionTimePercentile(75),
P90 = commandMetrics.GetServiceExecutionTimePercentile(90),
P95 = commandMetrics.GetServiceExecutionTimePercentile(95),
P99 = commandMetrics.GetServiceExecutionTimePercentile(99),
P99DOT5 = commandMetrics.GetServiceExecutionTimePercentile(99.5),
P100 = commandMetrics.GetServiceExecutionTimePercentile(100),
},
LatencyTotalMean = commandMetrics.GetTotalTimeMean(),
LatencyTotal = new PercentileInfo
{
P0 = commandMetrics.GetTotalTimePercentile(0),
P25 = commandMetrics.GetTotalTimePercentile(25),
P50 = commandMetrics.GetTotalTimePercentile(50),
P75 = commandMetrics.GetTotalTimePercentile(75),
P90 = commandMetrics.GetTotalTimePercentile(90),
P95 = commandMetrics.GetTotalTimePercentile(95),
P99 = commandMetrics.GetTotalTimePercentile(99),
P99DOT5 = commandMetrics.GetTotalTimePercentile(99.5),
P100 = commandMetrics.GetTotalTimePercentile(100),
},
PropertyValue_CircuitBreakerRequestVolumeThreshold = commandProperties.CircuitBreakerRequestVolumeThreshold.Get(),
PropertyValue_CircuitBreakerSleepWindowInMilliseconds = (long)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds,
PropertyValue_CircuitBreakerErrorThresholdPercentage = commandProperties.CircuitBreakerErrorThresholdPercentage.Get(),
PropertyValue_CircuitBreakerForceOpen = commandProperties.CircuitBreakerForceOpen.Get(),
PropertyValue_CircuitBreakerForceClosed = commandProperties.CircuitBreakerForceClosed.Get(),
PropertyValue_CircuitBreakerEnabled = commandProperties.CircuitBreakerEnabled.Get(),
PropertyValue_ExecutionIsolationThreadTimeoutInMilliseconds = (long)operation.HystrixCommand.GetExecutionTimeout().TotalMilliseconds,
PropertyValue_MetricsRollingStatisticalWindowInMilliseconds = commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get(),
PropertyValue_RequestLogEnabled = commandProperties.RequestLogEnabled.Get(),
ReportingHosts = 1,
};
result.Add(commandInfo);
}
return(result);
}
/// <summary>
/// Produces JSON formatted metrics data from an instance of <see cref="HystrixCommandMetrics"/>.
/// </summary>
/// <param name="commandMetrics">The metrics of a command.</param>
/// <returns>JSON formatted metrics data.</returns>
private static string CreateCommandSampleData(HystrixCommandMetrics commandMetrics)
{
IHystrixCircuitBreaker circuitBreaker = HystrixCircuitBreakerFactory.GetInstance(commandMetrics.CommandKey);
HealthCounts healthCounts = commandMetrics.GetHealthCounts();
IHystrixCommandProperties commandProperties = commandMetrics.Properties;
JObject data = new JObject(
new JProperty("type", "HystrixCommand"),
new JProperty("name", commandMetrics.CommandKey.Name),
new JProperty("group", commandMetrics.CommandGroup.Name),
new JProperty("currentTime", GetCurrentTimeForJavascript()),
circuitBreaker == null ? new JProperty("isCircuitBreakerOpen", false) : new JProperty("isCircuitBreakerOpen", circuitBreaker.IsOpen()),
new JProperty("errorPercentage", healthCounts.ErrorPercentage), // health counts
new JProperty("errorCount", healthCounts.ErrorCount),
new JProperty("requestCount", healthCounts.TotalRequests),
new JProperty("rollingCountCollapsedRequests", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Collapsed)), // rolling counters
new JProperty("rollingCountExceptionsThrown", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ExceptionThrown)),
new JProperty("rollingCountFailure", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Failure)),
new JProperty("rollingCountFallbackFailure", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackFailure)),
new JProperty("rollingCountFallbackRejection", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackRejection)),
new JProperty("rollingCountFallbackSuccess", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.FallbackSuccess)),
new JProperty("rollingCountResponsesFromCache", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ResponseFromCache)),
new JProperty("rollingCountSemaphoreRejected", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.SemaphoreRejected)),
new JProperty("rollingCountShortCircuited", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ShortCircuited)),
new JProperty("rollingCountSuccess", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Success)),
new JProperty("rollingCountThreadPoolRejected", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.ThreadPoolRejected)),
new JProperty("rollingCountTimeout", commandMetrics.GetRollingCount(HystrixRollingNumberEvent.Timeout)),
new JProperty("currentConcurrentExecutionCount", commandMetrics.CurrentConcurrentExecutionCount),
new JProperty("latencyExecute_mean", commandMetrics.GetExecutionTimeMean()), // latency percentiles
new JProperty(
"latencyExecute",
new JObject(
new JProperty("0", commandMetrics.GetExecutionTimePercentile(0)),
new JProperty("25", commandMetrics.GetExecutionTimePercentile(25)),
new JProperty("50", commandMetrics.GetExecutionTimePercentile(50)),
new JProperty("75", commandMetrics.GetExecutionTimePercentile(75)),
new JProperty("90", commandMetrics.GetExecutionTimePercentile(90)),
new JProperty("95", commandMetrics.GetExecutionTimePercentile(95)),
new JProperty("99", commandMetrics.GetExecutionTimePercentile(99)),
new JProperty("99.5", commandMetrics.GetExecutionTimePercentile(99.5)),
new JProperty("100", commandMetrics.GetExecutionTimePercentile(100)))),
new JProperty("latencyTotal_mean", commandMetrics.GetTotalTimeMean()),
new JProperty(
"latencyTotal",
new JObject(
new JProperty("0", commandMetrics.GetTotalTimePercentile(0)),
new JProperty("25", commandMetrics.GetTotalTimePercentile(25)),
new JProperty("50", commandMetrics.GetTotalTimePercentile(50)),
new JProperty("75", commandMetrics.GetTotalTimePercentile(75)),
new JProperty("90", commandMetrics.GetTotalTimePercentile(90)),
new JProperty("95", commandMetrics.GetTotalTimePercentile(95)),
new JProperty("99", commandMetrics.GetTotalTimePercentile(99)),
new JProperty("99.5", commandMetrics.GetTotalTimePercentile(99.5)),
new JProperty("100", commandMetrics.GetTotalTimePercentile(100)))),
new JProperty("propertyValue_circuitBreakerRequestVolumeThreshold", commandProperties.CircuitBreakerRequestVolumeThreshold.Get()), // property values for reporting what is actually seen by the command rather than what was set somewhere
new JProperty("propertyValue_circuitBreakerSleepWindowInMilliseconds", (long)commandProperties.CircuitBreakerSleepWindow.Get().TotalMilliseconds),
new JProperty("propertyValue_circuitBreakerErrorThresholdPercentage", commandProperties.CircuitBreakerErrorThresholdPercentage.Get()),
new JProperty("propertyValue_circuitBreakerForceOpen", commandProperties.CircuitBreakerForceOpen.Get()),
new JProperty("propertyValue_circuitBreakerForceClosed", commandProperties.CircuitBreakerForceClosed.Get()),
new JProperty("propertyValue_circuitBreakerEnabled", commandProperties.CircuitBreakerEnabled.Get()),
new JProperty("propertyValue_executionIsolationStrategy", commandProperties.ExecutionIsolationStrategy.Get()),
new JProperty("propertyValue_executionIsolationThreadTimeoutInMilliseconds", (long)commandProperties.ExecutionIsolationThreadTimeout.Get().TotalMilliseconds),
new JProperty("propertyValue_executionIsolationThreadInterruptOnTimeout", commandProperties.ExecutionIsolationThreadInterruptOnTimeout.Get()),
new JProperty("propertyValue_executionIsolationThreadPoolKeyOverride", commandProperties.ExecutionIsolationThreadPoolKeyOverride.Get()),
new JProperty("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests", commandProperties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Get()),
new JProperty("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests", commandProperties.FallbackIsolationSemaphoreMaxConcurrentRequests.Get()),
new JProperty("propertyValue_metricsRollingStatisticalWindowInMilliseconds", commandProperties.MetricsRollingStatisticalWindowInMilliseconds.Get()),
new JProperty("propertyValue_requestCacheEnabled", commandProperties.RequestCacheEnabled.Get()),
new JProperty("propertyValue_requestLogEnabled", commandProperties.RequestLogEnabled.Get()),
new JProperty("reportingHosts", 1));
return(data.ToString(Formatting.None));
}
/**
* Servo will flatten metric names as: getServoTypeTag()_getServoInstanceTag()_monitorName
*/
private List <IMonitor> getServoMonitors()
{
List <IMonitor> monitors = new List <IMonitor>();
monitors.Add(ServoMetricFactory.InformationalMetric <bool>(this,
MonitorConfig.builder("isCircuitBreakerOpen").build(),
() => circuitBreaker.IsOpen()));
// allow Servo and monitor to know exactly at what point in time these stats are for so they can be plotted accurately
monitors.Add(getCurrentValueMonitor("currentTime", () => DateTime.UtcNow.ToUnixTimestamp(), DataSourceLevel.DEBUG));
// cumulative counts
//monitors.Add(getCumulativeMonitor("countBadRequests", HystrixEventType.BAD_REQUEST));
// monitors.add(getCumulativeMonitor("countCollapsedRequests", HystrixEventType.COLLAPSED));
// monitors.add(getCumulativeMonitor("countEmit", HystrixEventType.EMIT));
monitors.Add(getCumulativeMonitor("countExceptionsThrown", HystrixEventType.ExceptionThrown));
monitors.Add(getCumulativeMonitor("countFailure", HystrixEventType.Failure));
// monitors.add(getCumulativeMonitor("countFallbackEmit", HystrixEventType.FALLBACK_EMIT));
monitors.Add(getCumulativeMonitor("countFallbackFailure", HystrixEventType.FallbackFailure));
// monitors.add(getCumulativeMonitor("countFallbackMissing", HystrixEventType.FALLBACK_MISSING));
monitors.Add(getCumulativeMonitor("countFallbackRejection", HystrixEventType.FallbackRejection));
monitors.Add(getCumulativeMonitor("countFallbackSuccess", HystrixEventType.FallbackSuccess));
monitors.Add(getCumulativeMonitor("countResponsesFromCache", HystrixEventType.ResponseFromCache));
monitors.Add(getCumulativeMonitor("countSemaphoreRejected", HystrixEventType.SemaphoreRejected));
monitors.Add(getCumulativeMonitor("countShortCircuited", HystrixEventType.ShortCircuited));
monitors.Add(getCumulativeMonitor("countSuccess", HystrixEventType.Success));
monitors.Add(getCumulativeMonitor("countThreadPoolRejected", HystrixEventType.ThreadPoolRejected));
monitors.Add(getCumulativeMonitor("countTimeout", HystrixEventType.Timeout));
// rolling counts
// monitors.add(getRollingMonitor("rollingCountBadRequests", HystrixEventType.BAD_REQUEST));
// monitors.add(getRollingMonitor("rollingCountCollapsedRequests", HystrixEventType.COLLAPSED));
// monitors.add(getRollingMonitor("rollingCountEmit", HystrixEventType.EMIT));
monitors.Add(getRollingMonitor("rollingCountExceptionsThrown", HystrixEventType.ExceptionThrown));
monitors.Add(getRollingMonitor("rollingCountFailure", HystrixEventType.Failure));
//monitors.Add(getRollingMonitor("rollingCountFallbackEmit", HystrixEventType.FALLBACK_EMIT));
monitors.Add(getRollingMonitor("rollingCountFallbackFailure", HystrixEventType.FallbackFailure));
//monitors.Add(getRollingMonitor("rollingCountFallbackMissing", HystrixEventType.FALLBACK_MISSING));
monitors.Add(getRollingMonitor("rollingCountFallbackRejection", HystrixEventType.FallbackRejection));
monitors.Add(getRollingMonitor("rollingCountFallbackSuccess", HystrixEventType.FallbackSuccess));
monitors.Add(getRollingMonitor("rollingCountResponsesFromCache", HystrixEventType.ResponseFromCache));
monitors.Add(getRollingMonitor("rollingCountSemaphoreRejected", HystrixEventType.SemaphoreRejected));
monitors.Add(getRollingMonitor("rollingCountShortCircuited", HystrixEventType.ShortCircuited));
monitors.Add(getRollingMonitor("rollingCountSuccess", HystrixEventType.Success));
monitors.Add(getRollingMonitor("rollingCountThreadPoolRejected", HystrixEventType.ThreadPoolRejected));
monitors.Add(getRollingMonitor("rollingCountTimeout", HystrixEventType.Timeout));
// the number of executionSemaphorePermits in use right now
monitors.Add(getCurrentValueMonitor("executionSemaphorePermitsInUse", () => metrics.CurrentConcurrentExecutionCount));
// error percentage derived from current metrics
monitors.Add(getCurrentValueMonitor("errorPercentage", () => metrics.GetHealthCounts().ErrorPercentage));
// execution latency metrics
monitors.Add(getExecutionLatencyMeanMonitor("latencyExecute_mean"));
monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_5", 5));
monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_25", 25));
monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_50", 50));
monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_75", 75));
monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_90", 90));
monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_99", 99));
monitors.Add(getExecutionLatencyPercentileMonitor("latencyExecute_percentile_995", 99.5));
// // total latency metrics
monitors.Add(getTotalLatencyMeanMonitor("latencyTotal_mean"));
monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_5", 5));
monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_25", 25));
monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_50", 50));
monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_75", 75));
monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_90", 90));
monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_99", 99));
monitors.Add(getTotalLatencyPercentileMonitor("latencyTotal_percentile_995", 995));
// group
//monitors.Add(ServoMetricFactory.InformationalMetric<string>(this,
// MonitorConfig.builder("commandGroup").build(),
// () => commandGroupKey != null ? commandGroupKey.Name : null));
// properties (so the values can be inspected and monitored)
monitors.Add(ServoMetricFactory.InformationalMetric <int>(this,
MonitorConfig.builder("propertyValue_rollingStatisticalWindowInMilliseconds").build(),
() => properties.MetricsRollingStatisticalWindowInMilliseconds.Get()));
monitors.Add(ServoMetricFactory.InformationalMetric <int>(this,
MonitorConfig.builder("propertyValue_circuitBreakerRequestVolumeThreshold").build(),
() => properties.CircuitBreakerRequestVolumeThreshold.Get()));
monitors.Add(ServoMetricFactory.InformationalMetric <long>(this,
MonitorConfig.builder("propertyValue_circuitBreakerSleepWindowInMilliseconds").build(),
() => properties.CircuitBreakerSleepWindow.Get().Milliseconds));
monitors.Add(ServoMetricFactory.InformationalMetric <int>(this,
MonitorConfig.builder("propertyValue_circuitBreakerErrorThresholdPercentage").build(),
() => properties.CircuitBreakerErrorThresholdPercentage.Get()));
monitors.Add(ServoMetricFactory.InformationalMetric <bool>(this,
MonitorConfig.builder("propertyValue_circuitBreakerForceOpen").build(),
() => properties.CircuitBreakerForceOpen.Get()));
monitors.Add(ServoMetricFactory.InformationalMetric <bool>(this,
MonitorConfig.builder("propertyValue_circuitBreakerForceClosed").build(),
() => properties.CircuitBreakerForceClosed.Get()));
monitors.Add(ServoMetricFactory.InformationalMetric <long>(this,
MonitorConfig.builder("propertyValue_executionIsolationThreadTimeoutInMilliseconds").build(),
() => properties.ExecutionIsolationThreadTimeout.Get().Milliseconds));
//monitors.Add(ServoMetricFactory.InformationalMetric<TimeSpan>(this,
// MonitorConfig.builder("propertyValue_executionTimeoutInMilliseconds").build(),
// () => properties.executionTimeoutInMilliseconds.Get()));
monitors.Add(ServoMetricFactory.InformationalMetric <ExecutionIsolationStrategy>(this,
MonitorConfig.builder("propertyValue_executionIsolationStrategy").build(),
() => properties.ExecutionIsolationStrategy.Get()));
monitors.Add(ServoMetricFactory.InformationalMetric <bool>(this,
MonitorConfig.builder("propertyValue_metricsRollingPercentileEnabled").build(),
() => properties.MetricsRollingPercentileEnabled.Get()));
monitors.Add(ServoMetricFactory.InformationalMetric <bool>(this,
MonitorConfig.builder("propertyValue_requestCacheEnabled").build(),
() => properties.RequestCacheEnabled.Get()));
monitors.Add(ServoMetricFactory.InformationalMetric <bool>(this,
MonitorConfig.builder("propertyValue_requestLogEnabled").build(),
() => properties.RequestLogEnabled.Get()));
monitors.Add(ServoMetricFactory.InformationalMetric <int>(this,
MonitorConfig.builder("propertyValue_executionIsolationSemaphoreMaxConcurrentRequests").build(),
() => properties.ExecutionIsolationSemaphoreMaxConcurrentRequests.Get()));
monitors.Add(ServoMetricFactory.InformationalMetric <int>(this,
MonitorConfig.builder("propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests").build(),
() => properties.FallbackIsolationSemaphoreMaxConcurrentRequests.Get()));
return(monitors);
}
public void CircuitBreaker_MultipleTimeWindowRetriesBeforeClosingCircuit()
{
try
{
int sleepWindow = 200;
HystrixCommandPropertiesSetter properties = UnitTestSetterFactory.GetCommandPropertiesSetter().WithCircuitBreakerSleepWindowInMilliseconds(sleepWindow);
HystrixCommandMetrics metrics = GetMetrics(properties);
IHystrixCircuitBreaker cb = GetCircuitBreaker(key, CommandGroupForUnitTest.OwnerTwo, metrics, properties);
// fail
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
metrics.MarkFailure(1000);
// everything has failed in the test window so we should return false now
Assert.IsFalse(cb.AllowRequest());
Assert.IsTrue(cb.IsOpen());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// the 'singleTest' fails so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep
metrics.MarkFailure(1000);
Assert.IsFalse(cb.AllowRequest());
Assert.IsFalse(cb.AllowRequest());
Assert.IsFalse(cb.AllowRequest());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// the 'singleTest' fails again so it should go back to sleep and not allow any requests again until another 'singleTest' after the sleep
metrics.MarkFailure(1000);
Assert.IsFalse(cb.AllowRequest());
Assert.IsFalse(cb.AllowRequest());
Assert.IsFalse(cb.AllowRequest());
// wait for sleepWindow to pass
Thread.Sleep(sleepWindow + 50);
// we should now allow 1 request
Assert.IsTrue(cb.AllowRequest());
// but the circuit should still be open
Assert.IsTrue(cb.IsOpen());
// and further requests are still blocked
Assert.IsFalse(cb.AllowRequest());
// now it finally succeeds
metrics.MarkSuccess(200);
cb.MarkSuccess();
// all requests should be open again
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
Assert.IsTrue(cb.AllowRequest());
// and the circuit should be closed again
Assert.IsFalse(cb.IsOpen());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Assert.Fail("Error occurred: " + e.Message);
}
}
