internal BaseCommand(string group, string name, string breakerKey, string bulkheadKey, TimeSpan? defaultTimeout = null)
{
if (string.IsNullOrWhiteSpace(group))
{
throw new ArgumentNullException("group");
}
if (string.IsNullOrWhiteSpace(breakerKey))
{
throw new ArgumentNullException("breakerKey");
}
if (string.IsNullOrWhiteSpace(bulkheadKey))
{
throw new ArgumentNullException("bulkheadKey");
}
if (defaultTimeout != null && defaultTimeout.Value.TotalMilliseconds <= 0)
{
throw new ArgumentException(
string.Format("Positive default timeout is required if passed (received invalid timeout of {0}ms)", defaultTimeout.Value.TotalMilliseconds),
"defaultTimeout");
}
_group = GroupKey.Named(group);
_name = string.IsNullOrWhiteSpace(name) ? GenerateAndCacheName(Group) : CacheProvidedName(Group, name);
_breakerKey = GroupKey.Named(breakerKey);
_bulkheadKey = GroupKey.Named(bulkheadKey);
_constructorTimeout = defaultTimeout ?? DefaultTimeout;
}
internal SemaphoreBulkhead(GroupKey key, int maxConcurrent)
{
if (maxConcurrent < 0)
{
throw new ArgumentOutOfRangeException("maxConcurrent", maxConcurrent, "Semaphore bulkhead must have a limit >= 0");
}
_key = key;
_semaphore = new SemaphoreSlim(maxConcurrent);
}
internal StandardCommandMetrics(GroupKey key, IConfigurableValue<long> windowMillis, IConfigurableValue<long> snapshotTtlMillis, IClock clock, IStats stats)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
_key = key;
_clock = clock;
_snapshotTtlMillis = snapshotTtlMillis;
_resettingNumbersBucket = new ResettingNumbersBucket(_clock, windowMillis);
if (stats == null)
{
throw new ArgumentNullException("stats");
}
_stats = stats;
}
// ReSharper restore NotAccessedField.Local
internal SemaphoreSlimIsolationSemaphore(GroupKey key, IConfigurableValue<int> maxConcurrent, IStats stats, IConfigurableValue<long> gaugeIntervalMillisOverride = null)
{
_key = key;
if (stats == null)
{
throw new ArgumentNullException("stats");
}
_stats = stats;
// Note: Changing the semaphore maximum at runtime is not currently supported.
_maxConcurrent = maxConcurrent.Value;
_semaphore = new SemaphoreSlim(_maxConcurrent);
_timer = new GaugeTimer((source, args) =>
{
var count = _semaphore.CurrentCount;
_stats.Gauge(StatsPrefix + " available", (count == 0 ? "Full" : "Available"), count);
}, gaugeIntervalMillisOverride);
}
internal FailurePercentageCircuitBreaker(GroupKey key, IClock clock, ICommandMetrics metrics, IStats stats, IMetricEvents metricEvents, FailurePercentageCircuitBreakerProperties properties, IConfigurableValue<long> gaugeIntervalMillisOverride = null)
{
_key = key;
_clock = clock;
_metrics = metrics;
if (stats == null)
{
throw new ArgumentNullException("stats");
}
if (metricEvents == null)
{
throw new ArgumentNullException("metricEvents");
}
_stats = stats;
_metricEvents = metricEvents;
Properties = properties;
_state = State.Fixed; // Start off assuming everything's fixed.
_lastTrippedTimestamp = 0; // 0 is fine since it'll be far less than the first compared value.
// Old gauge, will be phased out in v3.0 when IStats are removed.
_statsTimer = new GaugeTimer((source, args) =>
{
var snapshot = _metrics.GetSnapshot();
_stats.Gauge(StatsPrefix + " total", snapshot.Total >= properties.MinimumOperations.Value ? "Above" : "Below", snapshot.Total);
_stats.Gauge(StatsPrefix + " error", snapshot.ErrorPercentage >= properties.ThresholdPercentage.Value ? "Above" : "Below", snapshot.ErrorPercentage);
}, gaugeIntervalMillisOverride);
_metricsTimer = new GaugeTimer((source, args) =>
{
_metricEvents.BreakerConfigGauge(
Name,
Properties.MinimumOperations.Value,
Properties.ThresholdPercentage.Value,
Properties.TrippedDurationMillis.Value);
}, ConfigGaugeIntervalMillis);
}
// Since creating the Command's name is non-trivial, we'll keep a local
// cache of them. They're accessed frequently (at least once per call), so avoiding all
// of this string manipulation every time seems like a good idea.
private string GenerateAndCacheName(GroupKey group)
{
var type = GetType();
var cacheKey = new Tuple<Type, GroupKey>(type, group);
return GeneratedNameCache.GetOrAdd(cacheKey, t =>
{
var className = cacheKey.Item1.Name;
if (className.EndsWith("Command", StringComparison.InvariantCulture))
{
className = className.Substring(0, className.LastIndexOf("Command", StringComparison.InvariantCulture));
}
return cacheKey.Item2.Name.Replace(".", "-") + "." + className;
});
}
private static string CacheProvidedName(GroupKey group, string name)
{
var cacheKey = new Tuple<string, GroupKey>(name, group);
return ProvidedNameCache.GetOrAdd(cacheKey, t => cacheKey.Item2.Name.Replace(".", "-") + "." + name.Replace(".", "-"));
}
internal static IIsolationSemaphore GetFallbackSemaphore(GroupKey key)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
return Instance._fallbackSemaphores.GetOrAddSafe(key, k =>
{
// For now, the default here is 5x the default pool threadCount, with the presumption that
// several commands may using the same pool, and we should therefore try to allow for a bit
// more concurrent fallback execution.
var maxConcurrent = new ConfigurableValue<int>("mjolnir.fallback." + key + ".maxConcurrent", DefaultFallbackMaxConcurrent);
return new SemaphoreSlimIsolationSemaphore(key, maxConcurrent, Stats);
});
}
internal static IIsolationThreadPool GetThreadPool(GroupKey key)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
return Instance._pools.GetOrAddSafe(key, k =>
new StpIsolationThreadPool(
key,
new ConfigurableValue<int>("mjolnir.pools." + key + ".threadCount", DefaultPoolThreadCount),
new ConfigurableValue<int>("mjolnir.pools." + key + ".queueLength", DefaultPoolQueueLength),
Stats));
}
private static ICommandMetrics GetCommandMetrics(GroupKey key)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
return Instance._metrics.GetOrAddSafe(key, k =>
new StandardCommandMetrics(
key,
new ConfigurableValue<long>("mjolnir.metrics." + key + ".windowMillis", DefaultMetricsWindowMillis),
new ConfigurableValue<long>("mjolnir.metrics." + key + ".snapshotTtlMillis", DefaultMetricsSnapshotTtlMillis),
Stats));
}
private ICommandMetrics GetCommandMetrics(GroupKey key)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
return _metrics.GetOrAddSafe(key, k =>
new StandardCommandMetrics(
key,
new ConfigurableValue<long>("mjolnir.metrics." + key + ".windowMillis", DefaultMetricsWindowMillis),
new ConfigurableValue<long>("mjolnir.metrics." + key + ".snapshotTtlMillis", DefaultMetricsSnapshotTtlMillis),
Stats),
LazyThreadSafetyMode.ExecutionAndPublication);
}
internal StandardCommandMetrics(GroupKey key, IConfigurableValue<long> windowMillis, IConfigurableValue<long> snapshotTtlMillis, IStats stats)
: this(key, windowMillis, snapshotTtlMillis, new SystemClock(), stats) {}
public SemaphoreBulkheadHolder(GroupKey key, IMetricEvents metricEvents)
{
if (metricEvents == null)
{
throw new ArgumentNullException("metricEvents");
}
_metricEvents = metricEvents;
// The order of things here is very intentional.
// We create the configurable value first, retrieve its current value, and then
// initialize the semaphore bulkhead. We register the change handler after that.
// That ought to help avoid a situation where we might fire a config change handler
// before we add the semaphore to the dictionary, potentially trying to add two
// entries with different values in rapid succession.
var configKey = "mjolnir.bulkhead." + key + ".maxConcurrent";
_config = new ConfigurableValue<int>(configKey, DefaultBulkheadMaxConcurrent);
var value = _config.Value;
_bulkhead = new SemaphoreBulkhead(key, value);
// On change, we'll replace the bulkhead. The assumption here is that a caller
// using the bulkhead will have kept a local reference to the bulkhead that they
// acquired a lock on, and will release the lock on that bulkhead and not one that
// has been replaced after a config change.
_config.AddChangeHandler(newLimit =>
{
if (newLimit < 0)
{
Log.ErrorFormat("Semaphore bulkhead config {0} changed to an invalid limit of {0}, the bulkhead will not be changed",
configKey,
newLimit);
return;
}
_bulkhead = new SemaphoreBulkhead(key, newLimit);
});
_timer = new GaugeTimer((source, args) =>
{
_metricEvents.BulkheadConfigGauge(_bulkhead.Name, "semaphore", _config.Value);
}, ConfigGaugeIntervalMillis);
}
/// <summary>
/// Callers should keep a local reference to the bulkhead object they receive from this
/// method, ensuring that they call TryEnter and Release on the same object reference.
/// Phrased differently: don't re-retrieve the bulkhead before calling Release().
/// </summary>
public IBulkheadSemaphore GetBulkhead(GroupKey key)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
var holder = _bulkheads.GetOrAddSafe(key,
k => new SemaphoreBulkheadHolder(key, _metricEvents),
LazyThreadSafetyMode.ExecutionAndPublication);
return holder.Bulkhead;
}
public IIsolationThreadPool GetThreadPool(GroupKey key)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
return _pools.GetOrAddSafe(key, k =>
new StpIsolationThreadPool(
key,
new ConfigurableValue<int>("mjolnir.pools." + key + ".threadCount", DefaultPoolThreadCount),
new ConfigurableValue<int>("mjolnir.pools." + key + ".queueLength", DefaultPoolQueueLength),
Stats,
MetricEvents),
LazyThreadSafetyMode.ExecutionAndPublication);
}
internal FailurePercentageCircuitBreaker(GroupKey key, ICommandMetrics metrics, IStats stats, FailurePercentageCircuitBreakerProperties properties)
: this(key, new SystemClock(), metrics, stats, properties) {}
internal static ICircuitBreaker GetCircuitBreaker(GroupKey key)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
return Instance._circuitBreakers.GetOrAddSafe(key, k =>
{
var metrics = GetCommandMetrics(key);
var properties = new FailurePercentageCircuitBreakerProperties(
new ConfigurableValue<long>("mjolnir.breaker." + key + ".minimumOperations", DefaultBreakerMinimumOperations),
new ConfigurableValue<int>("mjolnir.breaker." + key + ".thresholdPercentage", DefaultBreakerThresholdPercentage),
new ConfigurableValue<long>("mjolnir.breaker." + key + ".trippedDurationMillis", DefaultBreakerTrippedDurationMillis),
new ConfigurableValue<bool>("mjolnir.breaker." + key + ".forceTripped", DefaultBreakerForceTripped),
new ConfigurableValue<bool>("mjolnir.breaker." + key + ".forceFixed", DefaultBreakerForceFixed));
return new FailurePercentageCircuitBreaker(key, metrics, Stats, properties);
});
}