public CircuitBreakerFactory(IMetricEvents metricEvents, IFailurePercentageCircuitBreakerConfig breakerConfig, IMjolnirLogFactory logFactory)
{
_metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
_breakerConfig = breakerConfig ?? throw new ArgumentNullException(nameof(breakerConfig));
_logFactory = logFactory ?? throw new ArgumentNullException(nameof(logFactory));
_log = logFactory.CreateLog <FailurePercentageCircuitBreaker>();
if (_log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(CircuitBreakerFactory)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_timer = new GaugeTimer(state =>
{
try
{
var keys = _circuitBreakers.Keys;
foreach (var key in keys)
{
if (_circuitBreakers.TryGetValue(key, out Lazy <FailurePercentageCircuitBreaker> lazy) && lazy.IsValueCreated)
{
var breaker = lazy.Value;
_metricEvents.BreakerGauge(
breaker.Name,
_breakerConfig.GetMinimumOperations(key),
_breakerConfig.GetWindowMillis(key),
_breakerConfig.GetThresholdPercentage(key),
_breakerConfig.GetTrippedDurationMillis(key),
_breakerConfig.GetForceTripped(key),
_breakerConfig.GetForceFixed(key),
breaker.IsTripped(),
breaker.Metrics.SuccessCount,
breaker.Metrics.FailureCount);
}
}
}
catch (Exception e)
{
_log.Error($"Error sending {nameof(IMetricEvents.BreakerGauge)} metric event", e);
}
});
}
// ReSharper restore NotAccessedField.Local
internal StpIsolationThreadPool(GroupKey key, IConfigurableValue <int> threadCount, IConfigurableValue <int> queueLength, IStats stats, IConfigurableValue <long> gaugeIntervalMillisOverride = null)
{
_key = key;
_threadCount = threadCount;
_queueLength = queueLength;
if (stats == null)
{
throw new ArgumentNullException("stats");
}
_stats = stats;
var count = _threadCount.Value;
var info = new STPStartInfo
{
ThreadPoolName = _key.Name,
MinWorkerThreads = count,
MaxWorkerThreads = count,
MaxQueueLength = queueLength.Value,
AreThreadsBackground = true,
UseCallerExecutionContext = true,
UseCallerHttpContext = true
};
_pool = new SmartThreadPool(info);
_timer = new GaugeTimer((source, args) =>
{
_stats.Gauge(StatsPrefix + " activeThreads", null, _pool.ActiveThreads);
_stats.Gauge(StatsPrefix + " inUseThreads", null, _pool.InUseThreads);
// Note: Don't use _pool.WaitingCallbacks. It has the potential to get locked out by
// queue/dequeue operations, and may block here if the pool's getting queued into heavily.
_stats.Gauge(StatsPrefix + " pendingCompletion", null, _pool.CurrentWorkItemsCount);
}, gaugeIntervalMillisOverride);
_pool.OnThreadInitialization += () => _stats.Event(StatsPrefix + " thread", "Initialized", null);
_pool.OnThreadTermination += () => _stats.Event(StatsPrefix + " thread", "Terminated", null);
_threadCount.AddChangeHandler(UpdateThreadCount);
_queueLength.AddChangeHandler(UpdateQueueLength);
}
// ReSharper restore NotAccessedField.Local
internal SemaphoreSlimIsolationSemaphore(GroupKey key, IConfigurableValue <int> maxConcurrent, IStats stats, IConfigurableValue <long> gaugeIntervalMillisOverride = null)
{
_key = key;
if (stats == null)
{
throw new ArgumentNullException("stats");
}
_stats = stats;
// Note: Changing the semaphore maximum at runtime is not currently supported.
_maxConcurrent = maxConcurrent.Value;
_semaphore = new SemaphoreSlim(_maxConcurrent);
_timer = new GaugeTimer((source, args) =>
{
var count = _semaphore.CurrentCount;
_stats.Gauge(StatsPrefix + " available", (count == 0 ? "Full" : "Available"), count);
}, gaugeIntervalMillisOverride);
}
// ReSharper restore PrivateFieldCanBeConvertedToLocalVariable
public BulkheadFactory(IMetricEvents metricEvents, MjolnirConfiguration config, IMjolnirLogFactory logFactory)
{
// No null checks on parameters; we don't use them, we're just passing them through to
// the objects we're creating.
_metricEvents = metricEvents;
_config = config;
_logFactory = logFactory ?? throw new ArgumentNullException(nameof(logFactory));
var log = logFactory.CreateLog <BulkheadFactory>();
if (log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(BulkheadFactory)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_timer = new GaugeTimer(state =>
{
try
{
var keys = _bulkheads.Keys;
foreach (var key in keys)
{
if (_bulkheads.TryGetValue(key, out Lazy <SemaphoreBulkheadHolder> holder) && holder.IsValueCreated)
{
var bulkhead = holder.Value.Bulkhead;
_metricEvents.BulkheadGauge(bulkhead.Name, "semaphore", _config.GetBulkheadConfiguration(key.Name).MaxConcurrent, bulkhead.CountAvailable);
}
}
}
catch (Exception e)
{
log.Error($"Error sending {nameof(IMetricEvents.BulkheadGauge)} metric event", e);
}
});
}
internal FailurePercentageCircuitBreaker(GroupKey key, IClock clock, ICommandMetrics metrics, IStats stats, FailurePercentageCircuitBreakerProperties properties, IConfigurableValue <long> gaugeIntervalMillisOverride = null)
{
_key = key;
_clock = clock;
_metrics = metrics;
if (stats == null)
{
throw new ArgumentNullException("stats");
}
_stats = stats;
Properties = properties;
_state = State.Fixed; // Start off assuming everything's fixed.
_lastTrippedTimestamp = 0; // 0 is fine since it'll be far less than the first compared value.
_timer = new GaugeTimer((source, args) =>
{
var snapshot = _metrics.GetSnapshot();
_stats.Gauge(StatsPrefix + " total", snapshot.Total >= properties.MinimumOperations.Value ? "Above" : "Below", snapshot.Total);
_stats.Gauge(StatsPrefix + " error", snapshot.ErrorPercentage >= properties.ThresholdPercentage.Value ? "Above" : "Below", snapshot.ErrorPercentage);
}, gaugeIntervalMillisOverride);
}
