public CircuitBreakerFactory(IMetricEvents metricEvents, IFailurePercentageCircuitBreakerConfig breakerConfig, IMjolnirLogFactory logFactory) { _metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents)); _breakerConfig = breakerConfig ?? throw new ArgumentNullException(nameof(breakerConfig)); _logFactory = logFactory ?? throw new ArgumentNullException(nameof(logFactory)); _log = logFactory.CreateLog <FailurePercentageCircuitBreaker>(); if (_log == null) { throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(CircuitBreakerFactory)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}"); } _timer = new GaugeTimer(state => { try { var keys = _circuitBreakers.Keys; foreach (var key in keys) { if (_circuitBreakers.TryGetValue(key, out Lazy <FailurePercentageCircuitBreaker> lazy) && lazy.IsValueCreated) { var breaker = lazy.Value; _metricEvents.BreakerGauge( breaker.Name, _breakerConfig.GetMinimumOperations(key), _breakerConfig.GetWindowMillis(key), _breakerConfig.GetThresholdPercentage(key), _breakerConfig.GetTrippedDurationMillis(key), _breakerConfig.GetForceTripped(key), _breakerConfig.GetForceFixed(key), breaker.IsTripped(), breaker.Metrics.SuccessCount, breaker.Metrics.FailureCount); } } } catch (Exception e) { _log.Error($"Error sending {nameof(IMetricEvents.BreakerGauge)} metric event", e); } }); }
// ReSharper restore NotAccessedField.Local internal StpIsolationThreadPool(GroupKey key, IConfigurableValue <int> threadCount, IConfigurableValue <int> queueLength, IStats stats, IConfigurableValue <long> gaugeIntervalMillisOverride = null) { _key = key; _threadCount = threadCount; _queueLength = queueLength; if (stats == null) { throw new ArgumentNullException("stats"); } _stats = stats; var count = _threadCount.Value; var info = new STPStartInfo { ThreadPoolName = _key.Name, MinWorkerThreads = count, MaxWorkerThreads = count, MaxQueueLength = queueLength.Value, AreThreadsBackground = true, UseCallerExecutionContext = true, UseCallerHttpContext = true }; _pool = new SmartThreadPool(info); _timer = new GaugeTimer((source, args) => { _stats.Gauge(StatsPrefix + " activeThreads", null, _pool.ActiveThreads); _stats.Gauge(StatsPrefix + " inUseThreads", null, _pool.InUseThreads); // Note: Don't use _pool.WaitingCallbacks. It has the potential to get locked out by // queue/dequeue operations, and may block here if the pool's getting queued into heavily. _stats.Gauge(StatsPrefix + " pendingCompletion", null, _pool.CurrentWorkItemsCount); }, gaugeIntervalMillisOverride); _pool.OnThreadInitialization += () => _stats.Event(StatsPrefix + " thread", "Initialized", null); _pool.OnThreadTermination += () => _stats.Event(StatsPrefix + " thread", "Terminated", null); _threadCount.AddChangeHandler(UpdateThreadCount); _queueLength.AddChangeHandler(UpdateQueueLength); }
// ReSharper restore NotAccessedField.Local internal SemaphoreSlimIsolationSemaphore(GroupKey key, IConfigurableValue <int> maxConcurrent, IStats stats, IConfigurableValue <long> gaugeIntervalMillisOverride = null) { _key = key; if (stats == null) { throw new ArgumentNullException("stats"); } _stats = stats; // Note: Changing the semaphore maximum at runtime is not currently supported. _maxConcurrent = maxConcurrent.Value; _semaphore = new SemaphoreSlim(_maxConcurrent); _timer = new GaugeTimer((source, args) => { var count = _semaphore.CurrentCount; _stats.Gauge(StatsPrefix + " available", (count == 0 ? "Full" : "Available"), count); }, gaugeIntervalMillisOverride); }
// ReSharper restore PrivateFieldCanBeConvertedToLocalVariable public BulkheadFactory(IMetricEvents metricEvents, MjolnirConfiguration config, IMjolnirLogFactory logFactory) { // No null checks on parameters; we don't use them, we're just passing them through to // the objects we're creating. _metricEvents = metricEvents; _config = config; _logFactory = logFactory ?? throw new ArgumentNullException(nameof(logFactory)); var log = logFactory.CreateLog <BulkheadFactory>(); if (log == null) { throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(BulkheadFactory)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}"); } _timer = new GaugeTimer(state => { try { var keys = _bulkheads.Keys; foreach (var key in keys) { if (_bulkheads.TryGetValue(key, out Lazy <SemaphoreBulkheadHolder> holder) && holder.IsValueCreated) { var bulkhead = holder.Value.Bulkhead; _metricEvents.BulkheadGauge(bulkhead.Name, "semaphore", _config.GetBulkheadConfiguration(key.Name).MaxConcurrent, bulkhead.CountAvailable); } } } catch (Exception e) { log.Error($"Error sending {nameof(IMetricEvents.BulkheadGauge)} metric event", e); } }); }
internal FailurePercentageCircuitBreaker(GroupKey key, IClock clock, ICommandMetrics metrics, IStats stats, FailurePercentageCircuitBreakerProperties properties, IConfigurableValue <long> gaugeIntervalMillisOverride = null) { _key = key; _clock = clock; _metrics = metrics; if (stats == null) { throw new ArgumentNullException("stats"); } _stats = stats; Properties = properties; _state = State.Fixed; // Start off assuming everything's fixed. _lastTrippedTimestamp = 0; // 0 is fine since it'll be far less than the first compared value. _timer = new GaugeTimer((source, args) => { var snapshot = _metrics.GetSnapshot(); _stats.Gauge(StatsPrefix + " total", snapshot.Total >= properties.MinimumOperations.Value ? "Above" : "Below", snapshot.Total); _stats.Gauge(StatsPrefix + " error", snapshot.ErrorPercentage >= properties.ThresholdPercentage.Value ? "Above" : "Below", snapshot.ErrorPercentage); }, gaugeIntervalMillisOverride); }