internal void UpdateMaxConcurrent(int newLimit) { if (!IsValidMaxConcurrent(newLimit)) { _log.Error($"Semaphore bulkhead config for key {_key} changed to an invalid limit of {newLimit}, the bulkhead will not be changed"); return; } _bulkhead = new SemaphoreBulkhead(_key, newLimit); }
public CircuitBreakerFactory(IMetricEvents metricEvents, IFailurePercentageCircuitBreakerConfig breakerConfig, IMjolnirLogFactory logFactory) { _metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents)); _breakerConfig = breakerConfig ?? throw new ArgumentNullException(nameof(breakerConfig)); _logFactory = logFactory ?? throw new ArgumentNullException(nameof(logFactory)); _log = logFactory.CreateLog <FailurePercentageCircuitBreaker>(); if (_log == null) { throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(CircuitBreakerFactory)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}"); } _timer = new GaugeTimer(state => { try { var keys = _circuitBreakers.Keys; foreach (var key in keys) { if (_circuitBreakers.TryGetValue(key, out Lazy <FailurePercentageCircuitBreaker> lazy) && lazy.IsValueCreated) { var breaker = lazy.Value; _metricEvents.BreakerGauge( breaker.Name, _breakerConfig.GetMinimumOperations(key), _breakerConfig.GetWindowMillis(key), _breakerConfig.GetThresholdPercentage(key), _breakerConfig.GetTrippedDurationMillis(key), _breakerConfig.GetForceTripped(key), _breakerConfig.GetForceFixed(key), breaker.IsTripped(), breaker.Metrics.SuccessCount, breaker.Metrics.FailureCount); } } } catch (Exception e) { _log.Error($"Error sending {nameof(IMetricEvents.BreakerGauge)} metric event", e); } }); }
/// <summary> /// Checks to see if the breaker should trip, and trips if it should. /// </summary> /// <returns><code>true</code> if breaker is tripped</returns> private bool CheckAndSetTripped() { if (_state == State.Tripped) { return(true); } if (!Monitor.TryEnter(_stateChangeLock)) { return(_state == State.Tripped); } try { var snapshot = _metrics.GetSnapshot(); // If we haven't met the minimum number of operations needed to trip, don't trip. if (snapshot.Total < _config.GetMinimumOperations(_key)) { return(false); } // If we're within the error threshold, don't trip. if (snapshot.ErrorPercentage < _config.GetThresholdPercentage(_key)) { return(false); } _state = State.Tripped; _lastTrippedTimestamp = _clock.GetMillisecondTimestamp(); _metricEvents.BreakerTripped(Name); _log.Error($"Tripped Breaker={_key} Operations={snapshot.Total} ErrorPercentage={snapshot.ErrorPercentage} Wait={_config.GetTrippedDurationMillis(_key)}"); return(true); } finally { Monitor.Exit(_stateChangeLock); } }
internal void Reset() { if (!Monitor.TryEnter(_resetBucketLock)) { // Another thread already requested Reset(), we'll let them take care of it. // If we were supposed to Reset(), but this thread couldn't acquire the lock, // we may end up incrementing counts in the previous bucket before the other // thread reassigns _counters. For now, I think that's okay. // One way to combat this would be to keep an intermediate "carryover" // counter that gets incremented in this block. // Subsequent calls to Increment() could look at the carryover and, if > 0 // add those to the current bucket. Carryover values would have to expire // at the same time the regular bucket periods do to avoid carryover that's // followed by a long (> period) gap of no increments. return; } try { var newBucket = CreateCounters(); _counters = newBucket; // Should be the last statement in the try - see comment in catch block. _lastResetAtTime = _clock.GetMillisecondTimestamp(); } catch (Exception e) { // If there's an exception, _lastResetAtTime won't get updated - the next // Increment() will try a Reset() again, which is good. _log.Error("Error resetting metrics", e); } finally { Monitor.Exit(_resetBucketLock); } }