Beispiel #1
0
            internal void UpdateMaxConcurrent(int newLimit)
            {
                if (!IsValidMaxConcurrent(newLimit))
                {
                    _log.Error($"Semaphore bulkhead config for key {_key} changed to an invalid limit of {newLimit}, the bulkhead will not be changed");
                    return;
                }

                _bulkhead = new SemaphoreBulkhead(_key, newLimit);
            }
Beispiel #2
0
        public CircuitBreakerFactory(IMetricEvents metricEvents, IFailurePercentageCircuitBreakerConfig breakerConfig, IMjolnirLogFactory logFactory)
        {
            _metricEvents  = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
            _breakerConfig = breakerConfig ?? throw new ArgumentNullException(nameof(breakerConfig));
            _logFactory    = logFactory ?? throw new ArgumentNullException(nameof(logFactory));

            _log = logFactory.CreateLog <FailurePercentageCircuitBreaker>();
            if (_log == null)
            {
                throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(CircuitBreakerFactory)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
            }

            _timer = new GaugeTimer(state =>
            {
                try
                {
                    var keys = _circuitBreakers.Keys;
                    foreach (var key in keys)
                    {
                        if (_circuitBreakers.TryGetValue(key, out Lazy <FailurePercentageCircuitBreaker> lazy) && lazy.IsValueCreated)
                        {
                            var breaker = lazy.Value;
                            _metricEvents.BreakerGauge(
                                breaker.Name,
                                _breakerConfig.GetMinimumOperations(key),
                                _breakerConfig.GetWindowMillis(key),
                                _breakerConfig.GetThresholdPercentage(key),
                                _breakerConfig.GetTrippedDurationMillis(key),
                                _breakerConfig.GetForceTripped(key),
                                _breakerConfig.GetForceFixed(key),
                                breaker.IsTripped(),
                                breaker.Metrics.SuccessCount,
                                breaker.Metrics.FailureCount);
                        }
                    }
                }
                catch (Exception e)
                {
                    _log.Error($"Error sending {nameof(IMetricEvents.BreakerGauge)} metric event", e);
                }
            });
        }
Beispiel #3
0
        /// <summary>
        /// Checks to see if the breaker should trip, and trips if it should.
        /// </summary>
        /// <returns><code>true</code> if breaker is tripped</returns>
        private bool CheckAndSetTripped()
        {
            if (_state == State.Tripped)
            {
                return(true);
            }

            if (!Monitor.TryEnter(_stateChangeLock))
            {
                return(_state == State.Tripped);
            }

            try
            {
                var snapshot = _metrics.GetSnapshot();

                // If we haven't met the minimum number of operations needed to trip, don't trip.
                if (snapshot.Total < _config.GetMinimumOperations(_key))
                {
                    return(false);
                }

                // If we're within the error threshold, don't trip.
                if (snapshot.ErrorPercentage < _config.GetThresholdPercentage(_key))
                {
                    return(false);
                }

                _state = State.Tripped;
                _lastTrippedTimestamp = _clock.GetMillisecondTimestamp();

                _metricEvents.BreakerTripped(Name);
                _log.Error($"Tripped Breaker={_key} Operations={snapshot.Total} ErrorPercentage={snapshot.ErrorPercentage} Wait={_config.GetTrippedDurationMillis(_key)}");

                return(true);
            }
            finally
            {
                Monitor.Exit(_stateChangeLock);
            }
        }
        internal void Reset()
        {
            if (!Monitor.TryEnter(_resetBucketLock))
            {
                // Another thread already requested Reset(), we'll let them take care of it.

                // If we were supposed to Reset(), but this thread couldn't acquire the lock,
                // we may end up incrementing counts in the previous bucket before the other
                // thread reassigns _counters. For now, I think that's okay.

                // One way to combat this would be to keep an intermediate "carryover"
                // counter that gets incremented in this block.

                // Subsequent calls to Increment() could look at the carryover and, if > 0
                // add those to the current bucket. Carryover values would have to expire
                // at the same time the regular bucket periods do to avoid carryover that's
                // followed by a long (> period) gap of no increments.
                return;
            }

            try
            {
                var newBucket = CreateCounters();
                _counters = newBucket;

                // Should be the last statement in the try - see comment in catch block.
                _lastResetAtTime = _clock.GetMillisecondTimestamp();
            }
            catch (Exception e)
            {
                // If there's an exception, _lastResetAtTime won't get updated - the next
                // Increment() will try a Reset() again, which is good.
                _log.Error("Error resetting metrics", e);
            }
            finally
            {
                Monitor.Exit(_resetBucketLock);
            }
        }