internal void UpdateMaxConcurrent(int newLimit)
{
if (!IsValidMaxConcurrent(newLimit))
{
_log.Error($"Semaphore bulkhead config for key {_key} changed to an invalid limit of {newLimit}, the bulkhead will not be changed");
return;
}
_bulkhead = new SemaphoreBulkhead(_key, newLimit);
}
public CircuitBreakerFactory(IMetricEvents metricEvents, IFailurePercentageCircuitBreakerConfig breakerConfig, IMjolnirLogFactory logFactory)
{
_metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
_breakerConfig = breakerConfig ?? throw new ArgumentNullException(nameof(breakerConfig));
_logFactory = logFactory ?? throw new ArgumentNullException(nameof(logFactory));
_log = logFactory.CreateLog <FailurePercentageCircuitBreaker>();
if (_log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(CircuitBreakerFactory)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_timer = new GaugeTimer(state =>
{
try
{
var keys = _circuitBreakers.Keys;
foreach (var key in keys)
{
if (_circuitBreakers.TryGetValue(key, out Lazy <FailurePercentageCircuitBreaker> lazy) && lazy.IsValueCreated)
{
var breaker = lazy.Value;
_metricEvents.BreakerGauge(
breaker.Name,
_breakerConfig.GetMinimumOperations(key),
_breakerConfig.GetWindowMillis(key),
_breakerConfig.GetThresholdPercentage(key),
_breakerConfig.GetTrippedDurationMillis(key),
_breakerConfig.GetForceTripped(key),
_breakerConfig.GetForceFixed(key),
breaker.IsTripped(),
breaker.Metrics.SuccessCount,
breaker.Metrics.FailureCount);
}
}
}
catch (Exception e)
{
_log.Error($"Error sending {nameof(IMetricEvents.BreakerGauge)} metric event", e);
}
});
}
/// <summary>
/// Checks to see if the breaker should trip, and trips if it should.
/// </summary>
/// <returns><code>true</code> if breaker is tripped</returns>
private bool CheckAndSetTripped()
{
if (_state == State.Tripped)
{
return(true);
}
if (!Monitor.TryEnter(_stateChangeLock))
{
return(_state == State.Tripped);
}
try
{
var snapshot = _metrics.GetSnapshot();
// If we haven't met the minimum number of operations needed to trip, don't trip.
if (snapshot.Total < _config.GetMinimumOperations(_key))
{
return(false);
}
// If we're within the error threshold, don't trip.
if (snapshot.ErrorPercentage < _config.GetThresholdPercentage(_key))
{
return(false);
}
_state = State.Tripped;
_lastTrippedTimestamp = _clock.GetMillisecondTimestamp();
_metricEvents.BreakerTripped(Name);
_log.Error($"Tripped Breaker={_key} Operations={snapshot.Total} ErrorPercentage={snapshot.ErrorPercentage} Wait={_config.GetTrippedDurationMillis(_key)}");
return(true);
}
finally
{
Monitor.Exit(_stateChangeLock);
}
}
internal void Reset()
{
if (!Monitor.TryEnter(_resetBucketLock))
{
// Another thread already requested Reset(), we'll let them take care of it.
// If we were supposed to Reset(), but this thread couldn't acquire the lock,
// we may end up incrementing counts in the previous bucket before the other
// thread reassigns _counters. For now, I think that's okay.
// One way to combat this would be to keep an intermediate "carryover"
// counter that gets incremented in this block.
// Subsequent calls to Increment() could look at the carryover and, if > 0
// add those to the current bucket. Carryover values would have to expire
// at the same time the regular bucket periods do to avoid carryover that's
// followed by a long (> period) gap of no increments.
return;
}
try
{
var newBucket = CreateCounters();
_counters = newBucket;
// Should be the last statement in the try - see comment in catch block.
_lastResetAtTime = _clock.GetMillisecondTimestamp();
}
catch (Exception e)
{
// If there's an exception, _lastResetAtTime won't get updated - the next
// Increment() will try a Reset() again, which is good.
_log.Error("Error resetting metrics", e);
}
finally
{
Monitor.Exit(_resetBucketLock);
}
}