public SemaphoreBulkheadHolder(GroupKey key, IMetricEvents metricEvents, MjolnirConfiguration config, IMjolnirLogFactory logFactory)
{
_key = key;
_metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
_config = config ?? throw new ArgumentNullException(nameof(config));
if (logFactory == null)
{
throw new ArgumentNullException(nameof(logFactory));
}
_log = logFactory.CreateLog <SemaphoreBulkheadHolder>();
if (_log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(SemaphoreBulkheadHolder)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
// The order of things here is very intentional.
// We get the MaxConcurrent value first and then initialize the semaphore bulkhead.
// The change handler is registered after that. The order ought to help avoid a
// situation where we might fire a config change handler before we add the
// semaphore to the dictionary, potentially trying to add two entries with
// different values in rapid succession.
var value = _config.GetBulkheadConfiguration(key.Name).MaxConcurrent;
_bulkhead = new SemaphoreBulkhead(_key, value);
// On change, we'll replace the bulkhead. The assumption here is that a caller
// using the bulkhead will have kept a local reference to the bulkhead that they
// acquired a lock on, and will release the lock on that bulkhead and not one that
// has been replaced after a config change.
_config.OnConfigurationChanged(c => c.GetBulkheadConfiguration(key.Name).MaxConcurrent, UpdateMaxConcurrent);
}
public CommandInvoker(
MjolnirConfiguration config = null,
IMjolnirLogFactory logFactory = null,
IMetricEvents metricEvents = null,
IBreakerExceptionHandler breakerExceptionHandler = null)
: this(config, logFactory, metricEvents, breakerExceptionHandler, null)
{ }
public BulkheadInvoker(IBreakerInvoker breakerInvoker, IBulkheadFactory bulkheadFactory, IMetricEvents metricEvents, MjolnirConfiguration config)
{
_breakerInvoker = breakerInvoker ?? throw new ArgumentNullException(nameof(breakerInvoker));
_bulkheadFactory = bulkheadFactory ?? throw new ArgumentNullException(nameof(bulkheadFactory));
_metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
_config = config ?? throw new ArgumentNullException(nameof(config));
}
public CircuitBreakerFactory(IMetricEvents metricEvents, IFailurePercentageCircuitBreakerConfig breakerConfig, IMjolnirLogFactory logFactory)
{
_metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
_breakerConfig = breakerConfig ?? throw new ArgumentNullException(nameof(breakerConfig));
_logFactory = logFactory ?? throw new ArgumentNullException(nameof(logFactory));
_log = logFactory.CreateLog <FailurePercentageCircuitBreaker>();
if (_log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(CircuitBreakerFactory)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_timer = new GaugeTimer(state =>
{
try
{
var keys = _circuitBreakers.Keys;
foreach (var key in keys)
{
if (_circuitBreakers.TryGetValue(key, out Lazy <FailurePercentageCircuitBreaker> lazy) && lazy.IsValueCreated)
{
var breaker = lazy.Value;
_metricEvents.BreakerGauge(
breaker.Name,
_breakerConfig.GetMinimumOperations(key),
_breakerConfig.GetWindowMillis(key),
_breakerConfig.GetThresholdPercentage(key),
_breakerConfig.GetTrippedDurationMillis(key),
_breakerConfig.GetForceTripped(key),
_breakerConfig.GetForceFixed(key),
breaker.IsTripped(),
breaker.Metrics.SuccessCount,
breaker.Metrics.FailureCount);
}
}
}
catch (Exception e)
{
_log.Error($"Error sending {nameof(IMetricEvents.BreakerGauge)} metric event", e);
}
});
}
internal FailurePercentageCircuitBreaker(GroupKey key, IClock clock, ICommandMetrics metrics, IStats stats, IMetricEvents metricEvents, FailurePercentageCircuitBreakerProperties properties, IConfigurableValue<long> gaugeIntervalMillisOverride = null)
{
_key = key;
_clock = clock;
_metrics = metrics;
if (stats == null)
{
throw new ArgumentNullException("stats");
}
if (metricEvents == null)
{
throw new ArgumentNullException("metricEvents");
}
_stats = stats;
_metricEvents = metricEvents;
Properties = properties;
_state = State.Fixed; // Start off assuming everything's fixed.
_lastTrippedTimestamp = 0; // 0 is fine since it'll be far less than the first compared value.
// Old gauge, will be phased out in v3.0 when IStats are removed.
_statsTimer = new GaugeTimer((source, args) =>
{
var snapshot = _metrics.GetSnapshot();
_stats.Gauge(StatsPrefix + " total", snapshot.Total >= properties.MinimumOperations.Value ? "Above" : "Below", snapshot.Total);
_stats.Gauge(StatsPrefix + " error", snapshot.ErrorPercentage >= properties.ThresholdPercentage.Value ? "Above" : "Below", snapshot.ErrorPercentage);
}, gaugeIntervalMillisOverride);
_metricsTimer = new GaugeTimer((source, args) =>
{
_metricEvents.BreakerConfigGauge(
Name,
Properties.MinimumOperations.Value,
Properties.ThresholdPercentage.Value,
Properties.TrippedDurationMillis.Value);
}, ConfigGaugeIntervalMillis);
}
internal FailurePercentageCircuitBreaker(GroupKey key, IClock clock, ICommandMetrics metrics, IMetricEvents metricEvents, IFailurePercentageCircuitBreakerConfig config, IMjolnirLogFactory logFactory)
{
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_clock = clock ?? throw new ArgumentNullException(nameof(config));
_config = config ?? throw new ArgumentNullException(nameof(config));
_metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
if (logFactory == null)
{
throw new ArgumentNullException(nameof(logFactory));
}
_key = key;
_log = logFactory.CreateLog <FailurePercentageCircuitBreaker>();
if (_log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(FailurePercentageCircuitBreaker)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_state = State.Fixed; // Start off assuming everything's fixed.
_lastTrippedTimestamp = 0; // 0 is fine since it'll be far less than the first compared value.
}
// ReSharper restore PrivateFieldCanBeConvertedToLocalVariable
public BulkheadFactory(IMetricEvents metricEvents, MjolnirConfiguration config, IMjolnirLogFactory logFactory)
{
// No null checks on parameters; we don't use them, we're just passing them through to
// the objects we're creating.
_metricEvents = metricEvents;
_config = config;
_logFactory = logFactory ?? throw new ArgumentNullException(nameof(logFactory));
var log = logFactory.CreateLog <BulkheadFactory>();
if (log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(BulkheadFactory)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_timer = new GaugeTimer(state =>
{
try
{
var keys = _bulkheads.Keys;
foreach (var key in keys)
{
if (_bulkheads.TryGetValue(key, out Lazy <SemaphoreBulkheadHolder> holder) && holder.IsValueCreated)
{
var bulkhead = holder.Value.Bulkhead;
_metricEvents.BulkheadGauge(bulkhead.Name, "semaphore", _config.GetBulkheadConfiguration(key.Name).MaxConcurrent, bulkhead.CountAvailable);
}
}
}
catch (Exception e)
{
log.Error($"Error sending {nameof(IMetricEvents.BulkheadGauge)} metric event", e);
}
});
}
// Internal constructor with a few extra arguments used by tests to inject mocks.
internal CommandInvoker(
MjolnirConfiguration config = null,
IMjolnirLogFactory logFactory = null,
IMetricEvents metricEvents = null,
IBreakerExceptionHandler breakerExceptionHandler = null,
IBulkheadInvoker bulkheadInvoker = null)
{
_config = config ?? new MjolnirConfiguration
{
IsEnabled = true,
UseCircuitBreakers = false,
IgnoreTimeouts = false
};
_logFactory = logFactory ?? new DefaultMjolnirLogFactory();
_log = _logFactory.CreateLog<CommandInvoker>();
if (_log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for name {nameof(CommandInvoker)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_metricEvents = metricEvents ?? new IgnoringMetricEvents();
_breakerExceptionHandler = breakerExceptionHandler ?? new IgnoredExceptionHandler(new HashSet<Type>());
_circuitBreakerFactory = new CircuitBreakerFactory(
_metricEvents,
new FailurePercentageCircuitBreakerConfig(_config),
_logFactory);
_bulkheadFactory = new BulkheadFactory(
_metricEvents,
_config,
_logFactory);
var breakerInvoker = new BreakerInvoker(_circuitBreakerFactory, _metricEvents, _breakerExceptionHandler);
_bulkheadInvoker = bulkheadInvoker ?? new BulkheadInvoker(breakerInvoker, _bulkheadFactory, _metricEvents, _config);
}
public BreakerBuilder WithMetricEvents(IMetricEvents metricEvents)
{
_metricEvents = metricEvents;
return(this);
}
internal FailurePercentageCircuitBreaker(GroupKey key, ICommandMetrics metrics, IMetricEvents metricEvents, IFailurePercentageCircuitBreakerConfig config, IMjolnirLogFactory logFactory)
: this(key, new UtcSystemClock(), metrics, metricEvents, config, logFactory)
{
}
public BreakerBuilder WithMetricEvents(IMetricEvents metricEvents)
{
_metricEvents = metricEvents;
return this;
}
internal FailurePercentageCircuitBreaker(GroupKey key, ICommandMetrics metrics, IStats stats, IMetricEvents metricEvents, FailurePercentageCircuitBreakerProperties properties)
: this(key, new SystemClock(), metrics, stats, metricEvents, properties) {}
public BreakerInvoker(ICircuitBreakerFactory circuitBreakerFactory, IMetricEvents metricEvents, IBreakerExceptionHandler ignoredExceptions)
{
_circuitBreakerFactory = circuitBreakerFactory ?? throw new ArgumentNullException(nameof(circuitBreakerFactory));
_metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
_ignoredExceptions = ignoredExceptions ?? throw new ArgumentNullException(nameof(ignoredExceptions));
}
public SemaphoreBulkheadHolder(GroupKey key, IMetricEvents metricEvents)
{
if (metricEvents == null)
{
throw new ArgumentNullException("metricEvents");
}
_metricEvents = metricEvents;
// The order of things here is very intentional.
// We create the configurable value first, retrieve its current value, and then
// initialize the semaphore bulkhead. We register the change handler after that.
// That ought to help avoid a situation where we might fire a config change handler
// before we add the semaphore to the dictionary, potentially trying to add two
// entries with different values in rapid succession.
var configKey = "mjolnir.bulkhead." + key + ".maxConcurrent";
_config = new ConfigurableValue<int>(configKey, DefaultBulkheadMaxConcurrent);
var value = _config.Value;
_bulkhead = new SemaphoreBulkhead(key, value);
// On change, we'll replace the bulkhead. The assumption here is that a caller
// using the bulkhead will have kept a local reference to the bulkhead that they
// acquired a lock on, and will release the lock on that bulkhead and not one that
// has been replaced after a config change.
_config.AddChangeHandler(newLimit =>
{
if (newLimit < 0)
{
Log.ErrorFormat("Semaphore bulkhead config {0} changed to an invalid limit of {0}, the bulkhead will not be changed",
configKey,
newLimit);
return;
}
_bulkhead = new SemaphoreBulkhead(key, newLimit);
});
_timer = new GaugeTimer((source, args) =>
{
_metricEvents.BulkheadConfigGauge(_bulkhead.Name, "semaphore", _config.Value);
}, ConfigGaugeIntervalMillis);
}
