public CommandInvoker(
MjolnirConfiguration config = null,
IMjolnirLogFactory logFactory = null,
IMetricEvents metricEvents = null,
IBreakerExceptionHandler breakerExceptionHandler = null)
: this(config, logFactory, metricEvents, breakerExceptionHandler, null)
{ }
public CircuitBreakerFactory(IMetricEvents metricEvents, IFailurePercentageCircuitBreakerConfig breakerConfig, IMjolnirLogFactory logFactory)
{
_metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
_breakerConfig = breakerConfig ?? throw new ArgumentNullException(nameof(breakerConfig));
_logFactory = logFactory ?? throw new ArgumentNullException(nameof(logFactory));
_log = logFactory.CreateLog <FailurePercentageCircuitBreaker>();
if (_log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(CircuitBreakerFactory)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_timer = new GaugeTimer(state =>
{
try
{
var keys = _circuitBreakers.Keys;
foreach (var key in keys)
{
if (_circuitBreakers.TryGetValue(key, out Lazy <FailurePercentageCircuitBreaker> lazy) && lazy.IsValueCreated)
{
var breaker = lazy.Value;
_metricEvents.BreakerGauge(
breaker.Name,
_breakerConfig.GetMinimumOperations(key),
_breakerConfig.GetWindowMillis(key),
_breakerConfig.GetThresholdPercentage(key),
_breakerConfig.GetTrippedDurationMillis(key),
_breakerConfig.GetForceTripped(key),
_breakerConfig.GetForceFixed(key),
breaker.IsTripped(),
breaker.Metrics.SuccessCount,
breaker.Metrics.FailureCount);
}
}
}
catch (Exception e)
{
_log.Error($"Error sending {nameof(IMetricEvents.BreakerGauge)} metric event", e);
}
});
}
// ReSharper restore PrivateFieldCanBeConvertedToLocalVariable
public BulkheadFactory(IMetricEvents metricEvents, MjolnirConfiguration config, IMjolnirLogFactory logFactory)
{
// No null checks on parameters; we don't use them, we're just passing them through to
// the objects we're creating.
_metricEvents = metricEvents;
_config = config;
_logFactory = logFactory ?? throw new ArgumentNullException(nameof(logFactory));
var log = logFactory.CreateLog <BulkheadFactory>();
if (log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(BulkheadFactory)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_timer = new GaugeTimer(state =>
{
try
{
var keys = _bulkheads.Keys;
foreach (var key in keys)
{
if (_bulkheads.TryGetValue(key, out Lazy <SemaphoreBulkheadHolder> holder) && holder.IsValueCreated)
{
var bulkhead = holder.Value.Bulkhead;
_metricEvents.BulkheadGauge(bulkhead.Name, "semaphore", _config.GetBulkheadConfiguration(key.Name).MaxConcurrent, bulkhead.CountAvailable);
}
}
}
catch (Exception e)
{
log.Error($"Error sending {nameof(IMetricEvents.BulkheadGauge)} metric event", e);
}
});
}
// Internal constructor with a few extra arguments used by tests to inject mocks.
internal CommandInvoker(
MjolnirConfiguration config = null,
IMjolnirLogFactory logFactory = null,
IMetricEvents metricEvents = null,
IBreakerExceptionHandler breakerExceptionHandler = null,
IBulkheadInvoker bulkheadInvoker = null)
{
_config = config ?? new MjolnirConfiguration
{
IsEnabled = true,
UseCircuitBreakers = false,
IgnoreTimeouts = false
};
_logFactory = logFactory ?? new DefaultMjolnirLogFactory();
_log = _logFactory.CreateLog<CommandInvoker>();
if (_log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for name {nameof(CommandInvoker)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_metricEvents = metricEvents ?? new IgnoringMetricEvents();
_breakerExceptionHandler = breakerExceptionHandler ?? new IgnoredExceptionHandler(new HashSet<Type>());
_circuitBreakerFactory = new CircuitBreakerFactory(
_metricEvents,
new FailurePercentageCircuitBreakerConfig(_config),
_logFactory);
_bulkheadFactory = new BulkheadFactory(
_metricEvents,
_config,
_logFactory);
var breakerInvoker = new BreakerInvoker(_circuitBreakerFactory, _metricEvents, _breakerExceptionHandler);
_bulkheadInvoker = bulkheadInvoker ?? new BulkheadInvoker(breakerInvoker, _bulkheadFactory, _metricEvents, _config);
}
public SemaphoreBulkheadHolder(GroupKey key, IMetricEvents metricEvents, MjolnirConfiguration config, IMjolnirLogFactory logFactory)
{
_key = key;
_metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
_config = config ?? throw new ArgumentNullException(nameof(config));
if (logFactory == null)
{
throw new ArgumentNullException(nameof(logFactory));
}
_log = logFactory.CreateLog <SemaphoreBulkheadHolder>();
if (_log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(SemaphoreBulkheadHolder)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
// The order of things here is very intentional.
// We get the MaxConcurrent value first and then initialize the semaphore bulkhead.
// The change handler is registered after that. The order ought to help avoid a
// situation where we might fire a config change handler before we add the
// semaphore to the dictionary, potentially trying to add two entries with
// different values in rapid succession.
var value = _config.GetBulkheadConfiguration(key.Name).MaxConcurrent;
_bulkhead = new SemaphoreBulkhead(_key, value);
// On change, we'll replace the bulkhead. The assumption here is that a caller
// using the bulkhead will have kept a local reference to the bulkhead that they
// acquired a lock on, and will release the lock on that bulkhead and not one that
// has been replaced after a config change.
_config.OnConfigurationChanged(c => c.GetBulkheadConfiguration(key.Name).MaxConcurrent, UpdateMaxConcurrent);
}
internal StandardCommandMetrics(GroupKey key, IFailurePercentageCircuitBreakerConfig config, IClock clock, IMjolnirLogFactory logFactory)
{
_config = config ?? throw new ArgumentNullException(nameof(config));
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
_key = key;
_resettingNumbersBucket = new ResettingNumbersBucket(_key, _clock, _config, logFactory);
}
internal StandardCommandMetrics(GroupKey key, IFailurePercentageCircuitBreakerConfig config, IMjolnirLogFactory logFactory)
: this(key, config, new UtcSystemClock(), logFactory)
{
}
internal FailurePercentageCircuitBreaker(GroupKey key, IClock clock, ICommandMetrics metrics, IMetricEvents metricEvents, IFailurePercentageCircuitBreakerConfig config, IMjolnirLogFactory logFactory)
{
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_clock = clock ?? throw new ArgumentNullException(nameof(config));
_config = config ?? throw new ArgumentNullException(nameof(config));
_metricEvents = metricEvents ?? throw new ArgumentNullException(nameof(metricEvents));
if (logFactory == null)
{
throw new ArgumentNullException(nameof(logFactory));
}
_key = key;
_log = logFactory.CreateLog <FailurePercentageCircuitBreaker>();
if (_log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(FailurePercentageCircuitBreaker)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_state = State.Fixed; // Start off assuming everything's fixed.
_lastTrippedTimestamp = 0; // 0 is fine since it'll be far less than the first compared value.
}
internal FailurePercentageCircuitBreaker(GroupKey key, ICommandMetrics metrics, IMetricEvents metricEvents, IFailurePercentageCircuitBreakerConfig config, IMjolnirLogFactory logFactory)
: this(key, new UtcSystemClock(), metrics, metricEvents, config, logFactory)
{
}
internal ResettingNumbersBucket(GroupKey key, IClock clock, IFailurePercentageCircuitBreakerConfig config, IMjolnirLogFactory logFactory)
{
_key = key;
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
_config = config ?? throw new ArgumentNullException(nameof(config));
if (logFactory == null)
{
throw new ArgumentNullException(nameof(logFactory));
}
_log = logFactory.CreateLog <ResettingNumbersBucket>();
if (_log == null)
{
throw new InvalidOperationException($"{nameof(IMjolnirLogFactory)} implementation returned null from {nameof(IMjolnirLogFactory.CreateLog)} for type {typeof(ResettingNumbersBucket)}, please make sure the implementation returns a non-null log for all calls to {nameof(IMjolnirLogFactory.CreateLog)}");
}
_counters = CreateCounters();
_lastResetAtTime = clock.GetMillisecondTimestamp();
}
internal ResettingNumbersBucket(GroupKey key, IFailurePercentageCircuitBreakerConfig config, IMjolnirLogFactory logFactory) : this(key, new UtcSystemClock(), config, logFactory)
{
}