private TResult Invoke<TResult>(SyncCommand<TResult> command, OnFailure failureAction, InformativeCancellationToken ct)
{
var stopwatch = Stopwatch.StartNew();
var logName = $"Hudl.Mjolnir.Command.{command.Name}";
var status = CommandCompletionStatus.RanToCompletion;
try
{
_log.Debug($"[{logName}] Invoke Command={command.Name} Breaker={command.BreakerKey} Bulkhead={command.BulkheadKey} Timeout={ct.DescriptionForLog}");
// If we've already timed out or been canceled, skip execution altogether.
ct.Token.ThrowIfCancellationRequested();
return _bulkheadInvoker.ExecuteWithBulkhead(command, ct.Token);
}
catch (Exception e)
{
status = GetCompletionStatus(e, ct);
AttachCommandExceptionData(command, e, status, ct);
throw;
}
finally
{
stopwatch.Stop();
_metricEvents.CommandInvoked(command.Name, stopwatch.Elapsed.TotalMilliseconds, command.ExecutionTimeMillis, status.ToString(), failureAction.ToString().ToLowerInvariant());
}
}
private CommandResult<TResult> InvokeAndWrap<TResult>(SyncCommand<TResult> command, InformativeCancellationToken ct)
{
// Even though we're in a "Return" method, multiple invokes are a bug on the calling
// side, hence the possible exception here for visibility so the caller can fix.
EnsureSingleInvoke(command);
try
{
TResult result;
if (!IsEnabled())
{
result = command.Execute(CancellationToken.None);
}
else
{
result = Invoke(command, OnFailure.Return, ct);
}
return new CommandResult<TResult>(result);
}
catch (Exception e)
{
return new CommandResult<TResult>(default(TResult), e);
}
}
public TResult ExecuteWithBreaker <TResult>(SyncCommand <TResult> command, CancellationToken ct)
{
var breaker = _circuitBreakerFactory.GetCircuitBreaker(command.BreakerKey);
if (!breaker.IsAllowing())
{
_metricEvents.RejectedByBreaker(breaker.Name, command.Name);
throw new CircuitBreakerRejectedException();
}
TResult result;
var success = true;
var breakerStopwatch = Stopwatch.StartNew();
var executionStopwatch = Stopwatch.StartNew();
try
{
result = command.Execute(ct);
executionStopwatch.Stop();
breaker.MarkSuccess(breakerStopwatch.ElapsedMilliseconds);
breaker.Metrics.MarkCommandSuccess();
}
catch (Exception e)
{
executionStopwatch.Stop();
success = false;
if (_ignoredExceptions.IsExceptionIgnored(e.GetType()))
{
success = true;
breaker.MarkSuccess(breakerStopwatch.ElapsedMilliseconds);
breaker.Metrics.MarkCommandSuccess();
}
else
{
breaker.Metrics.MarkCommandFailure();
}
throw;
}
finally
{
command.ExecutionTimeMillis = executionStopwatch.Elapsed.TotalMilliseconds;
if (success)
{
_metricEvents.BreakerSuccessCount(breaker.Name, command.Name);
}
else
{
_metricEvents.BreakerFailureCount(breaker.Name, command.Name);
}
}
return(result);
}
public TResult InvokeThrow<TResult>(SyncCommand<TResult> command, CancellationToken ct)
{
EnsureSingleInvoke(command);
if (!IsEnabled())
{
return command.Execute(CancellationToken.None);
}
var token = GetCancellationTokenForCommand(ct);
return Invoke(command, OnFailure.Throw, token);
}
public TResult ExecuteWithBulkhead <TResult>(SyncCommand <TResult> command, CancellationToken ct)
{
var bulkhead = _bulkheadFactory.GetBulkhead(command.BulkheadKey);
if (!bulkhead.TryEnter())
{
_metricEvents.RejectedByBulkhead(bulkhead.Name, command.Name);
throw new BulkheadRejectedException();
}
_metricEvents.EnterBulkhead(bulkhead.Name, command.Name);
// This stopwatch should begin stopped (hence the constructor instead of the usual
// Stopwatch.StartNew(). We'll only use it if we aren't using circuit breakers.
var stopwatch = new Stopwatch();
// If circuit breakers are enabled, the execution will happen down in the circuit
// breaker invoker. If they're disabled, it'll happen here. Since we want an accurate
// timing of the method execution, we'll use this to track where the execution happens
// and then set ExecutionTimeMillis in the finally block conditionally.
var executedHere = false;
try
{
if (_config.UseCircuitBreakers)
{
return(_breakerInvoker.ExecuteWithBreaker(command, ct));
}
executedHere = true;
stopwatch.Start();
return(command.Execute(ct));
}
finally
{
bulkhead.Release();
_metricEvents.LeaveBulkhead(bulkhead.Name, command.Name);
// If not executed here, the circuit breaker invoker will set the execution time.
if (executedHere)
{
stopwatch.Stop();
command.ExecutionTimeMillis = stopwatch.Elapsed.TotalMilliseconds;
}
}
}
public CommandResult<TResult> InvokeReturn<TResult>(SyncCommand<TResult> command, CancellationToken ct)
{
var token = GetCancellationTokenForCommand(ct);
return InvokeAndWrap(command, token);
}
public CommandResult<TResult> InvokeReturn<TResult>(SyncCommand<TResult> command, long timeoutMillis)
{
var token = GetCancellationTokenForCommand(command, timeoutMillis);
return InvokeAndWrap(command, token);
}