private InformativeCancellationToken GetCancellationTokenForCommand(BaseCommand command, long? invocationTimeout = null)
{
if (IgnoreCancellation())
{
return InformativeCancellationToken.ForIgnored();
}
var timeout = command.DetermineTimeout(_config, invocationTimeout);
return InformativeCancellationToken.ForTimeout(timeout);
}
private InformativeCancellationToken GetCancellationTokenForCommand(CancellationToken ct)
{
if (IgnoreCancellation())
{
return InformativeCancellationToken.ForIgnored();
}
if (!IsEnabled())
{
return InformativeCancellationToken.ForDisabled();
}
return InformativeCancellationToken.ForOverridingToken(ct);
}
private static void AttachCommandExceptionData(BaseCommand command, Exception exception, CommandCompletionStatus status, InformativeCancellationToken ct)
{
// Use namespaced keys here to avoid clobbering data that the application might've
// already attached (or will attach later).
exception.Data["MjolnirCommand"] = command.Name;
exception.Data["MjolnirStatus"] = status;
exception.Data["MjolnirBreaker"] = command.BreakerKey.Name;
exception.Data["MjolnirBulkhead"] = command.BulkheadKey.Name;
exception.Data["MjolnirTimeoutMillis"] = ct.DescriptionForLog;
exception.Data["MjolnirExecuteMillis"] = command.ExecutionTimeMillis;
}
private static CommandCompletionStatus GetCompletionStatus(Exception exception, InformativeCancellationToken ct)
{
if (IsCancellationException(exception))
{
// If the timeout cancellationTokenSource was cancelled and we got an TaskCancelledException here then this means the call actually timed out.
// Otherwise an TaskCancelledException would have been raised if a user CancellationToken was passed through to the method call, and was explicitly
// cancelled from the client side.
if (ct.Timeout.HasValue && ct.Token.IsCancellationRequested)
{
return CommandCompletionStatus.TimedOut;
}
return CommandCompletionStatus.Canceled;
}
if (exception is CircuitBreakerRejectedException || exception is BulkheadRejectedException)
{
return CommandCompletionStatus.Rejected;
}
return CommandCompletionStatus.Faulted;
}
private TResult Invoke<TResult>(SyncCommand<TResult> command, OnFailure failureAction, InformativeCancellationToken ct)
{
var stopwatch = Stopwatch.StartNew();
var logName = $"Hudl.Mjolnir.Command.{command.Name}";
var status = CommandCompletionStatus.RanToCompletion;
try
{
_log.Debug($"[{logName}] Invoke Command={command.Name} Breaker={command.BreakerKey} Bulkhead={command.BulkheadKey} Timeout={ct.DescriptionForLog}");
// If we've already timed out or been canceled, skip execution altogether.
ct.Token.ThrowIfCancellationRequested();
return _bulkheadInvoker.ExecuteWithBulkhead(command, ct.Token);
}
catch (Exception e)
{
status = GetCompletionStatus(e, ct);
AttachCommandExceptionData(command, e, status, ct);
throw;
}
finally
{
stopwatch.Stop();
_metricEvents.CommandInvoked(command.Name, stopwatch.Elapsed.TotalMilliseconds, command.ExecutionTimeMillis, status.ToString(), failureAction.ToString().ToLowerInvariant());
}
}
private CommandResult<TResult> InvokeAndWrap<TResult>(SyncCommand<TResult> command, InformativeCancellationToken ct)
{
// Even though we're in a "Return" method, multiple invokes are a bug on the calling
// side, hence the possible exception here for visibility so the caller can fix.
EnsureSingleInvoke(command);
try
{
TResult result;
if (!IsEnabled())
{
result = command.Execute(CancellationToken.None);
}
else
{
result = Invoke(command, OnFailure.Return, ct);
}
return new CommandResult<TResult>(result);
}
catch (Exception e)
{
return new CommandResult<TResult>(default(TResult), e);
}
}