private T InnerExecute <T>(Func <T> primaryFunction, Func <IEnumerable <Exception>, T> fallbackFunction, CancellationTokenSource cancellationTokenSource)
{
if (!ConfigurationService.GetHystrixCommandEnabled())
{
return(primaryFunction.Invoke());
}
var innerExceptions = new List <Exception>();
Stopwatch userThreadStopWatch = Stopwatch.StartNew();
if (CircuitBreaker.AllowRequest())
{
CommandMetrics.IncrementConcurrentExecutionCount();
ThreadPoolMetrics.MarkThreadExecution();
Stopwatch commandStopWatch = Stopwatch.StartNew();
try
{
var result = timeoutWrapper.Execute(primaryFunction, cancellationTokenSource);
commandStopWatch.Stop();
CircuitBreaker.CloseCircuit();
CommandMetrics.MarkSuccess();
return(result);
}
catch (HystrixTimeoutException hte)
{
commandStopWatch.Stop();
CommandMetrics.MarkTimeout();
innerExceptions.Add(hte);
}
catch (Exception ex)
{
commandStopWatch.Stop();
CommandMetrics.MarkFailure();
CommandMetrics.MarkExceptionThrown();
innerExceptions.Add(ex);
}
finally
{
// track command execution time
commandStopWatch.Stop();
CommandMetrics.AddCommandExecutionTime(commandStopWatch.Elapsed.TotalMilliseconds);
CommandMetrics.DecrementConcurrentExecutionCount();
ThreadPoolMetrics.MarkThreadCompletion();
// track execution time including threading overhead
userThreadStopWatch.Stop();
CommandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds);
}
}
else
{
CommandMetrics.MarkShortCircuited();
// track execution time including threading overhead
userThreadStopWatch.Stop();
CommandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds);
}
T fallbackResult = fallbackFunction.Invoke(innerExceptions);
CommandMetrics.MarkFallbackSuccess();
return(fallbackResult);
}
private T ExecuteWithSemaphoreIsolation()
{
if (!CircuitBreaker.AllowRequest())
{
string errorMessage = "Circuit Breaker is open. Execution was short circuited.";
Log.Log(LogLevelEnum.Error, errorMessage, GetLogTagInfo().AddLogTagData("FXD303019"));
Status = CommandStatusEnum.ShortCircuited;
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.ShortCircuited);
throw new HystrixException(FailureTypeEnum.ShortCircuited, this.GetType(), Key, errorMessage);
}
if (ExecutionSemaphore.TryAcquire())
{
Stopwatch stopwatch = new Stopwatch();
try
{
stopwatch.Start();
Status = CommandStatusEnum.Started;
T result = Execute();
stopwatch.Stop();
if (stopwatch.ElapsedMilliseconds > ConfigSet.CommandTimeoutInMilliseconds)
{
Status = CommandStatusEnum.Timeout;
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Timeout);
Log.Log(LogLevelEnum.Warning, string.Format("HystrixCommand execution timeout: {0}ms.", stopwatch.ElapsedMilliseconds),
GetLogTagInfo().AddLogTagData("FXD303020"));
}
else
{
Status = CommandStatusEnum.Success;
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Success);
CircuitBreaker.MarkSuccess();
}
return(result);
}
catch (Exception ex)
{
if (ex.IsBadRequestException())
{
Status = CommandStatusEnum.Failed;
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.BadRequest);
if (ConfigSet.LogExecutionError)
{
Log.Log(LogLevelEnum.Error, "HystrixCommand request is bad.", ex, GetLogTagInfo().AddLogTagData("FXD303021"));
}
throw;
}
Status = CommandStatusEnum.Failed;
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Failed);
if (ConfigSet.LogExecutionError)
{
Log.Log(LogLevelEnum.Error, "HystrixCommand execution failed.", ex, GetLogTagInfo().AddLogTagData("FXD303022"));
}
throw;
}
finally
{
ExecutionSemaphore.Release();
stopwatch.Stop();
Metrics.MarkExecutionLatency(stopwatch.ElapsedMilliseconds);
}
}
else
{
Status = CommandStatusEnum.Rejected;
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Rejected);
string errorMessage = "HystrixCommand execution was rejected.";
Log.Log(LogLevelEnum.Error, errorMessage, GetLogTagInfo().AddLogTagData("FXD303023"));
throw new HystrixException(FailureTypeEnum.SemaphoreIsolationRejected, GetType(), Key, errorMessage);
}
}
private Task <T> ExecuteWithThreadPool()
{
TaskCompletionSource <T> tcs = new TaskCompletionSource <T>();
try
{
if (!CircuitBreaker.AllowRequest())
{
var shortCircuMsg = "Circuit Breaker is open. Execution was short circuited.";
Log.Log(LogLevelEnum.Error, shortCircuMsg, GetLogTagInfo().AddLogTagData("FXD303033"));
Status = CommandStatusEnum.ShortCircuited;
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.ShortCircuited);
throw new HystrixException(FailureTypeEnum.ShortCircuited, this.GetType(), Key, shortCircuMsg);
}
Status = CommandStatusEnum.Started;
var newTask = this.QueueWorkItem(this.Execute,
(o, e) =>
{
var t = o as CWorkItem <T>;
var needFallback = false;
try
{
if (t.IsCompleted || t.IsCanceled)
{
Metrics.MarkTotalExecutionLatency(t.ExeuteMilliseconds);
Metrics.MarkExecutionLatency(t.RealExecuteMilliseconds);
}
switch (e.Status)
{
case CTaskStatus.WaitingToRun:
break;
case CTaskStatus.Running:
Status = CommandStatusEnum.Started;
break;
case CTaskStatus.RanToCompletion:
Status = CommandStatusEnum.Success;
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Success);
CircuitBreaker.MarkSuccess();
tcs.SetResult(t.Result);
break;
case CTaskStatus.Faulted:
Status = CommandStatusEnum.Failed;
if (t.Exception.IsBadRequestException())
{
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.BadRequest);
Log.Log(LogLevelEnum.Error, "HystrixCommand request is bad.", t.Exception, GetLogTagInfo().AddLogTagData("FXD303035"));
}
else
{
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Failed);
Log.Log(LogLevelEnum.Error, "HystrixCommand execution failed.", t.Exception, GetLogTagInfo().AddLogTagData("FXD303036"));
}
needFallback = true;
break;
case CTaskStatus.Canceled:
Status = CommandStatusEnum.Timeout;
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Timeout);
Log.Log(LogLevelEnum.Warning, string.Format("timed out before executing run(), the wait time was {0} milliseconds; ", t.ExeuteMilliseconds)
, GetLogTagInfo().AddLogTagData("FXD303034"));
needFallback = true;
break;
default:
break;
}
if (needFallback)
{
if (HasFallback)
{
try
{
tcs.SetResult(ExecuteFallback());
}
catch (Exception ex)
{
throw(ex);
}
}
else if (e.Status == CTaskStatus.Faulted)
{
throw(t.Exception);
}
else if (e.Status == CTaskStatus.Canceled)
{
throw(new HystrixException(FailureTypeEnum.ExecutionTimeout, this.GetType(), Key,
string.Format("timed out before executing run(), maxt waiting time was {0} milliseconds,the task waiting time was {1} milliseconds",
ConfigSet.CommandTimeoutInMilliseconds, t.ExeuteMilliseconds),
t.Exception, new Exception("no fallback found")));
}
}
}
catch (Exception ex)
{
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.ExceptionThrown);
tcs.TrySetException(ex);
}
});
}
catch (Exception ex)
{
if (ex is HystrixException)
{
if (((HystrixException)ex).FailureType == FailureTypeEnum.ThreadIsolationRejected)
{
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Rejected);
Log.Log(LogLevelEnum.Error, "HystrixCommand execution rejected.", ex,
GetLogTagInfo().AddLogTagData("FXD303037"));
Status = CommandStatusEnum.Rejected;
}
}
if (HasFallback)
{
tcs.Task.ContinueWith(t =>
{
try
{
var x = t.Exception;
}
catch
{
}
});
return(GetFallBack());
}
Status = CommandStatusEnum.Failed;
Metrics.MarkExecutionEvent(CommandExecutionEventEnum.ExceptionThrown);
tcs.TrySetException(ex);
}
return(tcs.Task);
}
