private T InnerExecute <T>(Func <T> primaryFunction, Func <IEnumerable <Exception>, T> fallbackFunction, CancellationTokenSource cancellationTokenSource) { if (!ConfigurationService.GetHystrixCommandEnabled()) { return(primaryFunction.Invoke()); } var innerExceptions = new List <Exception>(); Stopwatch userThreadStopWatch = Stopwatch.StartNew(); if (CircuitBreaker.AllowRequest()) { CommandMetrics.IncrementConcurrentExecutionCount(); ThreadPoolMetrics.MarkThreadExecution(); Stopwatch commandStopWatch = Stopwatch.StartNew(); try { var result = timeoutWrapper.Execute(primaryFunction, cancellationTokenSource); commandStopWatch.Stop(); CircuitBreaker.CloseCircuit(); CommandMetrics.MarkSuccess(); return(result); } catch (HystrixTimeoutException hte) { commandStopWatch.Stop(); CommandMetrics.MarkTimeout(); innerExceptions.Add(hte); } catch (Exception ex) { commandStopWatch.Stop(); CommandMetrics.MarkFailure(); CommandMetrics.MarkExceptionThrown(); innerExceptions.Add(ex); } finally { // track command execution time commandStopWatch.Stop(); CommandMetrics.AddCommandExecutionTime(commandStopWatch.Elapsed.TotalMilliseconds); CommandMetrics.DecrementConcurrentExecutionCount(); ThreadPoolMetrics.MarkThreadCompletion(); // track execution time including threading overhead userThreadStopWatch.Stop(); CommandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds); } } else { CommandMetrics.MarkShortCircuited(); // track execution time including threading overhead userThreadStopWatch.Stop(); CommandMetrics.AddUserThreadExecutionTime(userThreadStopWatch.Elapsed.TotalMilliseconds); } T fallbackResult = fallbackFunction.Invoke(innerExceptions); CommandMetrics.MarkFallbackSuccess(); return(fallbackResult); }
private T ExecuteWithSemaphoreIsolation() { if (!CircuitBreaker.AllowRequest()) { string errorMessage = "Circuit Breaker is open. Execution was short circuited."; Log.Log(LogLevelEnum.Error, errorMessage, GetLogTagInfo().AddLogTagData("FXD303019")); Status = CommandStatusEnum.ShortCircuited; Metrics.MarkExecutionEvent(CommandExecutionEventEnum.ShortCircuited); throw new HystrixException(FailureTypeEnum.ShortCircuited, this.GetType(), Key, errorMessage); } if (ExecutionSemaphore.TryAcquire()) { Stopwatch stopwatch = new Stopwatch(); try { stopwatch.Start(); Status = CommandStatusEnum.Started; T result = Execute(); stopwatch.Stop(); if (stopwatch.ElapsedMilliseconds > ConfigSet.CommandTimeoutInMilliseconds) { Status = CommandStatusEnum.Timeout; Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Timeout); Log.Log(LogLevelEnum.Warning, string.Format("HystrixCommand execution timeout: {0}ms.", stopwatch.ElapsedMilliseconds), GetLogTagInfo().AddLogTagData("FXD303020")); } else { Status = CommandStatusEnum.Success; Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Success); CircuitBreaker.MarkSuccess(); } return(result); } catch (Exception ex) { if (ex.IsBadRequestException()) { Status = CommandStatusEnum.Failed; Metrics.MarkExecutionEvent(CommandExecutionEventEnum.BadRequest); if (ConfigSet.LogExecutionError) { Log.Log(LogLevelEnum.Error, "HystrixCommand request is bad.", ex, GetLogTagInfo().AddLogTagData("FXD303021")); } throw; } Status = CommandStatusEnum.Failed; Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Failed); if (ConfigSet.LogExecutionError) { Log.Log(LogLevelEnum.Error, "HystrixCommand execution failed.", ex, GetLogTagInfo().AddLogTagData("FXD303022")); } throw; } finally { ExecutionSemaphore.Release(); stopwatch.Stop(); Metrics.MarkExecutionLatency(stopwatch.ElapsedMilliseconds); } } else { Status = CommandStatusEnum.Rejected; Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Rejected); string errorMessage = "HystrixCommand execution was rejected."; Log.Log(LogLevelEnum.Error, errorMessage, GetLogTagInfo().AddLogTagData("FXD303023")); throw new HystrixException(FailureTypeEnum.SemaphoreIsolationRejected, GetType(), Key, errorMessage); } }
private Task <T> ExecuteWithThreadPool() { TaskCompletionSource <T> tcs = new TaskCompletionSource <T>(); try { if (!CircuitBreaker.AllowRequest()) { var shortCircuMsg = "Circuit Breaker is open. Execution was short circuited."; Log.Log(LogLevelEnum.Error, shortCircuMsg, GetLogTagInfo().AddLogTagData("FXD303033")); Status = CommandStatusEnum.ShortCircuited; Metrics.MarkExecutionEvent(CommandExecutionEventEnum.ShortCircuited); throw new HystrixException(FailureTypeEnum.ShortCircuited, this.GetType(), Key, shortCircuMsg); } Status = CommandStatusEnum.Started; var newTask = this.QueueWorkItem(this.Execute, (o, e) => { var t = o as CWorkItem <T>; var needFallback = false; try { if (t.IsCompleted || t.IsCanceled) { Metrics.MarkTotalExecutionLatency(t.ExeuteMilliseconds); Metrics.MarkExecutionLatency(t.RealExecuteMilliseconds); } switch (e.Status) { case CTaskStatus.WaitingToRun: break; case CTaskStatus.Running: Status = CommandStatusEnum.Started; break; case CTaskStatus.RanToCompletion: Status = CommandStatusEnum.Success; Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Success); CircuitBreaker.MarkSuccess(); tcs.SetResult(t.Result); break; case CTaskStatus.Faulted: Status = CommandStatusEnum.Failed; if (t.Exception.IsBadRequestException()) { Metrics.MarkExecutionEvent(CommandExecutionEventEnum.BadRequest); Log.Log(LogLevelEnum.Error, "HystrixCommand request is bad.", t.Exception, GetLogTagInfo().AddLogTagData("FXD303035")); } else { Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Failed); Log.Log(LogLevelEnum.Error, "HystrixCommand execution failed.", t.Exception, GetLogTagInfo().AddLogTagData("FXD303036")); } needFallback = true; break; case CTaskStatus.Canceled: Status = CommandStatusEnum.Timeout; Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Timeout); Log.Log(LogLevelEnum.Warning, string.Format("timed out before executing run(), the wait time was {0} milliseconds; ", t.ExeuteMilliseconds) , GetLogTagInfo().AddLogTagData("FXD303034")); needFallback = true; break; default: break; } if (needFallback) { if (HasFallback) { try { tcs.SetResult(ExecuteFallback()); } catch (Exception ex) { throw(ex); } } else if (e.Status == CTaskStatus.Faulted) { throw(t.Exception); } else if (e.Status == CTaskStatus.Canceled) { throw(new HystrixException(FailureTypeEnum.ExecutionTimeout, this.GetType(), Key, string.Format("timed out before executing run(), maxt waiting time was {0} milliseconds,the task waiting time was {1} milliseconds", ConfigSet.CommandTimeoutInMilliseconds, t.ExeuteMilliseconds), t.Exception, new Exception("no fallback found"))); } } } catch (Exception ex) { Metrics.MarkExecutionEvent(CommandExecutionEventEnum.ExceptionThrown); tcs.TrySetException(ex); } }); } catch (Exception ex) { if (ex is HystrixException) { if (((HystrixException)ex).FailureType == FailureTypeEnum.ThreadIsolationRejected) { Metrics.MarkExecutionEvent(CommandExecutionEventEnum.Rejected); Log.Log(LogLevelEnum.Error, "HystrixCommand execution rejected.", ex, GetLogTagInfo().AddLogTagData("FXD303037")); Status = CommandStatusEnum.Rejected; } } if (HasFallback) { tcs.Task.ContinueWith(t => { try { var x = t.Exception; } catch { } }); return(GetFallBack()); } Status = CommandStatusEnum.Failed; Metrics.MarkExecutionEvent(CommandExecutionEventEnum.ExceptionThrown); tcs.TrySetException(ex); } return(tcs.Task); }