private Task Notify(ICircuitRequest request, ICircuitResponse response, TimeSpan executionTime)
{
if (_listeners == null || !_listeners.Any())
{
return(Task.CompletedTask);
}
try
{
var data = new CircuitStatusData
{
RequestId = request.RequestId,
CircuitKey = request.CircuitKey,
Context = request.Context,
ExecutionTime = executionTime,
Status = response.Status
};
var tasks = _listeners.Select(x => x.Notify(data)).ToArray();
return(Task.WhenAll(tasks));
}
catch (Exception e)
{
CircuitBreakerLog.LogError(e, e.Message);
}
return(Task.CompletedTask);
}
public async Task <IAsyncPolicy> Get(ICircuitRequest request)
{
IAsyncPolicy result;
var key = $"{request.CircuitKey}:{request.Retry}:{request.Timeout}";
if (_source.TryGetValue(key, out result))
{
return(result);
}
await _semaphore.WaitAsync();
try
{
if (_source.TryGetValue(key, out result))
{
return(result);
}
CircuitBreakerLog.LogTrace("Loading policy from provider...");
PolicySettings settings = null;
if (_settingsProvider != null)
{
foreach (var provider in _settingsProvider)
{
settings = await provider.Get(request);
if (settings != null)
{
break;
}
}
}
var shouldReloadSettings = settings?.ShouldReload ?? false;
var policy = BuildPolicy(request, settings);
if (!shouldReloadSettings)
{
_source.TryAdd(key, policy);
}
else
{
CircuitBreakerLog.LogWarning("Policy need to reload from provider next time as requested by provider. So escape storing the policy to reuse");
}
return(policy);
}
finally
{
_semaphore.Release();
}
}
private IAsyncPolicy BuildPolicy(ICircuitRequest request, PolicySettings settings)
{
settings = settings ?? new PolicySettings();
var maxParallelization = settings.MaxParallelization ?? 100;
var maxQueingActions = settings.MaxQueingActions ?? 10;
var timeout = AppliedTimespan(request.Timeout, settings.Timeout, TimeSpan.FromMinutes(10));
var retry = request.Retry ?? settings.Retry ?? 0;
var failurePercentThresholdPercent = EmptyAlternative(settings.FailurePercentThreshold, 50);
var failureThreshold = failurePercentThresholdPercent / 100d;
var samplingDuration = AppliedTimespan(settings.SamplingDuration, TimeSpan.FromMilliseconds(1000));
var minThroughput = settings.MinimumThroughput ?? 5;
var durationOfBreak = AppliedTimespan(settings.DurationOfBreak, TimeSpan.FromMilliseconds(500));
if (CircuitBreakerLog.IsTraceEnabled)
{
CircuitBreakerLog.LogTrace($@"RequestId:{request.RequestId}
|AppName:{request.Context.GetAppName()}
|ServiceName:{request.Context.GetServiceName()}
|CircuitKey:{request.CircuitKey}
|Request.Retry:{request.Retry}
|Request.Timeout:{request.Timeout}
|Settings.Retry:{settings.Retry}
|Settings.Timeout:{settings.Timeout}
|Settings.FailureThreshold:{failureThreshold}
|Settings.SamplingDuration:{samplingDuration}
|Settings.MinThroughput:{minThroughput}
|Settings.DurationOfBreak:{durationOfBreak}
|Settings.MaxParallelization:{maxParallelization}
|Settings.MaxQueingActions:{maxQueingActions}");
}
var bulkHead = Policy.BulkheadAsync(maxParallelization, maxQueingActions);
var timeoutPolicy = Policy.TimeoutAsync(timeout, Polly.Timeout.TimeoutStrategy.Pessimistic);
var retryPolicy = Policy.Handle <Exception>().RetryAsync(retry);
var circuit = Policy.Handle <Exception>()
.AdvancedCircuitBreakerAsync(failureThreshold,
samplingDuration,
minThroughput,
durationOfBreak);
return(circuit.WrapAsync(bulkHead.WrapAsync(retryPolicy.WrapAsync(timeoutPolicy))));
}
private async Task <TResponse> HandleError <TResponse>(ICircuitRequest request, Exception e) where TResponse : ICircuitResponse, new()
{
CircuitBreakerLog.LogError(e, e.Message);
var response = new TResponse {
Status = CircuitStatus.Failed
};
await Notify(request, response, TimeSpan.Zero);
return(response);
}
private TResponse BuildResponse <TResponse>(ICircuitRequest request, Polly.OutcomeType outcome, Exception finalException, TimeSpan executionTime) where TResponse : ICircuitResponse, new()
{
var result = new TResponse();
if (outcome == Polly.OutcomeType.Failure)
{
if (finalException is Polly.Timeout.TimeoutRejectedException || finalException is TimeoutException)
{
Trace(CircuitStatus.Timeout, request, finalException, executionTime);
result.Status = CircuitStatus.Timeout;
}
else if (finalException is Polly.CircuitBreaker.BrokenCircuitException)
{
Trace(CircuitStatus.Broken, request, finalException, executionTime);
result.Status = CircuitStatus.Broken;
}
else
{
CircuitBreakerLog.LogError(finalException, finalException.Message);
Trace(CircuitStatus.Failed, request, finalException, executionTime);
result.Status = CircuitStatus.Failed;
}
}
else
{
Trace(CircuitStatus.Succeed, request, finalException, executionTime);
result.Status = CircuitStatus.Succeed;
}
return(result);
}
private void Trace(CircuitStatus status, ICircuitRequest request, Exception exception, TimeSpan executiontime)
{
CircuitBreakerLog.LogTrace($"RequestId:{request.RequestId}|CircuitKey:{request.CircuitKey}|Status:{status}|ExecutionTime:{executiontime.TotalMilliseconds}ms|Msg:{exception?.Message}");
}
