private Task Notify(ICircuitRequest request, ICircuitResponse response, TimeSpan executionTime)
{
if (_listeners == null || !_listeners.Any())
{
return(Task.CompletedTask);
}
try
{
var data = new CircuitStatusData
{
RequestId = request.RequestId,
CircuitKey = request.CircuitKey,
Context = request.Context,
ExecutionTime = executionTime,
Status = response.Status
};
var tasks = _listeners.Select(x => x.Notify(data)).ToArray();
return(Task.WhenAll(tasks));
}
catch (Exception e)
{
CircuitBreakerLog.LogError(e, e.Message);
}
return(Task.CompletedTask);
}
public async Task <ICircuitResponse <T> > ExecuteAsync <T>(ICircuitRequest request, Func <ICircuitRequest, Task <T> > funcAsync)
{
if (string.IsNullOrWhiteSpace(request.CircuitKey))
{
throw new ArgumentNullException(nameof(request.CircuitKey));
}
if (funcAsync == null)
{
throw new ArgumentNullException(nameof(funcAsync));
}
var policy = await _policyProvider.Get(request);
T value = default(T);
TimeSpan executionTime = TimeSpan.Zero;
var policyResult = await policy.ExecuteAndCaptureAsync(async() => {
var sw = Stopwatch.StartNew();
value = await funcAsync(request);
sw.Stop();
executionTime = sw.Elapsed;
});
var result = BuildResponse <CircuitResponse <T> >(request, policyResult.Outcome, policyResult.FinalException, executionTime);
result.Value = value;
await Notify(request, result, executionTime);
return(result);
}
public async Task <IAsyncPolicy> Get(ICircuitRequest request)
{
IAsyncPolicy result;
var key = $"{request.CircuitKey}:{request.Retry}:{request.Timeout}";
if (_source.TryGetValue(key, out result))
{
return(result);
}
await _semaphore.WaitAsync();
try
{
if (_source.TryGetValue(key, out result))
{
return(result);
}
CircuitBreakerLog.LogTrace("Loading policy from provider...");
PolicySettings settings = null;
if (_settingsProvider != null)
{
foreach (var provider in _settingsProvider)
{
settings = await provider.Get(request);
if (settings != null)
{
break;
}
}
}
var shouldReloadSettings = settings?.ShouldReload ?? false;
var policy = BuildPolicy(request, settings);
if (!shouldReloadSettings)
{
_source.TryAdd(key, policy);
}
else
{
CircuitBreakerLog.LogWarning("Policy need to reload from provider next time as requested by provider. So escape storing the policy to reuse");
}
return(policy);
}
finally
{
_semaphore.Release();
}
}
private static async Task Execute(ICircuitBreaker cb, ICircuitRequest request)
{
var result = await cb.ExecuteAsync(request, cxt => DoSomething(cxt));
Console.WriteLine(result.Value);
WriteStatus(result);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("******");
Console.WriteLine();
}
private int?GetInt(ICircuitRequest request, string key)
{
var strValue = request.Context.Get <string>(key);
if (string.IsNullOrWhiteSpace(strValue))
{
return(null);
}
return(int.TryParse(strValue, out var result) ? result : (int?)null);
}
private IAsyncPolicy BuildPolicy(ICircuitRequest request, PolicySettings settings)
{
settings = settings ?? new PolicySettings();
var maxParallelization = settings.MaxParallelization ?? 100;
var maxQueingActions = settings.MaxQueingActions ?? 10;
var timeout = AppliedTimespan(request.Timeout, settings.Timeout, TimeSpan.FromMinutes(10));
var retry = request.Retry ?? settings.Retry ?? 0;
var failurePercentThresholdPercent = EmptyAlternative(settings.FailurePercentThreshold, 50);
var failureThreshold = failurePercentThresholdPercent / 100d;
var samplingDuration = AppliedTimespan(settings.SamplingDuration, TimeSpan.FromMilliseconds(1000));
var minThroughput = settings.MinimumThroughput ?? 5;
var durationOfBreak = AppliedTimespan(settings.DurationOfBreak, TimeSpan.FromMilliseconds(500));
if (CircuitBreakerLog.IsTraceEnabled)
{
CircuitBreakerLog.LogTrace($@"RequestId:{request.RequestId}
|AppName:{request.Context.GetAppName()}
|ServiceName:{request.Context.GetServiceName()}
|CircuitKey:{request.CircuitKey}
|Request.Retry:{request.Retry}
|Request.Timeout:{request.Timeout}
|Settings.Retry:{settings.Retry}
|Settings.Timeout:{settings.Timeout}
|Settings.FailureThreshold:{failureThreshold}
|Settings.SamplingDuration:{samplingDuration}
|Settings.MinThroughput:{minThroughput}
|Settings.DurationOfBreak:{durationOfBreak}
|Settings.MaxParallelization:{maxParallelization}
|Settings.MaxQueingActions:{maxQueingActions}");
}
var bulkHead = Policy.BulkheadAsync(maxParallelization, maxQueingActions);
var timeoutPolicy = Policy.TimeoutAsync(timeout, Polly.Timeout.TimeoutStrategy.Pessimistic);
var retryPolicy = Policy.Handle <Exception>().RetryAsync(retry);
var circuit = Policy.Handle <Exception>()
.AdvancedCircuitBreakerAsync(failureThreshold,
samplingDuration,
minThroughput,
durationOfBreak);
return(circuit.WrapAsync(bulkHead.WrapAsync(retryPolicy.WrapAsync(timeoutPolicy))));
}
private async Task <TResponse> HandleError <TResponse>(ICircuitRequest request, Exception e) where TResponse : ICircuitResponse, new()
{
CircuitBreakerLog.LogError(e, e.Message);
var response = new TResponse {
Status = CircuitStatus.Failed
};
await Notify(request, response, TimeSpan.Zero);
return(response);
}
private static async Task <string> DoSomething(ICircuitRequest request)
{
//Console.WriteLine($"Start...");
//var toss = _rnd.Next(1, 5);
//if (toss > 2) throw new Exception("Toss failed");
var delay = _rnd.Next(100, 100);
await Task.Delay(delay);
return($"Delay {delay}ms");
}
public Task <PolicySettings> Get(ICircuitRequest request)
{
var settings = new PolicySettings();
settings.DurationOfBreak = GetTimeSpan(request, "Polly.DurationOfBreak");
settings.FailurePercentThreshold = GetInt(request, "Polly.FailurePercentThreshold");
settings.MaxParallelization = GetInt(request, "Polly.MaxParallelization");
settings.MaxQueingActions = GetInt(request, "Polly.MaxQueingActions");
settings.MinimumThroughput = GetInt(request, "Polly.MinimumThroughput");
settings.SamplingDuration = GetTimeSpan(request, "Polly.SamplingDuration");
settings.Timeout = GetTimeSpan(request, "Polly.Timeout");
settings.Retry = GetInt(request, "Polly.Retry");
return(Task.FromResult(settings));
}
public Task <PolicySettings> Get(ICircuitRequest request)
{
var result = _config.Policies?.FirstOrDefault(x => string.Equals(x.CircuitKey, request.CircuitKey));
if (result != null)
{
return(Task.FromResult(BuildSettings(result)));
}
var serviceName = request.Context.GetServiceName();
result = _config.Policies?.FirstOrDefault(x => string.Equals(x.CircuitKey, $"{serviceName}"));
if (result != null)
{
return(Task.FromResult(BuildSettings(result)));
}
return(Task.FromResult <PolicySettings>(null));
}
public async Task <ICircuitResponse <T> > ExecuteAsync <T>(ICircuitRequest request, Func <ICircuitRequest, Task <T> > funcAsync)
{
try
{
var sw = Stopwatch.StartNew();
var value = await funcAsync(request);
sw.Stop();
var response = new CircuitResponse <T> {
Status = CircuitStatus.Succeed, Value = value
};
await Notify(request, response, sw.Elapsed);
return(response);
}
catch (Exception e)
{
return(await HandleError <CircuitResponse <T> >(request, e));
}
}
private TResponse BuildResponse <TResponse>(ICircuitRequest request, Polly.OutcomeType outcome, Exception finalException, TimeSpan executionTime) where TResponse : ICircuitResponse, new()
{
var result = new TResponse();
if (outcome == Polly.OutcomeType.Failure)
{
if (finalException is Polly.Timeout.TimeoutRejectedException || finalException is TimeoutException)
{
Trace(CircuitStatus.Timeout, request, finalException, executionTime);
result.Status = CircuitStatus.Timeout;
}
else if (finalException is Polly.CircuitBreaker.BrokenCircuitException)
{
Trace(CircuitStatus.Broken, request, finalException, executionTime);
result.Status = CircuitStatus.Broken;
}
else
{
CircuitBreakerLog.LogError(finalException, finalException.Message);
Trace(CircuitStatus.Failed, request, finalException, executionTime);
result.Status = CircuitStatus.Failed;
}
}
else
{
Trace(CircuitStatus.Succeed, request, finalException, executionTime);
result.Status = CircuitStatus.Succeed;
}
return(result);
}
private void Trace(CircuitStatus status, ICircuitRequest request, Exception exception, TimeSpan executiontime)
{
CircuitBreakerLog.LogTrace($"RequestId:{request.RequestId}|CircuitKey:{request.CircuitKey}|Status:{status}|ExecutionTime:{executiontime.TotalMilliseconds}ms|Msg:{exception?.Message}");
}
private TimeSpan?GetTimeSpan(ICircuitRequest request, string key)
{
var value = GetInt(request, key);
return(value.HasValue ? TimeSpan.FromMilliseconds(value.Value) : (TimeSpan?)null);
}
