public EncompassHttpClientBuilder AddEncompassRetryPolicyHandler()
{
var retryPolicy = HttpPolicyExtensions.HandleTransientHttpError().RetryAsync(_options.EncompassClientOptions.RetryCount);
_builder.AddPolicyHandler(retryPolicy);
return(this);
}
public static IHttpClientBuilder UseTimeoutPolicy(this IHttpClientBuilder builder, TimeSpan timeout)
{
TimeoutPolicy <HttpResponseMessage>?timeoutPolicy = Policy.TimeoutAsync <HttpResponseMessage>(timeout);
builder.AddPolicyHandler(timeoutPolicy);
return(builder);
}
public static IHttpClientBuilder AddCircuitBreaker(this IHttpClientBuilder builder, int numEventsBeforeBreak = 3)
{
var services = builder.Services.BuildServiceProvider();
var logger = services.GetService <Logger>();
var policy = Policy.HandleResult <HttpResponseMessage>(r =>
{
if (!r.IsSuccessStatusCode)
{
logger.WriteError($"Circuit Breaker: Response status code not success => {r.StatusCode}");
}
return(!r.IsSuccessStatusCode);
}).Or <Exception>(r =>
{
logger.WriteError("Polly Circuit Breaker catch exception");
logger.WriteError(r.Message);
return(true);
})
.CircuitBreakerAsync(numEventsBeforeBreak, TimeSpan.FromSeconds(10), (message, timeSpan) =>
{
logger.WriteInfo("Circuit break start");
}, () => {
logger.WriteInfo("Circuit break resume");
});
return(builder.AddPolicyHandler(policy));
}
public static IHttpClientBuilder AddResiliencePolicies(this IHttpClientBuilder httpClientBuilder)
{
var timesToRetry = 10;
var secondsToWaitForSingleResponse = 10;
// this will handle transient errors (500 errros, 408 and httprequestexceptions) by delaying
// and then retrying based on the array below
var jitterer = new Random();
var retryPolicy = HttpPolicyExtensions
.HandleTransientHttpError()
.Or <TimeoutRejectedException>()
.WaitAndRetryAsync(
retryCount: timesToRetry,
sleepDurationProvider: retryAttempt =>
(TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)) // exponential back-off: 2, 4, 8 etc
+ TimeSpan.FromMilliseconds(jitterer.Next(0, 1000))),
onRetry: (result, timeSpan, retryAttempt, ctx) =>
{
Log
.ForContext("RequestUri", result.Result.RequestMessage.RequestUri)
.ForContext("ResponseCode", result.Result.StatusCode)
.Information(result.Exception, $"Retrying HTTP call -- ({retryAttempt} of {timesToRetry})");
}); // plus some jitter: up to 1 second);
// This will kill / fail an api call if it takes longer than specified seconds to receive response
var timeoutPolicy = Policy.TimeoutAsync <HttpResponseMessage>(secondsToWaitForSingleResponse);
return(httpClientBuilder
.AddPolicyHandler(retryPolicy)
.AddPolicyHandler(timeoutPolicy));
}
/// <summary>
/// Configures Polly Policy Handlers using options defined in <typeparamref name="T"/>.
/// Adds retry, circuit breaker and bulkhead policies depending on the configured <see cref="HttpOptions"/> values.
/// </summary>
/// <typeparam name="T">The option type.</typeparam>
/// <param name="builder">The <see cref="IHttpClientBuilder"/>.</param>
/// <returns>The updated <see cref="IHttpClientBuilder"/>.</returns>
public static IHttpClientBuilder AddPoliciesFromOptions <T>(this IHttpClientBuilder builder)
where T : HttpOptions, new()
{
if (builder == null)
{
throw new ArgumentNullException(nameof(builder));
}
var options = builder.Services.BuildServiceProvider().GetRequiredService <IOptions <T> >().Value;
if (options.ErrorsAllowedBeforeBreaking > 0)
{
builder = builder.AddTransientHttpErrorPolicy(p => p.CircuitBreakerAsync(options.ErrorsAllowedBeforeBreaking, options.BreakDuration));
}
if (options.NumberOfRetries > 0)
{
if (options.RetriesMaximumSleepDuration == TimeSpan.FromTicks(0))
{
builder = builder.AddTransientHttpErrorPolicy(p => p.WaitAndRetryAsync(options.NumberOfRetries, _ => options.RetriesSleepDuration));
}
else
{
builder = builder.AddTransientHttpErrorPolicy(
p => p.WaitAndRetryAsync(DecorrelatedJitter(options.NumberOfRetries, options.RetriesSleepDuration, options.RetriesMaximumSleepDuration)));
}
}
if (options.MaxParallelization > 0)
{
builder = builder.AddPolicyHandler(Policy.BulkheadAsync(options.MaxParallelization).AsAsyncPolicy <HttpResponseMessage>());
}
return(builder);
}
// request.GetPolicyExecutionContext()
public static IHttpClientBuilder AddGeolocationServicePolicyHandler(this IHttpClientBuilder builder, int retryCount)
=> builder.AddPolicyHandler(Policy.HandleInner <HttpRequestException>()
.OrResult <HttpResponseMessage>((response) => {
var successStatusCode = response.IsSuccessStatusCode;
var noContent = response.StatusCode == HttpStatusCode.NoContent;
if (noContent)
{
var context = response.RequestMessage.GetPolicyExecutionContext();
context[ContextKey.ProvidersFailed] = context.GetFailedProviders().Concat(new Provider[] { context.GetProvider() });
}
return(!successStatusCode || noContent);
})
.RetryAsync(retryCount, (ex, retryCount, context) =>
{
var request = context.GetRequest();
var handler = context.GetHandler();
var providerIndex = context.GetProviderIndex();
var failedProviders = context.GetFailedProviders();
if (request != null && handler != null)
{
var(provider, nextProviderIndex) = handler.SetRequestMessageUri(request, providerIndex, failedProviders);
context[ContextKey.Provider] = provider;
context[ContextKey.RetryCount] = retryCount;
context[ContextKey.ProviderIndex] = nextProviderIndex;
}
}));
public static IHttpClientBuilder AddSearchPolicyHandlers(
this IHttpClientBuilder builder,
ILogger logger,
string searchName,
ITelemetryService telemetryService,
IAppConfiguration config)
{
return(builder
.AddPolicyHandler(SearchClientPolicies.SearchClientFallBackCircuitBreakerPolicy(
logger,
searchName,
telemetryService))
.AddPolicyHandler(SearchClientPolicies.SearchClientWaitAndRetryPolicy(
config.SearchCircuitBreakerWaitAndRetryCount,
config.SearchCircuitBreakerWaitAndRetryIntervalInMilliseconds,
logger,
searchName,
telemetryService))
.AddPolicyHandler(SearchClientPolicies.SearchClientCircuitBreakerPolicy(
config.SearchCircuitBreakerBreakAfterCount,
TimeSpan.FromSeconds(config.SearchCircuitBreakerDelayInSeconds),
logger,
searchName,
telemetryService))
.AddPolicyHandler(SearchClientPolicies.SearchClientTimeoutPolicy(
TimeSpan.FromMilliseconds(config.SearchHttpRequestTimeoutInMilliseconds),
logger,
searchName,
telemetryService)));
}
public static IHttpClientBuilder AddRetryPolicyHandler(this IHttpClientBuilder builder, int retryCount = 6)
{
return(builder.AddPolicyHandler(
HttpPolicyExtensions.HandleTransientHttpError()
.OrResult(message => message.StatusCode == HttpStatusCode.NotFound)
.WaitAndRetryAsync(retryCount, sleepDuration => TimeSpan.FromSeconds(Math.Pow(2, sleepDuration)))));
}
public static IHttpClientBuilder AddPolicyHandlers(this IHttpClientBuilder builder, IAsyncPolicy <HttpResponseMessage>[] policies)
{
foreach (var policy in policies)
{
builder.AddPolicyHandler(policy);
}
return(builder);
}
public static IHttpClientBuilder AddPolly(this IHttpClientBuilder builder)
{
if (builder == null)
{
throw new ArgumentNullException(nameof(builder));
}
return(builder.AddPolicyHandler(PollyUtils.PolicySelector));
}
private static IHttpClientBuilder AddPolicyHandlers(IHttpClientBuilder httpClientBuilder,
IEnumerable <IAsyncPolicy <HttpResponseMessage> > policies)
{
foreach (var asyncPolicy in policies)
{
httpClientBuilder.AddPolicyHandler(asyncPolicy);
}
return(httpClientBuilder);
}
private static IHttpClientBuilder AddRetryPolicy(
this IHttpClientBuilder clientBuilder,
RetryPolicySettings settings)
{
return(clientBuilder
.AddPolicyHandler(HttpPolicyExtensions
.HandleTransientHttpError()
.Or <TimeoutRejectedException>()
.WaitAndRetryAsync(settings)));
}
public static IHttpClientBuilder AddCircuitBreakerPolicy(this IHttpClientBuilder clientBuilder)
{
IAsyncPolicy <HttpResponseMessage> policy = HttpPolicyExtensions
.HandleTransientHttpError()
.CircuitBreakerAsync(5, TimeSpan.FromSeconds(30));
clientBuilder.AddPolicyHandler(policy);
return(clientBuilder);
}
/// <summary>
/// Add Retry policy with 3 attempts before exception
/// </summary>
/// <param name="builder">IHttpClientBuilder</param>
public static IHttpClientBuilder AddRetryPolicy(this IHttpClientBuilder builder)
{
return(builder.AddPolicyHandler((serviceProvider, request) => HttpPolicyExtensions.HandleTransientHttpError()
.WaitAndRetryAsync(3, attempt => TimeSpan.FromSeconds(attempt * 3),
onRetry: (outcome, timespan, retryAttempt, context) =>
{
var logger = serviceProvider.GetRequiredService <ILoggerFactory>().CreateLogger <ExchangeRateCacheStrategy>();
logger.LogWarning("Delaying for {delay}ms, then making retry {retry}. CorrelationId {correlationId}",
timespan.TotalMilliseconds, retryAttempt, context.CorrelationId);
})));
}
public static IHttpClientBuilder AddPolicies(
this IHttpClientBuilder clientBuilder,
IConfiguration configuration
)
{
var policyOptions = configuration.Get <PolicyOptions>();
return(clientBuilder.AddPolicyHandler(HttpPolicy.GetTimeout(policyOptions.Timeout))
.AddPolicyHandler(HttpPolicy.GetRetryPolicy(policyOptions.HttpRetry))
.AddPolicyHandler(HttpPolicy.GetCircuitBreakerPolicy(policyOptions.HttpCircuitBreaker)));
}
public static IHttpClientBuilder AddRetryPolicy(this IHttpClientBuilder clientBuilder)
{
IAsyncPolicy <HttpResponseMessage> policy = HttpPolicyExtensions
.HandleTransientHttpError()
.OrResult(msg => msg.StatusCode == HttpStatusCode.NotFound)
.WaitAndRetryAsync(4, t => TimeSpan.FromMilliseconds(Math.Pow(2, t) * 500));
clientBuilder.AddPolicyHandler(policy);
return(clientBuilder);
}
/// <summary>
/// This policy will be added and configured in Startup.cs or where you add your http clients to services. It will retry the http call as many times as it is configured.
/// </summary>
/// <param name="builder">The http client builder parameter. This is actually the AddHttpClient<>().</param>
/// <param name="onRetryAsyncFunc">This represents the delegate with the parameters obtained after each call. It can be used when it is configured in order to add you custom logic after the http call is done (log, do something else).</param>
/// <param name="retryAttempts">Represents the number of retries of your http call.</param>
/// <param name="httpStatusCodeToRetry">Represents a list of http status codes. You can add you own logic here (for which http codes the retry should happen). Be aware that default behaviour is to not retry an http call if the result code belongs to 2xx http status codes.</param>
/// <param name="onResultFunc">Represent a custom action which can be condigured at startup in order to execeute that specific action on the received http response message.</param>
/// <param name="sleepDuration">Represent the number of seconds used to wait between http calls retries.</param>
/// <param name="timeoutDurationPerPolicyRequest">Represent the number of seconds used to wait for each policy retry request. The exception thrown in case the request will overlap the value is <see cref="T:Polly.Timeout.TimeoutRejectedException" />. In case this property is not set (null) then no timeout policy will be added to this http client.</param>
/// <returns>The policy handlers with all the configurable rules.</returns>
public static IHttpClientBuilder ConfigureRetryPolicy(
this IHttpClientBuilder builder,
Func<DelegateResult<HttpResponseMessage>, TimeSpan, int, Context, Task> onRetryAsyncFunc = null,
int retryAttempts = 3,
IEnumerable<HttpStatusCode> httpStatusCodeToRetry = null,
Func<HttpResponseMessage, Task<bool>> onResultFuncAsync = null,
TimeSpan? sleepDuration = null,
TimeSpan? timeoutDurationPerPolicyRequest = null)
{
var concreteTimeoutDurationPerPolicyRequest = GetConcreteTimeoutPerPolicyRequest(timeoutDurationPerPolicyRequest);
var timeOutPolicy = BuildTimeoutPolicy(concreteTimeoutDurationPerPolicyRequest);
var retryPolicy = BuildRetryPolicy(retryAttempts, onRetryAsyncFunc, httpStatusCodeToRetry, onResultFuncAsync, sleepDuration);
builder.AddPolicyHandler(retryPolicy);
builder.AddPolicyHandler(timeOutPolicy);
builder.ChangeHttpClientTimoutValue(retryAttempts, concreteTimeoutDurationPerPolicyRequest, sleepDuration);
return builder;
}
private static IHttpClientBuilder AddRetryPolicy(this IHttpClientBuilder httpClientBuilder,
RetryPolicyOptions options)
{
return(httpClientBuilder.AddPolicyHandler(
HttpPolicyExtensions.HandleTransientHttpError()
.Or <TimeoutRejectedException>()
.Or <HttpRequestException>()
.WaitAndRetryAsync(
options.RetryCount,
options.SleepDurationProvider(options.RetryCount, TimeSpan.FromMilliseconds(options.MedianFirstRetryDelayMilliseconds)),
options.OnRetry)));
}
public static IHttpClientBuilder AddFaultHandlePolicies(this IHttpClientBuilder builder)
{
return(builder
.AddPolicyHandler((serviceProvider, request) =>
{
var factory = serviceProvider.GetService <IWebApiPolicyFactory>();
// Retries should only be performed on idempotent operations
return request.Method == HttpMethod.Get ? factory.CreateWaitAndRetryPolicy() : factory.NoPolicy();
})
.AddPolicyHandler((serviceProvider, request) => serviceProvider.GetService <IWebApiPolicyFactory>().CreateTimeoutPolicy()) // Place the timeoutPolicy inside the retryPolicy, to make it time out each try.
.AddPolicyHandler((serviceProvider, request) => serviceProvider.GetService <IWebApiPolicyFactory>().CreateCircuitBreakerPolicy(request)));
}
public static IHttpClientBuilder AddDefaultFaultHandlingPolicies(this IHttpClientBuilder builder)
{
return(builder
.AddPolicyHandler((serviceProvider, request) => serviceProvider.GetService <IWebApiPolicyFactory>().CreateCircuitBreakerPolicy(builder.Name))
.AddPolicyHandler((serviceProvider, request) =>
{
var factory = serviceProvider.GetService <IWebApiPolicyFactory>();
// Retries should only be performed on idempotent operations
return request.Method == HttpMethod.Get ? factory.CreateRetryPolicy() : factory.CreateNoOpPolicy();
})
.AddPolicyHandler((serviceProvider, request) => serviceProvider.GetService <IWebApiPolicyFactory>().CreateTimeoutPolicy()));
}
public static IHttpClientBuilder AddRetryPolicy(this IHttpClientBuilder value)
{
return(value.AddPolicyHandler((_) =>
HttpPolicyExtensions
.HandleTransientHttpError()
.OrResult(msg => msg.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
.WaitAndRetryAsync(new TimeSpan[] {
TimeSpan.FromSeconds(1),
TimeSpan.FromSeconds(5),
TimeSpan.FromSeconds(10)
})));
}
private static void ApplyPollyConfiguration <TClient, TMessageHandler>(HttpOptions options, IHttpClientBuilder httpClientBuilder) where TClient : class
{
if (options.RetryLevel == RetryLevel.Update)
{
httpClientBuilder
.AddPolicyHandler((serviceCollection, request) => GetRetryPolicy <TClient, TMessageHandler>(serviceCollection));
}
if (options.RetryLevel == RetryLevel.Read)
{
httpClientBuilder
.AddPolicyHandler((serviceCollection, request) =>
request.Method == HttpMethod.Get
? GetRetryPolicy <TClient, TMessageHandler>(serviceCollection)
: GetNoRetryPolicy());
}
if (options.UseCircuitBreaker)
{
httpClientBuilder.AddPolicyHandler((serviceCollection, request) => GetCircuitBreakerPolicy <TClient, TMessageHandler>(serviceCollection));
}
}
public static IHttpClientBuilder AddDefaultPolicy(this IHttpClientBuilder builder, int retryCount, int timeout)
{
var retryPolicy = HttpPolicyExtensions
.HandleTransientHttpError()
.Or <TimeoutRejectedException>()
.WaitAndRetryAsync(retryCount, x => TimeSpan.FromSeconds(2 << x));
var timeoutPolicy = Policy
.TimeoutAsync <HttpResponseMessage>(timeout);
return(builder
.AddPolicyHandler(retryPolicy)
.AddPolicyHandler(timeoutPolicy));
}
public static void AddResiliency(this IHttpClientBuilder builder, Queue <Customer> customerQueue)
{
/// LAB WORK GOES HERE ///
// use DemoPolicyFactory as example
// 1. add fallback policy to requeue customers
builder.AddPolicyHandler(Policy
.HandleResult <HttpResponseMessage>(resp => !resp.IsSuccessStatusCode) // catch any bad responses, transient or not
.Or <Exception>() // handle ANY exception we get back
.FallbackAsync(fallbackAction: (result, context, ct) =>
{
var customer = context["Customer"];
customerQueue.Enqueue((Customer)customer);
return(Task.FromResult(result.Result ?? new HttpResponseMessage()
{
StatusCode = HttpStatusCode.ServiceUnavailable
})); // Result null if exception, so need to return something.
}, onFallbackAsync: (exception, context) =>
{
// log something
return(Task.CompletedTask);
}));
// 2. OPTIONAL: add retry policy (not really needed since we are using a queue and re-adding on failure)
builder.AddPolicyHandler(HttpPolicyExtensions
.HandleTransientHttpError()
.Or <TimeoutRejectedException>() // handle timeouts as failures so they get retried, decide if this is appropriate for your use case
.RetryAsync(1));
// 3. add circuit breaker to allow service to recover
builder.AddPolicyHandler(HttpPolicyExtensions
.HandleTransientHttpError()
.Or <TimeoutRejectedException>()
.CircuitBreakerAsync(5, TimeSpan.FromSeconds(30)));
// 4. add timeout policy so we don't wait on slow calls
builder.AddPolicyHandler(Policy
.TimeoutAsync <HttpResponseMessage>(TimeSpan.FromSeconds(1), TimeoutStrategy.Optimistic));
}
/// <summary>
/// Add the policies to an http client
/// </summary>
/// <param name="httpClientBuilder">http client builder</param>
/// <param name="policies">policies to add</param>
/// <returns>the same http client builder</returns>
public static IHttpClientBuilder AddResiliencyPolicies(this IHttpClientBuilder httpClientBuilder, params IPolicyBuilder <HttpResponseMessage>[] policies)
{
//materialize and pin policies in memory
var wrappedPolicy = Policy.WrapAsync(policies.Select(p => p.Build().Policy).ToArray());
return(httpClientBuilder.AddPolicyHandler((sp, request) =>
{
var context = request.GetPolicyExecutionContext() ?? new Context();
//store service provider in context
context[spKey] = sp;
request.SetPolicyExecutionContext(context);
return wrappedPolicy;
}));
}
private static IHttpClientBuilder AddCircuitBreakerPolicy(this IHttpClientBuilder httpClientBuilder,
CircuitBreakerPolicyOptions options)
{
return(httpClientBuilder.AddPolicyHandler(
HttpPolicyExtensions.HandleTransientHttpError()
.Or <TimeoutRejectedException>()
.Or <HttpRequestException>()
.OrResult(m => m.StatusCode == HttpStatusCode.TooManyRequests)
.CircuitBreakerAsync(
options.HandledEventsAllowedBeforeBreaking,
TimeSpan.FromSeconds(options.DurationOfBreakSeconds),
options.OnBreak,
options.OnReset,
options.OnHalfOpen)));
}
public static IHttpClientBuilder AddLCUTimeoutPolicy(this IHttpClientBuilder httpClientBuilder,
IPolicyRegistry <string> registry)
{
return(httpClientBuilder
.AddPolicyHandler(request =>
{
var timeoutPolicy = "regular";
if (request.Method != HttpMethod.Get)
{
timeoutPolicy = "long";
}
return registry.Get <IAsyncPolicy <HttpResponseMessage> >(timeoutPolicy);
}));
}
public static IHttpClientBuilder AddResiliencyPolicies(this IHttpClientBuilder builder, IConfiguration configuration)
{
var resiliencyConfiguration = configuration.GetSection("ServiceRequest").Get <ResiliencyConfiguration>();
builder
.AddPolicyHandler(
Policy.BulkheadAsync <HttpResponseMessage>(resiliencyConfiguration.MaxBulkheadSize, resiliencyConfiguration.MaxBulkheadQueueSize))
.AddTransientHttpErrorPolicy(p =>
p.AdvancedCircuitBreakerAsync(
resiliencyConfiguration.CircuitBreakerThreshold,
TimeSpan.FromSeconds(resiliencyConfiguration.CircuitBreakerSamplingPeriodSeconds),
resiliencyConfiguration.CircuitBreakerMinimumThroughput,
TimeSpan.FromSeconds(resiliencyConfiguration.CircuitBreakerBreakDurationSeconds)))
.AddTransientHttpErrorPolicy(p =>
p.WaitAndRetryAsync(resiliencyConfiguration.MaxRetries, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt) - 2)));
return(builder);
}
public static IHttpClientBuilder UseRetryPolicy(this IHttpClientBuilder builder, int retries, BackoffTime backoffTime)
{
// Add a policy that will handle transient HTTP & Networking errors
RetryPolicy <HttpResponseMessage>?exceptionPolicy = Policy <HttpResponseMessage>
// Handle network errors
.Handle <IOException>()
// Handle other HttpClient errors
.Or <HttpRequestException>()
// Handle Polly timeouts
.Or <TimeoutRejectedException>()
// Handle transient-error status codes
.OrResult(_transientHttpStatusCodePredicate)
// Action
.WaitAndRetryAsync(retries, retryAttempt => backoffTime(retryAttempt));
builder.AddPolicyHandler(exceptionPolicy);
builder.Services.AddSingleton <IRequestStreamWrapper, RetryableBufferingStreamWrapper>();
return(builder);
}
private static IHttpClientBuilder AddCircuitBreakerPolicy(
this IHttpClientBuilder clientBuilder,
ICircuitBreakerPolicySettings settings)
{
// This implementation takes into consideration situations
// when you use the only HttpClient against different hosts.
// In this case we want to have separate CircuitBreaker metrics for each host.
// It allows us avoid situations when all requests to all hosts
// will be stopped by CircuitBreaker due to single host is not available.
var registry = new PolicyRegistry();
return(clientBuilder.AddPolicyHandler(message =>
{
var policyKey = message.RequestUri.Host;
var policy = registry.GetOrAdd(policyKey, BuildCircuitBreakerPolicy(settings));
return policy;
}));
}
