/// <summary>
/// Configures Polly Policy Handlers using options defined in <typeparamref name="T"/>.
/// Adds retry, circuit breaker and bulkhead policies depending on the configured <see cref="HttpOptions"/> values.
/// </summary>
/// <typeparam name="T">The option type.</typeparam>
/// <param name="builder">The <see cref="IHttpClientBuilder"/>.</param>
/// <returns>The updated <see cref="IHttpClientBuilder"/>.</returns>
public static IHttpClientBuilder AddPoliciesFromOptions <T>(this IHttpClientBuilder builder)
where T : HttpOptions, new()
{
if (builder == null)
{
throw new ArgumentNullException(nameof(builder));
}
var options = builder.Services.BuildServiceProvider().GetRequiredService <IOptions <T> >().Value;
if (options.ErrorsAllowedBeforeBreaking > 0)
{
builder = builder.AddTransientHttpErrorPolicy(p => p.CircuitBreakerAsync(options.ErrorsAllowedBeforeBreaking, options.BreakDuration));
}
if (options.NumberOfRetries > 0)
{
if (options.RetriesMaximumSleepDuration == TimeSpan.FromTicks(0))
{
builder = builder.AddTransientHttpErrorPolicy(p => p.WaitAndRetryAsync(options.NumberOfRetries, _ => options.RetriesSleepDuration));
}
else
{
builder = builder.AddTransientHttpErrorPolicy(
p => p.WaitAndRetryAsync(DecorrelatedJitter(options.NumberOfRetries, options.RetriesSleepDuration, options.RetriesMaximumSleepDuration)));
}
}
if (options.MaxParallelization > 0)
{
builder = builder.AddPolicyHandler(Policy.BulkheadAsync(options.MaxParallelization).AsAsyncPolicy <HttpResponseMessage>());
}
return(builder);
}
public IHttpServiceBuilder WaitAndRetry(WaitAndRetryOptions options)
{
_httpClientBuilder.AddTransientHttpErrorPolicy(builder => builder.WaitAndRetryAsync(
options.WaitList
));
return(this);
}
public S3Client(S3Config config, HttpMessageHandler messageHandler)
{
ServiceCollection services = new ServiceCollection();
services.AddSingleton(x => Options.Create(config));
IS3ClientBuilder builder = services.AddSimpleS3Core();
IHttpClientBuilder httpBuilder = builder.UseHttpClientFactory();
if (messageHandler != null)
{
httpBuilder.ConfigurePrimaryHttpMessageHandler(x => messageHandler);
}
httpBuilder.SetHandlerLifetime(TimeSpan.FromMinutes(5));
Random random = new Random();
// Policy is:
// Retries: 3
// Timeout: 2^attempt seconds (2, 4, 8 seconds) + -100 to 100 ms jitter
httpBuilder.AddTransientHttpErrorPolicy(p => p.WaitAndRetryAsync(3,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))
+ TimeSpan.FromMilliseconds(random.Next(-100, 100))));
_provider = services.BuildServiceProvider();
_objectClient = _provider.GetRequiredService <IS3ObjectClient>();
_bucketClient = _provider.GetRequiredService <IS3BucketClient>();
_serviceClient = _provider.GetRequiredService <IS3ServiceClient>();
}
public static IHttpClientBuilder AddResilience(this IHttpClientBuilder clientBuilder, IPolicyConfig policyConfig)
{
if (policyConfig.HasResilicence)
{
clientBuilder.AddTransientHttpErrorPolicy(builder => builder.WaitAndRetryAsync(
new[] {
TimeSpan.FromSeconds(1),
TimeSpan.FromSeconds(5),
TimeSpan.FromSeconds(10),
}));
clientBuilder.AddTransientHttpErrorPolicy(builder => builder.CircuitBreakerAsync(
handledEventsAllowedBeforeBreaking: 3,
durationOfBreak: TimeSpan.FromSeconds(30)));
if (policyConfig.HandlerLifetime.Ticks > 0)
{
clientBuilder.SetHandlerLifetime(policyConfig.HandlerLifetime);
}
}
return(clientBuilder);
}
public static S3Client Create()
{
//In this example we are using Dependency Injection (DI) using Microsoft's DI framework
ServiceCollection services = new ServiceCollection();
//We use Microsoft.Extensions.Configuration framework here to load our config file
ConfigurationBuilder configBuilder = new ConfigurationBuilder();
configBuilder.AddJsonFile("Config.json", false);
IConfigurationRoot root = configBuilder.Build();
//We use Microsoft.Extensions.Logging here to add support for logging
services.AddLogging(x =>
{
x.AddConsole();
x.AddConfiguration(root.GetSection("Logging"));
});
//Here we setup our S3Client
IS3ClientBuilder clientBuilder = services.AddSimpleS3Core(s3Config =>
{
root.Bind(s3Config);
s3Config.Credentials = new StringAccessKey("Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG");
s3Config.Region = AwsRegion.USEast1;
s3Config.Endpoint = new Uri("https://play.min.io:9000/");
});
//We enable HTTP Factory support here.
IHttpClientBuilder httpBuilder = clientBuilder.UseHttpClientFactory();
//Every 5 minutes we create a new connection, thereby reacting to DNS changes
httpBuilder.SetHandlerLifetime(TimeSpan.FromMinutes(5));
//Uncomment this line if you want to use a proxy
//httpBuilder.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler { Proxy = new WebProxy("http://<YourProxy>:<ProxyPort>") });
//Here we enable retrying. We retry 3 times with a delay of 600 ms between attempts. For more examples, see https://github.com/App-vNext/Polly
httpBuilder.AddTransientHttpErrorPolicy(p => p.WaitAndRetryAsync(3, _ => TimeSpan.FromMilliseconds(600)));
//Finally we build the service provider and return the S3Client
IServiceProvider serviceProvider = services.BuildServiceProvider();
return(serviceProvider.GetRequiredService <S3Client>());
}
/// <summary>
/// The extension method to handle transient exceptions.
/// It leverages the Polly lib in handling the exceptions.
/// </summary>
/// <param name="httpClientBuilder">The IHttpClientBuilder object.</param>
/// <param name="configurationSettings">The configuration settings object.</param>
/// <returns>Returns the same IHttpClientBuilder object.</returns>
public static IHttpClientBuilder AddTransientExceptionHandlerPolicy(
this IHttpClientBuilder httpClientBuilder,
ConfigurationSettings configurationSettings)
{
var retryCount = configurationSettings.RetryCount;
var retryDelay = configurationSettings.RetryDelay;
return(httpClientBuilder.AddTransientHttpErrorPolicy(policyBuilder =>
{
return policyBuilder.WaitAndRetryAsync(
retryCount + 1,
(count) => TimeSpan.FromSeconds(retryDelay),
(response, timespan, count, context) =>
{
if (count > retryCount)
{
throw new ApplicationException($"Transient exception still happens after {retryCount} re-tries!");
}
});
}));
}
public static IHttpClientBuilder ApplyResiliencePolicies(this IHttpClientBuilder builder)
{
Random jitter = new Random();
AsyncCircuitBreakerPolicy <HttpResponseMessage> circuitBreaker = Policy
.Handle <HttpRequestException>()
.OrResult <HttpResponseMessage>(x => x.StatusCode >= HttpStatusCode.InternalServerError)
.AdvancedCircuitBreakerAsync(
failureThreshold: 0.5,
samplingDuration: TimeSpan.FromSeconds(10),
minimumThroughput: 6,
durationOfBreak: TimeSpan.FromMinutes(1));
builder
.AddTransientHttpErrorPolicy(transient => transient
.WaitAndRetryAsync(3, (retryCount) =>
TimeSpan.FromSeconds(Math.Pow(retryCount, 2))
+ TimeSpan.FromMilliseconds(jitter.NextDouble() * 500d)))
.AddPolicyHandler(Policy.TimeoutAsync <HttpResponseMessage>(TimeSpan.FromSeconds(60)))
.AddPolicyHandler(circuitBreaker);
return(builder);
}
/// <summary>
/// Adds a Polly circiut breaker to your clients.
/// </summary>
/// <param name="client"></param>
/// <param name="handledEventsAllowedBeforeBreaking"></param>
/// <param name="durationOfBreak"></param>
/// <returns></returns>
public static IHttpClientBuilder AddCircuitBreakerPolicy(this IHttpClientBuilder client, int handledEventsAllowedBeforeBreaking, TimeSpan durationOfBreak)
{
client.AddTransientHttpErrorPolicy(builder => CircuitBreakerPolicy(builder, handledEventsAllowedBeforeBreaking, durationOfBreak));
return(client);
}
/// <summary>
/// Applies configuration from <see cref="HttpClientOptions"/> <typeparamref name="TOptions"/>
/// to the current <see cref="HttpClient"/> registration.
/// Options are automatically registered as well.
/// </summary>
/// <typeparam name="TOptions">The type of <see cref="HttpClientOptions"/> to register and use.</typeparam>
/// <param name="builder">The <see cref="IHttpClientBuilder"/>.</param>
/// <param name="configuration">The <see cref="IConfiguration"/>.</param>
/// <param name="key">
/// The configuration section key name to use.
/// If not provided, it will be the <typeparamref name="T"/> type name without the -Options prefix.
/// (see <see cref="ConfigurationExtensions.DefaultOptionsName(Type)"/>.
/// </param>
public static IHttpClientBuilder ConfigureWithOptions <TOptions>(
this IHttpClientBuilder builder,
IConfiguration configuration,
string?key = null)
where TOptions : HttpClientOptions, new()
{
builder.ConfigureHttpClient((sp, client) =>
{
var options = sp.GetRequiredService <IOptionsMonitor <TOptions> >().CurrentValue;
if (options.BaseAddress != null)
{
client.BaseAddress = options.BaseAddress;
}
if (options.Headers != null)
{
foreach (var header in options.Headers.Where(x => !string.IsNullOrEmpty(x.Value)))
{
client.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value);
}
}
if (client.DefaultRequestHeaders.UserAgent.Count == 0)
{
var appInfo = sp.GetRequiredService <IApplicationInfo>();
client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", $"{appInfo.Name}/{appInfo.Version} ({appInfo.Environment})");
}
});
var options = configuration.ReadOptionsAndValidate <TOptions>(key);
if (options.Timeout != TimeSpan.Zero)
{
builder = builder.AddPolicyHandler(
Policy.TimeoutAsync(options.Timeout).AsAsyncPolicy <HttpResponseMessage>());
}
if (options.ErrorsAllowedBeforeBreaking > 0)
{
builder = builder.AddTransientHttpErrorPolicy(p => p.CircuitBreakerAsync(options.ErrorsAllowedBeforeBreaking, options.BreakDuration));
}
if (options.NumberOfRetries > 0)
{
if (options.RetriesMaximumSleepDuration == TimeSpan.FromTicks(0))
{
builder = builder.AddTransientHttpErrorPolicy(
p => p.WaitAndRetryAsync(options.NumberOfRetries, _ => options.RetriesSleepDuration));
}
else
{
builder = builder.AddTransientHttpErrorPolicy(
p => p.WaitAndRetryAsync(
DecorrelatedJitter(options.NumberOfRetries, options.RetriesSleepDuration, options.RetriesMaximumSleepDuration)));
}
}
if (options.MaxParallelization > 0)
{
builder = builder.AddPolicyHandler(
Policy.BulkheadAsync(options.MaxParallelization).AsAsyncPolicy <HttpResponseMessage>());
}
if (options.IgnoreCertificateValidation)
{
builder.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler()
{
ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator,
});
}
return(builder);
}
public static void DefaultStrategy(Serilog.ILogger logger, IHttpClientBuilder builder)
{
if (!int.TryParse(Environment.GetEnvironmentVariable($"apiRetriesCount"), out var retries))
{
retries = 3;
}
if (!int.TryParse(Environment.GetEnvironmentVariable($"apiRetriesDelayMin"), out var delayMin))
{
delayMin = 100;
}
if (!int.TryParse(Environment.GetEnvironmentVariable($"apiRetriesDelayStep"), out var delayStep))
{
delayStep = 50;
}
if (!int.TryParse(Environment.GetEnvironmentVariable($"apiRetriesJitterMin"), out var jitterMin))
{
jitterMin = 0;
}
if (!int.TryParse(Environment.GetEnvironmentVariable($"apiRetriesJitterMax"), out var jitterMax))
{
jitterMax = 100;
}
if (!int.TryParse(Environment.GetEnvironmentVariable($"apiCircuitBreakerCount"), out var circuitCount))
{
circuitCount = 5;
}
if (!int.TryParse(Environment.GetEnvironmentVariable($"apiCircuitBreakerDelaySeconds"),
out var circuitDelay))
{
circuitDelay = 30;
}
if (!int.TryParse(Environment.GetEnvironmentVariable($"apiHandlerLifetimeSeconds"), out var lifetime))
{
lifetime = 300;
}
Action <DelegateResult <HttpResponseMessage>, TimeSpan, Context> onRetry = (d, timespan, context) =>
{
// var isTimeout = d.Exception?.Message == "The operation was canceled."
// && d.Exception?.InnerException?.Message == "The operation was canceled.";
//
var isTimeout = d.Exception is HttpRequestException &&
d.Exception?.InnerException is TaskCanceledException;
if (isTimeout)
{
logger.Warning("Retry - Timeout for api client {0}", builder.Name);
}
else
{
var uri = d.Result?.RequestMessage?.RequestUri;
var message = d.Result?.ToString();
logger.Warning("Retry - Exception for api client {0} - Url {1} Message {2}",
builder.Name, uri, message);
}
};
Action <DelegateResult <HttpResponseMessage>, TimeSpan, Context> onBreak = (d, timespan, context) =>
{
logger.Error("CircuitBreaker - onBreak for api client {0}", builder.Name);
};
Action <Context> onReset = (context) =>
{
logger.Information("CircuitBreaker - onReset for api client {0}", builder.Name);
};
Action onHalfOpen = () =>
{
logger.Warning("CircuitBreaker - onHalfOpen for api client {0}", builder.Name);
};
//https://docs.microsoft.com/en-us/aspnet/core/fundamentals/http-requests?view=aspnetcore-2.1
//https://github.com/App-vNext/Polly
//https://github.com/App-vNext/Polly/wiki/Polly-and-HttpClientFactory
//HttpRequestException, 5XX and 408
var jitter = new Random();
builder
.AddTransientHttpErrorPolicy(p =>
p.WaitAndRetryAsync(retries, retryAttempt =>
//DecorrelatedExponent(jitter)
LinearAndJitter(retryAttempt, jitter, jitterMin, jitterMax, delayMin, delayStep)
, onRetry)
)
.AddTransientHttpErrorPolicy(p => /* or p.AdvancedCircuitBreakerAsync() */
p.CircuitBreakerAsync(circuitCount, TimeSpan.FromSeconds(circuitDelay),
onBreak, onReset, onHalfOpen)
)
//SetHandlerLifetime can prevent the handler from reacting to DNS changes
.SetHandlerLifetime(TimeSpan.FromSeconds(lifetime))
.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler()
{
AllowAutoRedirect = true,
MaxAutomaticRedirections = 3,
UseDefaultCredentials = false,
AutomaticDecompression = DecompressionMethods.GZip,
});
}
