private static IAsyncPolicy BuildRetryPolicy(IDbConnectionFactory connectionFactory, QueryLoggingContext loggingContext)
{
return(connectionFactory.RetryPolicy.WaitAndRetryAsync(
Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromMilliseconds(100), MaxRetryAttempts),
(ex, delay, retryAttempt, ctx) => loggingContext.Retry(retryAttempt, delay)
));
}
private static void AddClients(this IServiceCollection services)
{
var delay = Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay: TimeSpan.FromMilliseconds(100), retryCount: 10, fastFirst: true);
services.AddHttpClient <IMarketDataClient, MarketDataClient>("Trakx.Shrimpy.ApiClient.MarketDataClient")
.AddPolicyHandler((s, request) =>
Policy <HttpResponseMessage>
.Handle <ApiException>()
.Or <HttpRequestException>()
.OrTransientHttpStatusCode()
.WaitAndRetryAsync(delay,
onRetry: (result, timeSpan, retryCount, context) =>
{
var logger = Log.Logger.ForContext <MarketDataClient>();
logger.LogApiFailure(result, timeSpan, retryCount, context);
})
.WithPolicyKey("Trakx.Shrimpy.ApiClient.MarketDataClient"));
services.AddHttpClient <IAccountsClient, AccountsClient>("Trakx.Shrimpy.ApiClient.AccountsClient")
.AddPolicyHandler((s, request) =>
Policy <HttpResponseMessage>
.Handle <ApiException>()
.Or <HttpRequestException>()
.OrTransientHttpStatusCode()
.WaitAndRetryAsync(delay,
onRetry: (result, timeSpan, retryCount, context) =>
{
var logger = Log.Logger.ForContext <AccountsClient>();
logger.LogApiFailure(result, timeSpan, retryCount, context);
})
.WithPolicyKey("Trakx.Shrimpy.ApiClient.AccountsClient"));
}
public static AsyncRetryPolicy GetWaitAndRetryPolicy <TException>(ILoggerService loggerService, int medianFirstRetryDelaySeconds = WaitFactor)
where TException : Exception =>
Policy
.Handle <TException>()
.WaitAndRetryAsync(
Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(medianFirstRetryDelaySeconds), MaxRetries),
GetOnRetryDelegate(MaxRetries, loggerService));
public void Backoff_WithFastFirstEqualToTrue_ResultIsZero()
{
// Arrange
var medianFirstDelay = TimeSpan.FromSeconds(2);
const int retryCount = 10;
const bool fastFirst = true;
const int seed = 1;
// Act
IEnumerable <TimeSpan> result = Backoff.DecorrelatedJitterBackoffV2(medianFirstDelay, retryCount, seed, fastFirst);
// Assert
result.Should().NotBeNull();
result = result.ToList();
result.Should().HaveCount(retryCount);
bool first = true;
int t = 0;
foreach (TimeSpan timeSpan in result)
{
if (first)
{
timeSpan.Should().Be(TimeSpan.FromMilliseconds(0));
first = false;
}
else
{
t++;
AssertOnRetryDelayForTry(t, timeSpan, medianFirstDelay);
}
}
}
public static void UseDefaultHttpRetryLogic()
{
ILogger logger = Logger.Factory.CreateLogger(typeof(PhilomenaClientRetryLogic));
// Use a jittered exponential backoff
var delay = Backoff.DecorrelatedJitterBackoffV2
(
medianFirstRetryDelay: TimeSpan.FromSeconds(1),
retryCount: 4
);
// Retry on transient http errors
var defaultRetryPolicy = HttpPolicyExtensions
.HandleTransientHttpError()
.Or <TimeoutRejectedException>()
.WaitAndRetryAsync(delay, (result, timeout, attempt, context) =>
{
logger.LogWarning(result.Exception, "Request #{Attempt} failed. Retrying in {Timeout}", attempt, timeout);
});
// Timeout requests
var defaultTimeoutPolicy = Policy
.TimeoutAsync <HttpResponseMessage>(TimeSpan.FromSeconds(2));
// Wrap the default policies
var defaultPolicy = Policy.WrapAsync(defaultRetryPolicy, defaultTimeoutPolicy);
// Configure Flurl to use the backoff policy by overriding the HttpClientFactory
FlurlHttp.Configure(settings =>
{
settings.HttpClientFactory = new PollyHttpClientFactory(defaultPolicy);
});
}
public static void Configure(IConfiguration configuration, IServiceCollection serviceCollection)
{
serviceCollection.AddOptions <ShikiOptions>().Bind(configuration.GetSection(Constants.NAME));
var policy = HttpPolicyExtensions.HandleTransientHttpError().OrResult(message => message.StatusCode == HttpStatusCode.TooManyRequests)
.WaitAndRetryAsync(Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(10), 5));
serviceCollection.AddHttpClient(Constants.NAME).AddPolicyHandler(policy).AddHttpMessageHandler(provider =>
{
var logger = provider.GetRequiredService <ILogger <IRateLimiter <ShikiClient> > >();
var rl = new RateLimit(90, TimeSpan.FromMinutes(1.05d)); // 90rpm with .05 as inaccuracy
return(RateLimiterFactory.Create(rl, logger).ToHttpMessageHandler());
}).AddHttpMessageHandler(provider =>
{
var logger = provider.GetRequiredService <ILogger <IRateLimiter <ShikiClient> > >();
var rl = new RateLimit(5, TimeSpan.FromSeconds(1.05d)); //5rps with .05 as inaccuracy
return(RateLimiterFactory.Create(rl, logger).ToHttpMessageHandler());
}).ConfigureHttpClient((provider, client) =>
{
client.DefaultRequestHeaders.UserAgent.Clear();
client.DefaultRequestHeaders.UserAgent.ParseAdd($"{provider.GetRequiredService<IOptions<ShikiOptions>>().Value.ShikimoriAppName}");
client.BaseAddress = new(Wrapper.Constants.BASE_URL);
});
serviceCollection.AddSingleton <ShikiClient>(provider =>
{
var factory = provider.GetRequiredService <IHttpClientFactory>();
var logger = provider.GetRequiredService <ILogger <ShikiClient> >();
return(new(factory.CreateClient(Constants.NAME), logger));
});
serviceCollection.AddSingleton <IExecuteOnStartupService, ShikiExecuteOnStartupService>();
serviceCollection.AddSingleton <IUserFeaturesService <ShikiUserFeatures>, ShikiUserFeaturesService>();
serviceCollection.AddSingleton <ShikiUserService>();
serviceCollection.AddSingleton <IUpdateProvider, ShikiUpdateProvider>();
}
private static IAsyncPolicy <HttpResponseMessage> GetRetryPolicy()
{
var delay = Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay: TimeSpan.FromSeconds(1), retryCount: 5);
return(HttpPolicyExtensions
.HandleTransientHttpError()
.WaitAndRetryAsync(delay));
}
public IAsyncPolicy <HttpResponseMessage> GetRetryPolicyWithContribJitter()
{
var delay = Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay: TimeSpan.FromSeconds(1), retryCount: 5, fastFirst: true);
return(HttpPolicyExtensions
.HandleTransientHttpError() //50xx or 408 (timeout)
.OrResult(msg => msg.StatusCode == System.Net.HttpStatusCode.NotFound)
.WaitAndRetryAsync(delay));
}
public ServiceRetry()
{
//see more details here https://github.com/Polly-Contrib/Polly.Contrib.WaitAndRetry#wait-and-retry-with-jittered-back-off
var delay = Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay: TimeSpan.FromSeconds(3), retryCount: 5);
this.RetryPolicy = Policy
.Handle <Exception>()
.WaitAndRetry(delay);
}
private static AsyncRetryPolicy <HttpResponseMessage> GetRetryPolicy()
{
var p = HttpPolicyExtensions
.HandleTransientHttpError()
.Or <BrokenCircuitException>()
.OrResult(msg => msg.StatusCode == System.Net.HttpStatusCode.NotFound)
.WaitAndRetryAsync(Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(1), 100));
return(p);
}
/// <summary>
/// Get the graph retry policy.
/// </summary>
/// <param name="maxAttempts">the number of max attempts.</param>
/// <returns>A retry policy that can be applied to async delegates.</returns>
public static AsyncRetryPolicy GetGraphRetryPolicy(int maxAttempts)
{
var delay = Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay: TimeSpan.FromSeconds(1), retryCount: maxAttempts);
// Only Handling 502 Bad Gateway Exception
// Other exception such as 429, 503, 504 is handled by default by Graph SDK.
return(Policy
.Handle <ServiceException>(e =>
e.StatusCode == HttpStatusCode.BadGateway)
.WaitAndRetryAsync(delay));
}
public HttpPolicyBuilder WithMeteredRetryPolicy(ILoggerService loggerService)
{
policies.Add(Policy
.HandleResult <HttpResponseMessage>(response => response.StatusCode == HttpStatusCode.TooManyRequests)
.Or <TaskCanceledException>(exception =>
exception.InnerException is IOException ioException &&
ioException.InnerException is SocketException)
.WaitAndRetryAsync(
Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(10), RetryHelper.MaxRetries),
RetryHelper.GetOnRetryDelegate <HttpResponseMessage>(RetryHelper.MaxRetries, loggerService)));
return(this);
}
static PolicyHelpers()
{
IEnumerable <TimeSpan> delay = Backoff.DecorrelatedJitterBackoffV2(
medianFirstRetryDelay: TimeSpan.FromSeconds(1),
retryCount: 5);
RetryPolicy = Policy
.Handle <Exception>()
.WaitAndRetryAsync(
delay,
(ex, ts) => _logger.Warn(ex, "Retrying in {TimeSpan}", ts));
}
public static AsyncRetryPolicy CreateAsyncRetryPolicy()
{
//Taken from https://github.com/Polly-Contrib/Polly.Contrib.WaitAndRetry#wait-and-retry-with-jittered-back-off
var delay = Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay: TimeSpan.FromSeconds(1), retryCount: 5);
var retryPolicy = Policy
.Handle <Exception>()
.WaitAndRetryAsync(delay);
return(retryPolicy);
}
internal static IEnumerable <TimeSpan> Jitter(int retryCount, TimeSpan medianFirstRetryDelay)
{
if (retryCount < 0)
{
throw new ArgumentOutOfRangeException(nameof(retryCount), retryCount, "should be >= 0");
}
if (medianFirstRetryDelay < TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(nameof(medianFirstRetryDelay), medianFirstRetryDelay, "should be >= 0ms");
}
return(Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay, retryCount));
}
/// <summary>
/// Constructor for SMS Service. Takes dependency injected services to
/// send SMS notifications.
/// </summary>
/// <param name="smsSender">The <see cref="ISMSSender"/> implementation to use in this SMS Service</param>
/// <param name="logger">The <see cref="ILogger"/> to be used in logging</param>
/// <param name="dataService">The <see cref="IDataService"/> used for data access</param>
/// <param name="settings">DI <see cref="SMSSettings"/> for the application.</param>
public SMSService(ISMSSender smsSender, ILogger <SMSService> logger, IDataService dataService, IOptions <SMSSettings> settings)
{
_smsSender = smsSender;
_logger = logger;
_dataService = dataService;
_smsSettings = settings.Value;
var maxDelay = TimeSpan.FromSeconds(36);
var delay = Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay: TimeSpan.FromSeconds(1), retryCount: 50)
.Select(s => TimeSpan.FromTicks(Math.Min(s.Ticks, maxDelay.Ticks)));
_retryPolicy = Policy
.Handle <Exception>()
.WaitAndRetryAsync(delay);
}
/// <summary>
/// Creates a Polly Policy to handle <see cref="HttpStatusCode"/> 429 (Too Many Requests)
/// </summary>
/// <remarks> Retries the request using decorrelated jitter up to 5 times.</remarks>
/// <returns> Returns a Polly Policy object.</returns>
internal static IAsyncPolicy <HttpResponseMessage> GetTooManyRequestsPolicy()
{
var delay = Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay: TimeSpan.FromSeconds(1),
retryCount: 5);
return(Policy
.HandleResult <HttpResponseMessage>(msg => msg.StatusCode == HttpStatusCode.TooManyRequests)
.WaitAndRetryAsync(
delay,
onRetry: (outcome, timespan, retryAttempt, context) =>
{
context.GetLogger()?.LogWarning("Too many requests submitted within 1 second; retrying in {Timespan}ms for the {RetryAttempt} time", timespan.TotalMilliseconds, retryAttempt);
}));
}
public static SleepDurationProvider Jitter(int retryCount, TimeSpan medianFirstRetryDelay)
{
if (retryCount < 0)
{
throw new ArgumentOutOfRangeException(nameof(retryCount), retryCount, "should be >= 0");
}
if (medianFirstRetryDelay < TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(nameof(medianFirstRetryDelay), medianFirstRetryDelay,
"should be >= 0ms");
}
return(new SleepDurationProvider(retryCount, Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay, retryCount)));
}
public static void Configure(IConfiguration configuration, IServiceCollection services)
{
services.AddOptions <MalOptions>().Bind(configuration.GetSection(Constants.Name));
services.AddSingleton(provider => RateLimiterExtensions.ConfigurationLambda <MalOptions, MyAnimeListClient>(provider));
var retryPolicy = HttpPolicyExtensions.HandleTransientHttpError()
.OrResult(message => message.StatusCode == HttpStatusCode.TooManyRequests)
.WaitAndRetryAsync(Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(10), 5));
services.AddHttpClient(MalOptions.MyAnimeList).AddPolicyHandler(retryPolicy).ConfigurePrimaryHttpMessageHandler(_ => new HttpClientHandler
{
UseCookies = true,
CookieContainer = new()
}).AddHttpMessageHandler(provider =>
/// <summary>
/// Constructor for Email Service. Takes dependency injected services to
/// send email notifications.
/// </summary>
/// <param name="emailSender">The <see cref="IEmailSender"/> implementation to use in this Email Service</param>
/// <param name="logger">The <see cref="ILogger"/> to be used in logging</param>
/// <param name="dataService">The <see cref="IDataService"/> used for data access</param>
public EmailService(IEmailSender emailSender, ILogger <EmailService> logger, IDataService dataService)
{
_emailSender = emailSender;
_logger = logger;
_dataService = dataService;
var maxDelay = TimeSpan.FromSeconds(36);
var delay = Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay: TimeSpan.FromSeconds(1), retryCount: 50)
.Select(s => TimeSpan.FromTicks(Math.Min(s.Ticks, maxDelay.Ticks)));
_retryPolicy = Policy
.Handle <Exception>()
.WaitAndRetryAsync(delay);
}
public void Backoff_WithRetryEqualToZero_ResultIsEmpty()
{
// Arrange
var medianFirstDelay = TimeSpan.FromSeconds(2);
const int retryCount = 0;
const bool fastFirst = false;
const int seed = 1;
// Act
IEnumerable <TimeSpan> result = Backoff.DecorrelatedJitterBackoffV2(medianFirstDelay, retryCount, seed, fastFirst);
// Assert
result.Should().NotBeNull();
result.Should().BeEmpty();
}
public void Backoff_should_not_overflow_to_give_negative_timespan()
{
// See https://github.com/Polly-Contrib/Polly.Contrib.WaitAndRetry/issues/24
const int seed = 1;
// Arrange
IEnumerable <TimeSpan> delays = Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(1), 100, seed, fastFirst: false);
// Act.
foreach (TimeSpan span in delays)
{
span.Should().BeGreaterOrEqualTo(TimeSpan.Zero);
}
}
/// <summary>
/// Creates a Polly Policy to handle <see cref="HttpStatusCode"/> 5XX and 408 status code errors
/// </summary>
/// <remarks> Retries the request using decorrelated jitter up to 6 times.</remarks>
/// <returns> Returns a Polly Policy object.</returns>
internal static IAsyncPolicy <HttpResponseMessage> GetRetryPolicy()
{
var delay = Backoff.DecorrelatedJitterBackoffV2(
TimeSpan.FromSeconds(1),
retryCount: 6);
return(HttpPolicyExtensions
.HandleTransientHttpError()
.WaitAndRetryAsync(
delay,
onRetry: (outcome, timespan, retryAttempt, context) =>
{
context.GetLogger()?.LogWarning("Delaying for {Delay}ms, then making retry {Retry}", timespan.TotalMilliseconds, retryAttempt);
}));
}
public void Backoff_WithRetryCountLessThanZero_ThrowsException()
{
// Arrange
var medianFirstDelay = TimeSpan.FromSeconds(1);
const int retryCount = -1;
const bool fastFirst = false;
const int seed = 1;
// Act
Action act = () => Backoff.DecorrelatedJitterBackoffV2(medianFirstDelay, retryCount, seed, fastFirst);
// Assert
act.Should().Throw <ArgumentOutOfRangeException>()
.And.ParamName.Should().Be("retryCount");
}
public SiriusWalletsService(
long brokerAccountId,
int retryCount,
TimeSpan retryTimeout,
IApiClient siriusApiClient,
ILog log)
{
_brokerAccountId = brokerAccountId;
_retryCount = retryCount;
_retryTimeout = retryTimeout;
_siriusApiClient = siriusApiClient;
_log = log;
_delay = Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay: TimeSpan.FromMilliseconds(100), retryCount: 7, fastFirst: true);
}
private AsyncRetryPolicy GetRetryPolicy(int maxAttempts, ILogger log)
{
var delay = Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay: TimeSpan.FromSeconds(1), retryCount: maxAttempts);
return(Policy
.Handle <ErrorResponseException>(e =>
{
var errorMessage = $"{e.GetType()}: {e.Message}";
log.LogError(e, $"Exception thrown: {errorMessage}");
// Handle throttling.
return e.Response.StatusCode == HttpStatusCode.TooManyRequests;
})
.WaitAndRetryAsync(delay));
}
protected override Task <HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken token)
{
if (request.Method == HttpMethod.Post || string.Equals(request.Method.Method, "PATCH"))
{
return(base.SendAsync(request, token));
}
var jittered = Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromMilliseconds(50), _retryCount, null, true);
return(Policy
.Handle <HttpRequestException>()
.OrResult <HttpResponseMessage>(r => (int)r.StatusCode >= 500 || r.StatusCode == HttpStatusCode.RequestTimeout)
.WaitAndRetryAsync(jittered)
.ExecuteAsync(() => base.SendAsync(request, token)));
}
public IEnumerator<TimeSpan> GetEnumerator()
{
var initial = Backoff.DecorrelatedJitterBackoffV2(MedianFirstRetryDelay, RetryCount, fastFirst: true);
foreach (var value in initial)
{
if (value < MaxDelay)
{
yield return value;
}
}
while (true)
{
yield return MaxDelay;
}
}
private static void AddClients(this IServiceCollection services)
{
var delay = Backoff.DecorrelatedJitterBackoffV2(medianFirstRetryDelay: TimeSpan.FromMilliseconds(100), retryCount: 10, fastFirst: true);
services.AddHttpClient <IAggregatesClient, AggregatesClient>()
.AddPolicyHandler((s, request) =>
Policy <HttpResponseMessage>
.Handle <ApiException>()
.Or <HttpRequestException>()
.OrTransientHttpStatusCode()
.WaitAndRetryAsync(delay,
onRetry: (result, timeSpan, retryCount, context) =>
{
var logger = Log.Logger.ForContext <AggregatesClient>();
LogFailure(logger, result, timeSpan, retryCount, context);
})
.WithPolicyKey("AggregatesClient"));
services.AddHttpClient <IExchangesClient, ExchangesClient>()
.AddPolicyHandler((s, request) =>
Policy <HttpResponseMessage>
.Handle <ApiException>()
.Or <HttpRequestException>()
.OrTransientHttpStatusCode()
.WaitAndRetryAsync(delay,
onRetry: (result, timeSpan, retryCount, context) =>
{
var logger = Log.Logger.ForContext <ExchangesClient>();
LogFailure(logger, result, timeSpan, retryCount, context);
})
.WithPolicyKey("ExchangesClient"));
services.AddHttpClient <IInstrumentsClient, InstrumentsClient>()
.AddPolicyHandler((s, request) =>
Policy <HttpResponseMessage>
.Handle <ApiException>()
.Or <HttpRequestException>()
.OrTransientHttpStatusCode()
.WaitAndRetryAsync(delay,
onRetry: (result, timeSpan, retryCount, context) =>
{
var logger = Log.Logger.ForContext <InstrumentsClient>();
LogFailure(logger, result, timeSpan, retryCount, context);
})
.WithPolicyKey("InstrumentsClient"));
}
public async Task IngestFromCsvAsync(
string csv, ServicePrincipalOptions servicePrincipal, string cluster, string database, string table, bool isDryRun)
{
KustoConnectionStringBuilder connectionBuilder =
new KustoConnectionStringBuilder($"https://{cluster}.kusto.windows.net")
.WithAadApplicationKeyAuthentication(
servicePrincipal.ClientId,
servicePrincipal.Secret,
servicePrincipal.Tenant);
using (IKustoIngestClient client = KustoIngestFactory.CreateDirectIngestClient(connectionBuilder))
{
KustoIngestionProperties properties =
new(database, table) { Format = DataSourceFormat.csv };
StreamSourceOptions sourceOptions = new() { SourceId = Guid.NewGuid() };
if (!isDryRun)
{
AsyncRetryPolicy retryPolicy = Policy
.Handle <Kusto.Data.Exceptions.KustoException>()
.Or <Kusto.Ingest.Exceptions.KustoException>()
.WaitAndRetryAsync(
Backoff.DecorrelatedJitterBackoffV2(TimeSpan.FromSeconds(10), RetryHelper.MaxRetries),
RetryHelper.GetOnRetryDelegate(RetryHelper.MaxRetries, _loggerService));
IKustoIngestionResult result = await retryPolicy.ExecuteAsync(
() => IngestFromStreamAsync(csv, client, properties, sourceOptions));
IngestionStatus ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
for (int i = 0; i < 10 && ingestionStatus.Status == Status.Pending; i++)
{
await Task.Delay(TimeSpan.FromSeconds(30));
ingestionStatus = result.GetIngestionStatusBySourceId(sourceOptions.SourceId);
}
if (ingestionStatus.Status == Status.Pending)
{
throw new InvalidOperationException($"Timeout while ingesting Kusto data.");
}
else if (ingestionStatus.Status != Status.Succeeded)
{
throw new InvalidOperationException(
$"Failed to ingest Kusto data.{Environment.NewLine}{ingestionStatus.Details}");
}
}
}
}
