public ResiliencePoliciesSettings() { OverallTimeoutPolicySettings = new OverallTimeoutPolicySettings(); TimeoutPerTryPolicySettings = new TimeoutPerTryPolicySettings(); RetrySettings = new RetryPolicySettings(); CircuitBreakerSettings = new CircuitBreakerPolicySettings(); }
public async Task Should_Open_Circuit_Breaker_for_RU_and_do_not_affect_EE() { const int retryCount = 5; const int minimumThroughput = 2; var settings = new ResiliencePoliciesSettings { OverallTimeout = TimeSpan.FromSeconds(5), RetryPolicySettings = RetryPolicySettings.Constant(retryCount, TimeSpan.FromMilliseconds(50)), CircuitBreakerPolicySettings = BuildCircuitBreakerSettings(minimumThroughput), }; var wrapper = Create.HttpClientWrapperWrapperBuilder .WithHostAndStatusCode("ru-prod.com", HttpStatusCode.ServiceUnavailable) .WithHostAndStatusCode("ee-prod.com", HttpStatusCode.OK) .WithResiliencePolicySettings(settings) .Please(); const int taskCount = 4; Assert.CatchAsync <BrokenCircuitException>(async() => await Helper.InvokeMultipleHttpRequests(wrapper.Client, taskCount, "http://ru-prod.com/Test1/Test2")); await Helper.InvokeMultipleHttpRequests(wrapper.Client, taskCount, "http://ee-prod.com/Test3/Test4/Test5"); Assert.AreEqual(minimumThroughput + taskCount, wrapper.NumberOfCalls); }
public WalletHistoryService( [NotNull] HistoryOperationPublisher historyOperationPublisher, [NotNull] IMerchantWalletService merchantWalletService, [NotNull] ILogFactory logFactory, [NotNull] RetryPolicySettings retryPolicySettings, [NotNull] IPayHistoryClient payHistoryClient) { _historyOperationPublisher = historyOperationPublisher ?? throw new ArgumentNullException(nameof(historyOperationPublisher)); _merchantWalletService = merchantWalletService ?? throw new ArgumentNullException(nameof(merchantWalletService)); _retryPolicySettings = retryPolicySettings ?? throw new ArgumentNullException(nameof(retryPolicySettings)); _payHistoryClient = payHistoryClient ?? throw new ArgumentNullException(nameof(payHistoryClient)); _log = logFactory.CreateLog(this); _publisherRetryPolicy = Policy .Handle <Exception>() .WaitAndRetryAsync( _retryPolicySettings.DefaultAttempts, attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)), (ex, timespan) => _log.Error(ex, "Publish wallet history with retry")); _clientRetryPolicy = Policy .Handle <Exception>(ex => !(ex is PayHistoryApiException)) .WaitAndRetryAsync( _retryPolicySettings.DefaultAttempts, attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)), (ex, timespan) => _log.Error(ex, "Connecting to history service with retry")); }
public EthereumIataApiClient( [NotNull] IEthereumCoreAPI ethereumServiceClient, [NotNull] EthereumBlockchainSettings ethereumSettings, [NotNull] IAssetsLocalCache assetsLocalCache, [NotNull] IAssetsService assetsService, [NotNull] ILykkeAssetsResolver lykkeAssetsResolver, [NotNull] ILogFactory logFactory, [NotNull] RetryPolicySettings retryPolicySettings) { _ethereumServiceClient = ethereumServiceClient ?? throw new ArgumentNullException(nameof(ethereumServiceClient)); _ethereumSettings = ethereumSettings ?? throw new ArgumentNullException(nameof(ethereumSettings)); _assetsLocalCache = assetsLocalCache ?? throw new ArgumentNullException(nameof(assetsLocalCache)); _assetsService = assetsService ?? throw new ArgumentNullException(nameof(assetsService)); _lykkeAssetsResolver = lykkeAssetsResolver ?? throw new ArgumentNullException(nameof(lykkeAssetsResolver)); _retryPolicySettings = retryPolicySettings ?? throw new ArgumentNullException(nameof(retryPolicySettings)); _log = logFactory.CreateLog(this); _retryPolicy = Policy .HandleResult <object>(r => { if (r is ApiException apiException) { return(apiException.Error?.Code == ExceptionType.None); } return(false); }) .WaitAndRetryAsync( _retryPolicySettings.DefaultAttempts, attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)), (ex, timespan) => _log.Error(message: "Connecting ethereum core with retry", context: ex)); }
public CashoutService( [NotNull] IAssetSettingsService assetSettingsService, [NotNull] IBcnSettingsResolver bcnSettingsResolver, [NotNull] ITransferService transferService, [NotNull] ITransactionsService transactionsService, [NotNull] IMerchantWalletService merchantWalletService, [NotNull] IWalletBalanceValidator walletBalanceValidator, [NotNull] IWalletHistoryService walletHistoryService, [NotNull] RetryPolicySettings retryPolicySettings, [NotNull] ILogFactory logFactory) { _assetSettingsService = assetSettingsService ?? throw new ArgumentNullException(nameof(assetSettingsService)); _bcnSettingsResolver = bcnSettingsResolver ?? throw new ArgumentNullException(nameof(bcnSettingsResolver)); _transferService = transferService ?? throw new ArgumentNullException(nameof(transferService)); _transactionsService = transactionsService ?? throw new ArgumentNullException(nameof(transactionsService)); _merchantWalletService = merchantWalletService ?? throw new ArgumentNullException(nameof(merchantWalletService)); _walletBalanceValidator = walletBalanceValidator ?? throw new ArgumentNullException(nameof(walletBalanceValidator)); _walletHistoryService = walletHistoryService ?? throw new ArgumentNullException(nameof(walletHistoryService)); _log = logFactory.CreateLog(this); _retryPolicy = Policy .Handle <InsufficientFundsException>() .Or <CashoutOperationFailedException>() .Or <CashoutOperationPartiallyFailedException>() .WaitAndRetryAsync( retryPolicySettings.DefaultAttempts, attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)), (ex, timespan) => _log.Error(ex, "Cashout with retry")); }
private static IHttpClientBuilder AddRetryPolicy( this IHttpClientBuilder clientBuilder, RetryPolicySettings settings) { return(clientBuilder .AddPolicyHandler(HttpPolicyExtensions .HandleTransientHttpError() .Or <TimeoutRejectedException>() .WaitAndRetryAsync(settings))); }
public PaymentRequestService( [NotNull] IPaymentRequestRepository paymentRequestRepository, [NotNull] IOrderService orderService, [NotNull] IPaymentRequestPublisher paymentRequestPublisher, [NotNull] ITransferService transferService, [NotNull] IPaymentRequestStatusResolver paymentRequestStatusResolver, [NotNull] ILogFactory logFactory, [NotNull] IWalletManager walletsManager, [NotNull] ITransactionsService transactionsService, [NotNull] ExpirationPeriodsSettings expirationPeriods, [NotNull] IMerchantWalletService merchantWalletService, [NotNull] IDistributedLocksService paymentLocksService, [NotNull] ITransactionPublisher transactionPublisher, [NotNull] IDistributedLocksService checkoutLocksService, [NotNull] IWalletBalanceValidator walletBalanceValidator, [NotNull] RetryPolicySettings retryPolicySettings, [NotNull] IAutoSettleSettingsResolver autoSettleSettingsResolver, [NotNull] IAssetSettingsService assetSettingsService) { _paymentRequestRepository = paymentRequestRepository ?? throw new ArgumentNullException(nameof(paymentRequestRepository)); _orderService = orderService ?? throw new ArgumentNullException(nameof(orderService)); _paymentRequestPublisher = paymentRequestPublisher ?? throw new ArgumentNullException(nameof(paymentRequestPublisher)); _transferService = transferService ?? throw new ArgumentNullException(nameof(transferService)); _paymentRequestStatusResolver = paymentRequestStatusResolver ?? throw new ArgumentNullException(nameof(paymentRequestStatusResolver)); _log = logFactory.CreateLog(this); _walletsManager = walletsManager ?? throw new ArgumentNullException(nameof(walletsManager)); _transactionsService = transactionsService ?? throw new ArgumentNullException(nameof(transactionsService)); _expirationPeriods = expirationPeriods ?? throw new ArgumentNullException(nameof(expirationPeriods)); _merchantWalletService = merchantWalletService ?? throw new ArgumentNullException(nameof(merchantWalletService)); _paymentLocksService = paymentLocksService ?? throw new ArgumentNullException(nameof(paymentLocksService)); _transactionPublisher = transactionPublisher ?? throw new ArgumentNullException(nameof(transactionPublisher)); _checkoutLocksService = checkoutLocksService ?? throw new ArgumentNullException(nameof(checkoutLocksService)); _walletBalanceValidator = walletBalanceValidator ?? throw new ArgumentNullException(nameof(walletBalanceValidator)); _autoSettleSettingsResolver = autoSettleSettingsResolver ?? throw new ArgumentNullException(nameof(autoSettleSettingsResolver)); _assetSettingsService = assetSettingsService ?? throw new ArgumentNullException(nameof(assetSettingsService)); _settlementRetryPolicy = Policy .Handle <InsufficientFundsException>() .Or <SettlementOperationFailedException>() .Or <SettlementOperationPartiallyFailedException>() .WaitAndRetryAsync( retryPolicySettings.SettlementAttempts, attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)), (ex, timespan) => _log.Error(ex, "Settlement with retry")); _paymentRetryPolicy = Policy .Handle <InsufficientFundsException>() .Or <PaymentOperationFailedException>() .Or <PaymentOperationPartiallyFailedException>() .WaitAndRetryAsync( retryPolicySettings.DefaultAttempts, attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)), (ex, timespan) => _log.Error(ex, "Payment with retry")); }
public AutofacModule( CacheExpirationPeriodsSettings cacheExpirationPeriods, DistributedCacheSettings distributedCacheSettings, RetryPolicySettings retryPolicySettings, string payInvoicePortalUrl) { _cacheExpirationPeriods = cacheExpirationPeriods; _distributedCacheSettings = distributedCacheSettings; _retryPolicySettings = retryPolicySettings; _payInvoicePortalUrl = payInvoicePortalUrl; }
public PushNotificationService( NotificationPublisher pushNotificationPublisher, RetryPolicySettings retryPolicySettings, ILogFactory logFactory) { _pushNotificationPublisher = pushNotificationPublisher; _log = logFactory.CreateLog(this); _retryPolicy = Policy .Handle <Exception>() .WaitAndRetryAsync( retryPolicySettings.DefaultAttempts, attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)), (ex, timespan) => _log.Error(ex, "Publish confirmations to callback with retry")); }
public async Task Should_retry_when_client_returns_500() { const int retryCount = 3; var retrySettings = RetryPolicySettings.Constant(retryCount, TimeSpan.FromMilliseconds(1)); var wrapper = Create.HttpClientWrapperWrapperBuilder .WithStatusCode(HttpStatusCode.InternalServerError) .WithRetrySettings(retrySettings) .Please(); await wrapper.Client.GetAsync("http://localhost"); Assert.AreEqual(retryCount + 1, wrapper.NumberOfCalls); }
public AutofacModule( [NotNull] ExpirationPeriodsSettings expirationPeriods, int transactionConfirmationCount, [NotNull] IList <BlockchainWalletAllocationPolicy> walletAllocationSettings, [NotNull] IReadOnlyList <AssetPairSetting> assetPairLocalStorageSettings, [NotNull] CacheSettings cacheSettings, [NotNull] RetryPolicySettings retryPolicySettings) { _expirationPeriods = expirationPeriods ?? throw new ArgumentNullException(nameof(expirationPeriods)); _transactionConfirmationCount = transactionConfirmationCount; _walletAllocationSettings = walletAllocationSettings ?? throw new ArgumentNullException(nameof(walletAllocationSettings)); _assetPairLocalStorageSettings = assetPairLocalStorageSettings ?? throw new ArgumentNullException(nameof(assetPairLocalStorageSettings)); _cacheSettings = cacheSettings ?? throw new ArgumentNullException(nameof(cacheSettings)); _retryPolicySettings = retryPolicySettings ?? throw new ArgumentNullException(nameof(retryPolicySettings)); }
public ConfirmationsService( [NotNull] InvoiceConfirmationPublisher invoiceConfirmationPublisher, [NotNull] RetryPolicySettings retryPolicySettings, [NotNull] ILogFactory logFactory) { _invoiceConfirmationPublisher = invoiceConfirmationPublisher ?? throw new ArgumentNullException(nameof(invoiceConfirmationPublisher)); _retryPolicySettings = retryPolicySettings ?? throw new ArgumentNullException(nameof(retryPolicySettings)); _log = logFactory.CreateLog(this); _retryPolicy = Policy .Handle <Exception>() .WaitAndRetryAsync( _retryPolicySettings.DefaultAttempts, attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)), (ex, timespan) => _log.Error(ex, "Publish invoice confirmation with retry")); }
public async Task Should_retry_6_times_for_two_threads_when_client_returns_503() { const int retryCount = 3; var retrySettings = RetryPolicySettings.Jitter(retryCount, medianFirstRetryDelay: TimeSpan.FromMilliseconds(50)); var wrapper = Create.HttpClientWrapperWrapperBuilder .WithStatusCode(HttpStatusCode.ServiceUnavailable) .WithRetrySettings(retrySettings) .Please(); const int taskCount = 2; await Helper.InvokeMultipleHttpRequests(wrapper.Client, taskCount); Assert.AreEqual((retryCount + 1) * taskCount, wrapper.NumberOfCalls); }
public async Task Should_retry_3_times_when_client_returns_503() { const int retryCount = 3; var retrySettings = RetryPolicySettings.Constant(retryCount, TimeSpan.FromMilliseconds(1)); var wrapper = Create.HttpClientWrapperWrapperBuilder .WithStatusCode(HttpStatusCode.ServiceUnavailable) .WithRetrySettings(retrySettings) .Please(); var result = await wrapper.Client.GetAsync("http://localhost"); Assert.AreEqual(HttpStatusCode.ServiceUnavailable, result.StatusCode); Assert.AreEqual(retryCount + 1, wrapper.NumberOfCalls); }
public HistoryOperationService( HistoryOperationPublisher historyOperationPublisher, IPayHistoryClient payHistoryClient, RetryPolicySettings retryPolicySettings, ILogFactory logFactory) { _historyOperationPublisher = historyOperationPublisher; _payHistoryClient = payHistoryClient; _log = logFactory.CreateLog(this); _retryPolicy = Policy .Handle <Exception>() .WaitAndRetryAsync( retryPolicySettings.DefaultAttempts, attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)), (ex, timespan) => _log.Error(ex, "Publish invoice payment to history with retry")); }
public async Task Should_catch_retry_in_OnRetry_handler_passed_before_RetryPolicySettings() { var retryCounter = 0; var settings = new ResiliencePoliciesSettings { OnRetry = (_, __) => { retryCounter++; }, RetryPolicySettings = RetryPolicySettings.Constant(Defaults.Retry.RetryCount), }; var wrapper = Create.HttpClientWrapperWrapperBuilder .WithStatusCode(HttpStatusCode.ServiceUnavailable) .WithResiliencePolicySettings(settings) .Please(); await wrapper.Client.GetAsync("http://localhost"); Assert.AreEqual(Defaults.Retry.RetryCount, retryCounter); }
public void Should_catch_timeout_because_of_overall_less_then_sleep_duration_of_RetryAfterDecorator() { const int retryCount = 3; var settings = new ResiliencePoliciesSettings { OverallTimeout = TimeSpan.FromSeconds(2), RetryPolicySettings = RetryPolicySettings.Constant(retryCount), }; var wrapper = Create.HttpClientWrapperWrapperBuilder .WithRetryAfterHeader(TimeSpan.FromSeconds(1)) .WithStatusCode(HttpStatusCode.InternalServerError) .WithResiliencePolicySettings(settings) .Please(); Assert.CatchAsync <TimeoutRejectedException>(async() => await wrapper.Client.GetAsync("http://localhost")); }
public Settings( int topSize, FundaApiClientSettings apiClientSettings, FundaApiSettings apiSettings, RetryPolicySettings retrySettings ) { if (topSize <= 0) { throw new ArgumentOutOfRangeException(nameof(topSize)); } TopSize = topSize; ApiClientSettings = apiClientSettings ?? throw new ArgumentNullException(nameof(apiClientSettings)); ApiSettings = apiSettings ?? throw new ArgumentNullException(nameof(apiSettings)); RetrySettings = retrySettings ?? throw new ArgumentNullException(nameof(retrySettings)); }
public void Should_retry_5_times_200_status_code_because_of_per_try_timeout() { const int retryCount = 5; var retrySettings = RetryPolicySettings.Constant(retryCount, TimeSpan.FromMilliseconds(200)); var wrapper = Create.HttpClientWrapperWrapperBuilder .WithStatusCode(HttpStatusCode.OK) .WithResponseLatency(TimeSpan.FromMilliseconds(200)) .WithTimeoutPerTry(TimeSpan.FromMilliseconds(100)) .WithRetrySettings(retrySettings) .Please(); Assert.CatchAsync <TimeoutRejectedException>(async() => await wrapper.Client.GetAsync("http://localhost")); Assert.AreEqual(retryCount + 1, wrapper.NumberOfCalls); }
public void Should_fail_on_HttpClient_timeout_with_retry() { const int retryCount = 5; var retrySettings = RetryPolicySettings.Constant(retryCount, TimeSpan.FromMilliseconds(1)); var wrapper = Create.HttpClientWrapperWrapperBuilder .WithStatusCode(HttpStatusCode.ServiceUnavailable) .WithResponseLatency(TimeSpan.FromMilliseconds(50)) .WithTimeoutOverall(TimeSpan.FromMilliseconds(100)) .WithRetrySettings(retrySettings) .Please(); Assert.CatchAsync <TimeoutRejectedException>(async() => await wrapper.Client.GetAsync("http://localhost")); Assert.AreEqual(2, wrapper.NumberOfCalls); }
public void Should_catch_timeout_1_times_because_of_overall_timeout_less_than_per_try_timeout() { const int retryCount = 5; var settings = new ResiliencePoliciesSettings { OverallTimeout = TimeSpan.FromMilliseconds(100), TimeoutPerTry = TimeSpan.FromMilliseconds(200), RetryPolicySettings = RetryPolicySettings.Constant(retryCount, TimeSpan.FromMilliseconds(200)), }; var wrapper = Create.HttpClientWrapperWrapperBuilder .WithStatusCode(HttpStatusCode.OK) .WithResponseLatency(TimeSpan.FromMilliseconds(300)) .WithResiliencePolicySettings(settings) .Please(); Assert.CatchAsync <TimeoutRejectedException>(async() => await wrapper.Client.GetAsync("http://localhost")); Assert.AreEqual(1, wrapper.NumberOfCalls); }
public async Task Should_retry_sleep_longer_when_RetryAfterDecorator_is_on() { const int retryCount = 3; var settings = new ResiliencePoliciesSettings { RetryPolicySettings = RetryPolicySettings.Constant(retryCount), }; var wrapper = Create.HttpClientWrapperWrapperBuilder .WithRetryAfterHeader(TimeSpan.FromSeconds(1)) .WithStatusCode(HttpStatusCode.InternalServerError) .WithResiliencePolicySettings(settings) .Please(); var stopWatch = Stopwatch.StartNew(); await wrapper.Client.GetAsync("http://localhost"); stopWatch.Stop(); Assert.That(3.0d, Is.GreaterThanOrEqualTo(stopWatch.Elapsed.TotalSeconds).Within(0.1)); }
public async Task Should_separately_distribute_retry_attempts_for_multiple_tasks() { const int retryCount = 3; var retryAttempts = new Dictionary <string, List <TimeSpan> >(); var settings = new ResiliencePoliciesSettings { RetryPolicySettings = RetryPolicySettings.Jitter(retryCount, TimeSpan.FromMilliseconds(50)), OnRetry = BuildOnRetryAction(retryAttempts), }; var wrapper = Create.HttpClientWrapperWrapperBuilder .WithStatusCode(HttpStatusCode.ServiceUnavailable) .WithResiliencePolicySettings(settings) .Please(); const int taskCount = 2; await Helper.InvokeMultipleHttpRequests(wrapper.Client, taskCount); CollectionAssert.AreNotEquivalent(retryAttempts.First().Value, retryAttempts.Last().Value); }
public void Should_break_after_4_concurrent_calls() { const int retryCount = 5; const int minimumThroughput = 2; var retrySettings = RetryPolicySettings.Constant(retryCount, TimeSpan.FromMilliseconds(50)); var wrapper = Create.HttpClientWrapperWrapperBuilder .WithStatusCode(HttpStatusCode.ServiceUnavailable) .WithTimeoutOverall(TimeSpan.FromSeconds(5)) .WithCircuitBreakerSettings(BuildCircuitBreakerSettings(minimumThroughput)) .WithRetrySettings(retrySettings) .Please(); const int taskCount = 4; Assert.CatchAsync <BrokenCircuitException>(async() => await Helper.InvokeMultipleHttpRequests(wrapper.Client, taskCount)); Assert.AreEqual(minimumThroughput, wrapper.NumberOfCalls); }
private static Settings GetSettings() { string apiKey = Environment.GetEnvironmentVariable("FUNDA_API_KEY"); var apiClientSettings = new FundaApiClientSettings(apiKey); const int BATCH_SIZE = 25; const int MAX_PAGES_TO_RETRIEVE = 5000; var apiSettings = new FundaApiSettings(BATCH_SIZE, MAX_PAGES_TO_RETRIEVE); const float DELAY = 2.0f; const float JITTER = 0.5f; const int RETRY_COUNT = 7; var retryPolicySettings = new RetryPolicySettings(DELAY, JITTER, RETRY_COUNT); const int TOP_SIZE = 10; return(new Settings(TOP_SIZE, apiClientSettings, apiSettings, retryPolicySettings)); }
public AssetsLocalCache( [NotNull] ILogFactory logFactory, [NotNull] IAssetsService assetsService, [NotNull] RetryPolicySettings retryPolicySettings, [NotNull] ExpirationPeriodsSettings expirationPeriodsSettings) { ILog log = logFactory.CreateLog(this); _assetsCache = new CachedDataDictionary <string, Asset> ( async() => { IList <Asset> assets = await Policy .Handle <Exception>() .WaitAndRetryAsync( retryPolicySettings.DefaultAttempts, attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)), (ex, timestamp) => log.Error("Getting assets dictionary with retry", ex)) .ExecuteAsync(() => assetsService.AssetGetAllAsync(true)); return(assets.ToDictionary(itm => itm.Id)); }, expirationPeriodsSettings.AssetsCache); _assetPairsCache = new CachedDataDictionary <string, AssetPair>( async() => { IList <AssetPair> assetPairs = await Policy .Handle <Exception>() .WaitAndRetryAsync( retryPolicySettings.DefaultAttempts, attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)), (ex, timestamp) => log.Error("Getting asset pairs dictionary with retry", ex)) .ExecuteAsync(() => assetsService.AssetPairGetAllAsync()); return(assetPairs.ToDictionary(itm => itm.Id)); }, expirationPeriodsSettings.AssetsCache); }
public CircuitBreakerMessageHandler(RetryPolicySettings settings, ILogger <CircuitBreakerMessageHandler> logger) { _settings = settings ?? throw new ArgumentNullException(nameof(settings)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); }