public static IHttpClientBuilder UseTimeoutPolicy(this IHttpClientBuilder builder, TimeSpan timeout)
{
TimeoutPolicy <HttpResponseMessage>?timeoutPolicy = Policy.TimeoutAsync <HttpResponseMessage>(timeout);
builder.AddPolicyHandler(timeoutPolicy);
return(builder);
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.AddControllers(options => options.Filters.Add(typeof(ExceptionFilter)))
// System.Text.Json does not support snake case strategy. Once it does (.NET 6), System.Text.Json should be used for better performance
.AddNewtonsoftJson(options =>
{
options.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore;
options.SerializerSettings.Converters.Add(new StringEnumConverter {
AllowIntegerValues = false
});
options.SerializerSettings.NullValueHandling = NullValueHandling.Ignore;
options.SerializerSettings.ContractResolver = new DefaultContractResolver
{
NamingStrategy = new SnakeCaseNamingStrategy()
};
JsonConvert.DefaultSettings = () => options.SerializerSettings;
});
services.AddAppMetricsToInfluxDb(Configuration)
.AddConfiguredApiVersioning();
services.AddSwaggerGen(options =>
{
options.SwaggerDoc("v1", new OpenApiInfo
{
Title = "Payment Getway API",
Description = "Payment Getway API",
Version = "1.0"
});
options.ExampleFilters();
options.IncludeXmlComments(Path.Combine(AppContext.BaseDirectory, "Documentation.xml"));
}).AddSwaggerGenNewtonsoftSupport()
.AddSwaggerExamplesFromAssemblyOf <Startup>();
services.AddSingleton(sp =>
{
var cfg = ConfigurationOptions.Parse(Configuration.GetValue <string>("RedisConnection"));
var connection = ConnectionMultiplexer.Connect(cfg);
return(connection);
});
services.AddTransient <PaymentService>();
services.AddTransient <IPaymentRepository, InMemoryPaymentRepository>();
services.AddTransient <IAcquiringBankConfigurationProvider>(p => new AcquiringBankConfigurationProvider(Configuration.GetSection(nameof(AcquiringBankConfiguration)).Get <AcquiringBankConfiguration>()));
services.AddTransient <IPaymentRequestDuplicateChecker, PaymentRequestDuplicateChecker>();
var httpPolicySettings = Configuration.GetSection(nameof(HttpClientPolicySettings)).Get <HttpClientPolicySettings>();
var acquiringBankCircuitBreaker = CircuitBreakerPolicy.Basic(httpPolicySettings.RetryCountBeforeBreaking, httpPolicySettings.DurationOfBreakInSeconds);
services.AddHttpClient <IAcquiringBank, AcquiringBank>()
.AddPolicyHandler((s, r) => Policy.WrapAsync(RetryPolicy.Basic(s.GetService <ILogger <IAcquiringBank> >(), r, httpPolicySettings.RetryCount),
RetryPolicy.HonouringRetry(s.GetService <ILogger <IAcquiringBank> >(), r, httpPolicySettings.RetryCount),
TimeoutPolicy.Basic(httpPolicySettings.RequestTimeoutInSeconds),
acquiringBankCircuitBreaker));
services.Replace(ServiceDescriptor.Singleton <IHttpMessageHandlerBuilderFilter, LoggingFilter>());
}
public PollyQoSProvider(ReRoute reRoute, IOcelotLoggerFactory loggerFactory)
{
_logger = loggerFactory.CreateLogger <PollyQoSProvider>();
_timeoutPolicy = Policy.TimeoutAsync(TimeSpan.FromMilliseconds(reRoute.QosOptionsOptions.TimeoutValue), reRoute.QosOptionsOptions.TimeoutStrategy);
_circuitBreakerPolicy = Policy
.Handle <HttpRequestException>()
.Or <TimeoutRejectedException>()
.Or <TimeoutException>()
.CircuitBreakerAsync(
exceptionsAllowedBeforeBreaking: reRoute.QosOptionsOptions.ExceptionsAllowedBeforeBreaking,
durationOfBreak: TimeSpan.FromMilliseconds(reRoute.QosOptionsOptions.DurationOfBreak),
onBreak: (ex, breakDelay) =>
{
_logger.LogError(
".Breaker logging: Breaking the circuit for " + breakDelay.TotalMilliseconds + "ms!", ex);
},
onReset: () =>
{
_logger.LogDebug(".Breaker logging: Call ok! Closed the circuit again.");
},
onHalfOpen: () =>
{
_logger.LogDebug(".Breaker logging: Half-open; next call is a trial.");
}
);
_circuitBreaker = new CircuitBreaker(_circuitBreakerPolicy, _timeoutPolicy);
}
public void BaseTest3()
{
TimeoutPolicy policy = new TimeoutPolicy(TimeSpan.Zero);
Assert.False(policy.IsValid());
Assert.Null(policy.GetSync());
Assert.Null(policy.GetAsync());
}
public void BaseTest1()
{
TimeoutPolicy policy = new TimeoutPolicy();
Assert.False(policy.IsValid());
Assert.Null(policy.GetSync());
Assert.Null(policy.GetAsync());
Assert.Equal(5, policy.Index);
}
public LoopAndPrintWithPollyTimeout()
{
timeoutPolicy = Policy
.Timeout(1, TimeoutStrategy.Pessimistic, onTimeout:
(context, span, t1, exception) =>
{
Console.WriteLine(exception.Message);
});
}
static ProxyConfigurator()
{
//TODO: Change this to use a policy registry?
RetryPolicy = HttpPolicyExtensions
.HandleTransientHttpError()
.RetryAsync(3);
TimeoutPolicy = Policy.TimeoutAsync <HttpResponseMessage>(TimeSpan.FromSeconds(10));
NoOpPolicy = Policy.NoOpAsync().AsAsyncPolicy <HttpResponseMessage>();
}
public CircuitBreakingHandler(int exceptionsAllowedBeforeBreaking,
TimeSpan durationOfBreak,
TimeSpan timeoutValue,
TimeoutStrategy timeoutStrategy = TimeoutStrategy.Pessimistic)
: base(string.Format("Handler:{0}", nameof(T)))
{
_exceptionsAllowedBeforeBreaking = exceptionsAllowedBeforeBreaking;
_durationOfBreak = durationOfBreak;
_timeoutBreakerPolicy = Policy.Timeout(timeoutValue, timeoutStrategy);
}
public void BaseTest4()
{
TimeoutPolicy policy = new TimeoutPolicy(TimeSpan.FromMilliseconds(2000));
Assert.True(policy.IsValid());
Assert.NotNull(policy.GetSync());
Assert.NotNull(policy.GetAsync());
Assert.Equal(TimeSpan.FromMilliseconds(2000).TotalMilliseconds, policy.TimeSpan.TotalMilliseconds);
}
public void BaseTest2()
{
TimeoutPolicy policy = new TimeoutPolicy(6);
Assert.True(policy.IsValid());
Assert.NotNull(policy.GetSync());
Assert.NotNull(policy.GetAsync());
Assert.Equal(6, policy.Seconds);
}
static void Wrap()
{
bool check = false;
RetryPolicy waitAndRetryPolicy = Policy
.Handle <Exception>()
.Retry(6,
onRetry: (exception, retryCount) =>
{
Console.WriteLine($"====Retry===== : 呼叫 API 異常, 進行第 {retryCount} 次重試");
if (retryCount == 5)
{
check = true;
}
});
TimeoutPolicy timeoutPolicys = Policy
.Timeout(TimeSpan.FromMilliseconds(1000), TimeoutStrategy.Pessimistic,
onTimeout: (context, timespan, task) =>
{
Console.WriteLine($"====TimeOut===== : execution timed out after {timespan} seconds.");
});
FallbackPolicy <String> fallbackForAnyException = Policy <String>
.Handle <Exception>()
.Fallback(
fallbackAction: () => { Console.WriteLine("999999999"); return("123"); },
onFallback: e => { Console.WriteLine($"[Polly fallback] : 重試失敗, say goodbye"); }
);
CircuitBreakerPolicy circuitPolicy = Policy
.Handle <Exception>()
.CircuitBreaker(3, TimeSpan.FromSeconds(0), (ex, ts) =>
{
Console.WriteLine($"====CircuitBreaker [OnBreak]===== ts = {ts.Seconds}s ,ex.message = {ex.Message}");
}, () =>
{
Console.WriteLine("AService OnReset");
});
try
{
PolicyWrap <String> policyWrap = fallbackForAnyException
.Wrap(waitAndRetryPolicy)
.Wrap(circuitPolicy)
.Wrap(timeoutPolicys);
policyWrap.Execute(() => doMockHTTPRequest(check));
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
public static PolicyWrap DefaultPolicy(int retryAttempts, int timeoutInSec)
{
// Set up Polly policy for retries, with increasing timeout between each attempt
RetryPolicy retryPolicy = Policy.Handle <HttpRequestException>()
.WaitAndRetry(retryAttempts, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)));
// Set up Polly timeout policy in case of network being down
TimeoutPolicy timeOutPolicy = Policy.Timeout(timeoutInSec, TimeoutStrategy.Pessimistic, (context, timeSpan, task) => ManageTimeoutException(context, timeSpan, task));
// Combine the two policies
return(Policy.Wrap(retryPolicy, timeOutPolicy));
}
public FtpTransferOperation(
OperationOptions oo,
ILoggerFactory loggerFactory)
{
_options = NamedNullException.Assert(oo, nameof(oo));
var fo = _options.Ftp;
NamedNullException.Assert(fo, nameof(fo));
_timeout = fo.Timeout.HasValue ? fo.Timeout.Value : FtpOptions.MaxFtpTimeout;
_ftpClient = new SimpleFtpClient(fo.FtpRoot, fo.UserName, fo.Password, _timeout);
_logger = NamedNullException.Assert(loggerFactory, nameof(loggerFactory)).CreateLogger <FtpTransferOperation>();
_timeoutPolicy = Policy.TimeoutAsync(TimeSpan.FromMilliseconds(_timeout), TimeoutStrategy.Pessimistic);
}
private PolicyWrap BuildProcessEventPolicy()
{
//重试策略
var retryPolicy = RetryPolicy
.Handle <Exception>()
.WaitAndRetry(_retryCount, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), (ex, time) =>
{
_logger.LogInformation($"Process event fails due to \"{ex.InnerException?.Message}\" and it will re-try after {time.TotalSeconds}s");
});
//超时策略
var timeoutPolicy = TimeoutPolicy.Timeout(30);
//组合策略
return(retryPolicy.Wrap(timeoutPolicy));
}
public CatalogController()
{
_timeoutPolicy = Policy.TimeoutAsync(1); // throws TimeoutRejectedException if timeout of 1 second is exceeded
_httpRetryPolicy =
Policy.HandleResult <HttpResponseMessage>(r => !r.IsSuccessStatusCode)
.Or <TimeoutRejectedException>()
.RetryAsync(3);
_httpRequestFallbackPolicy = Policy.HandleResult <HttpResponseMessage>(r => !r.IsSuccessStatusCode)
.Or <TimeoutRejectedException>()
.FallbackAsync(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new ObjectContent(_cachedResult.GetType(), _cachedResult, new JsonMediaTypeFormatter())
});
}
public CatalogController()
{
_timeoutPolicy = Policy.TimeoutAsync(1, onTimeoutAsync: TimeoutDelegate);
_httpRetryPolicy =
Policy.HandleResult <HttpResponseMessage>(r => !r.IsSuccessStatusCode)
.Or <TimeoutRejectedException>()
.RetryAsync(3, onRetry: HttpRetryPolicyDelegate);
_httpRequestFallbackPolicy = Policy.HandleResult <HttpResponseMessage>(r => !r.IsSuccessStatusCode)
.Or <TimeoutRejectedException>()
.FallbackAsync(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new ObjectContent(_cachedResult.GetType(), _cachedResult, new JsonMediaTypeFormatter())
}, onFallbackAsync: HttpRequestFallbackPolicyDelegate);
_policyWrap = _httpRequestFallbackPolicy.WrapAsync(_httpRetryPolicy.WrapAsync(_timeoutPolicy));
}
/// <summary>
/// 调用func with timeout，不会抛出异常
/// </summary>
/// <param name="func">调用的业务逻辑，func在task中执行(即和调用线程不是一个)</param>
/// <param name="timeoutSpan">多少时间后过期</param>
/// <param name="onTimeout">过期后的回调，过期后Polly不会杀死task，task仍然执行</param>
/// <returns></returns>
public static PolicyResult <T> RunWithTimeOut <T>(Func <T> func, TimeSpan timeoutSpan, Action <Context, TimeSpan, Task> onTimeout = null)
{
//TimeoutStrategy.Pessimistic recognises that there are cases where you need to execute delegates which have no in-built timeout, and do not honor cancellation.Policy使用Task执行user delegate
//在悲观模式下，Polly不会杀死底层的线程，而是将task传递到onTimeout回调中
TimeoutPolicy policy = Policy.Timeout(timeoutSpan, TimeoutStrategy.Pessimistic,
(context, timeSpan, task) =>
{
if (onTimeout != null)
{
onTimeout(context, timeSpan, task);
}
else
{
//过期后Polly不会杀死task，task仍然执行，需要在这里进行处理
}
});
return(policy.ExecuteAndCapture(func));
}
async Task <T> ExecutePostAsync <T>(string requestUri, HttpContent content, bool handlerEncrypted = true) where T : class
{
HttpClient client = ConfigureHttpClient(handlerEncrypted);
TimeSpan timeSpan = _minimunTimeUntilFail;
if (content is StreamContent)
{
timeSpan = _maximunTimeUntilFail;
}
CancellationToken cts = new CancellationTokenSource(timeSpan).Token;
TimeoutPolicy timeOutPolicy = Policy.TimeoutAsync(timeSpan);
using (content)
using (HttpResponseMessage response = await timeOutPolicy.WrapAsync(_policy).ExecuteAsync(async() => await client.PostAsync(requestUri, content, cts)))
{
return(await ProcessResponse <T>(response));
}
}
public PollyController(IReadOnlyPolicyRegistry <string> policyRegistry)
{
#region Configuracao API
var config = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json")
.Build();
_baseUri = config.GetSection("API:UrlBase").Get <string>();
#endregion
#region Policies da Resiliencia
_policyRegistry = policyRegistry;
_timeoutPolicy = Policy.TimeoutAsync(3);
_httpRetryPolicy =
Policy.HandleResult <HttpResponseMessage>(r => !r.IsSuccessStatusCode)
.Or <TimeoutRejectedException>()
.RetryAsync(6);
#endregion
}
public ResilientHttpClient(HttpClient client, CircuitBreakerPolicy <HttpResponseMessage> circuitBreakerPolicy)
{
_client = client;
_client.DefaultRequestHeaders.Accept.Clear();
_client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
//circuit breaker policy injected as defined in the Startup class
_circuitBreakerPolicy = circuitBreakerPolicy;
//Defining retry policy
_retryPolicy = Policy.HandleResult <HttpResponseMessage>(x =>
{
var result = !x.IsSuccessStatusCode;
return(result);
})
.Or <TimeoutException>()
//Retry 3 times and for each retry wait for 3 seconds
.WaitAndRetry(3, sleepDuration => TimeSpan.FromSeconds(3));
_fallbackCircuitBreakerPolicy = Policy <HttpResponseMessage>
.Handle <BrokenCircuitException>()
.Fallback(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("Please try again later[Circuit breaker is Open]")
}
);
_fallbackPolicy = Policy.HandleResult <HttpResponseMessage>(r => r.StatusCode == HttpStatusCode.InternalServerError)
.Or <TimeoutRejectedException>()
.Fallback(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("Some error occured")
});
_timeoutPolicy = Policy.Timeout <HttpResponseMessage>(1);
}
/// <summary>
/// Build default circuit breaker, wait and retry and Timeout policy.
/// </summary>
/// <param name="transientPolicySettings">retry,timeout and circuit breaker settings.</param>
/// <returns>policy wrapper.</returns>
public virtual PolicyWrap <HttpResponseMessage> GetDefaultPolicy(TransientPolicySettings transientPolicySettings)
{
//Time out policy to set Http Client request timeout.
TimeoutPolicy timeoutPolicy = Policy.TimeoutAsync(TimeSpan.FromSeconds(transientPolicySettings.HttpTimeOut), timeoutStrategy: TimeoutStrategy.Pessimistic, TimeOutHandler);
//If http client requiest fails, it will be retried.
IAsyncPolicy <HttpResponseMessage> retryPolicy = Policy.HandleResult <HttpResponseMessage>(r => !r.IsSuccessStatusCode)
.Or <HttpRequestException>()
.WaitAndRetryAsync(transientPolicySettings.RetryAttempts, retryafter => TimeSpan.FromSeconds(Math.Pow(2, retryafter) / 2), OnRetry);
//Circuit breaker to fail fast.
IAsyncPolicy <HttpResponseMessage> cb = HttpPolicyExtensions
.HandleTransientHttpError()
.AdvancedCircuitBreakerAsync(
transientPolicySettings.CircuitBreakerSettings.FailureThreshold,
TimeSpan.FromSeconds(transientPolicySettings.CircuitBreakerSettings.SamplingDuration),
transientPolicySettings.CircuitBreakerSettings.MinimumThroughput,
TimeSpan.FromSeconds(transientPolicySettings.CircuitBreakerSettings.DurationOfBreak),
OnBreak,
OnReset,
OnHalfOpen);
//above policies are wrapped.
return(timeoutPolicy.WrapAsync(retryPolicy.WrapAsync(cb)));
}
public PollyQoSProvider()
{
_timeoutPolicy = Policy.TimeoutAsync(TimeSpan.FromMilliseconds(20000));
_circuitBreakerPolicy = Policy
.Handle <HttpRequestException>()
.Or <TimeoutRejectedException>()
.Or <TimeoutException>()
.CircuitBreakerAsync(
exceptionsAllowedBeforeBreaking: 1,
durationOfBreak: TimeSpan.FromMilliseconds(2000),
onBreak: (ex, breakDelay) =>
{
},
onReset: () =>
{
},
onHalfOpen: () =>
{
}
);
_circuitBreaker = new CircuitBreaker(_circuitBreakerPolicy, _timeoutPolicy);
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.AddOptions();
services.AddApiVersioning(
o =>
{
o.AssumeDefaultVersionWhenUnspecified = true;
o.DefaultApiVersion = new ApiVersion(1, 0);
});
var azureStorageSettings = GetSettings <AzureStorageSettings>();
var notificationServiceSettings = GetSettings <NotificationServiceSettings>();
var faultToleranceSettings = GetSettings <FaultToleranceSettings>();
services.AddSingleton <IMetricsClient>(context =>
{
var metricsClientFactory = new MetricsClientFactory(context.GetRequiredService <ILoggerFactory>(),
TimeSpan.FromSeconds(15));
return(metricsClientFactory.CreateMetricsClient <LoggerMetricsClient>());
});
TimeoutPolicy timeoutPolicy = Policy.Timeout(faultToleranceSettings.TimeoutLength, TimeoutStrategy.Pessimistic);
CircuitBreakerPolicy circuitBreakerPolicy = Policy
.Handle <Exception>()
.CircuitBreaker(
exceptionsAllowedBeforeBreaking: faultToleranceSettings.ExceptionsAllowedBeforeBreaking,
durationOfBreak: TimeSpan.FromMinutes(faultToleranceSettings.DurationOfBreakInMinutes)
);
PolicyWrap policyWrap = Policy.Wrap(circuitBreakerPolicy, timeoutPolicy);
services.AddSingleton <ISyncPolicy>(policyWrap);
QueueClient queueClient = new QueueClient(notificationServiceSettings.ServiceBusConnectionString,
notificationServiceSettings.QueueName);
ServiceBusNotificationServiceClient notificationService = new ServiceBusNotificationServiceClient(queueClient);
services.AddSingleton <INotificationService>(context =>
new NotificationServiceMetricsDecorator(
new NotificationServiceFaultToleranceDecorator(
notificationService,
context.GetRequiredService <ISyncPolicy>()),
context.GetRequiredService <IMetricsClient>()));
AzureCloudTable profileCloudTable = new AzureCloudTable(azureStorageSettings.ConnectionString, azureStorageSettings.ProfilesTableName);
AzureTableProfileStore profileStore = new AzureTableProfileStore(profileCloudTable);
services.AddSingleton <IProfileStore>(context =>
new ProfileStoreMetricsDecorator(
new ProfileStoreFaultToleranceDecorator(
profileStore,
context.GetRequiredService <ISyncPolicy>()),
context.GetRequiredService <IMetricsClient>()));
AzureCloudTable messagesCloudTable = new AzureCloudTable(azureStorageSettings.ConnectionString, azureStorageSettings.MessagesTableName);
AzureTableMessagesStore messagesStore = new AzureTableMessagesStore(messagesCloudTable);
services.AddSingleton <IMessagesStore>(context =>
new MessagesStoreMetricsDecorator(
new MessagesStoreFaultToleranceDecorator(
messagesStore,
context.GetRequiredService <ISyncPolicy>()),
context.GetRequiredService <IMetricsClient>()));
AzureCloudTable conversationsCloudTable = new AzureCloudTable(azureStorageSettings.ConnectionString, azureStorageSettings.UsersTableName);
AzureTableConversationsStore conversationsStore = new AzureTableConversationsStore(conversationsCloudTable, messagesStore);
services.AddSingleton <IConversationsStore>(context =>
new ConversationStoreMetricsDecorator(
new ConversationsStoreFaultToleranceDecorator(
conversationsStore,
context.GetRequiredService <ISyncPolicy>()),
context.GetRequiredService <IMetricsClient>()));
services.AddSingleton <IConversationService>(context =>
new ConversationServiceMetricsDecorator(
new ConversationService(
context.GetRequiredService <IConversationsStore>(),
context.GetRequiredService <ILogger <ConversationService> >(),
context.GetRequiredService <INotificationService>()),
context.GetRequiredService <IMetricsClient>()));
services.AddLogging();
services.AddMvc();
}
/// <summary>
/// Sets the timeout policy.
/// </summary>
/// <param name="value">The timeout policy.</param>
/// <returns></returns>
public SubscriberBuilder TimeoutAction(TimeoutPolicy value)
{
Configuration.TimeoutPolicy = value;
return(this);
}
private static void RegisterResilientConsumers(IServiceCollection services)
{
const string baseUrl = "http://localhost:5000";
// Transient errors are: HttpRequestException, 5XX and 408
var backOffs = new[] { TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5) };
Action <DelegateResult <HttpResponseMessage>, TimeSpan, int, Context> onRetry = (result, timeSpan, retryCount, context) =>
{
Log.Warning($"Request failed with <{result.Result?.StatusCode}>. Waiting <{timeSpan}> before next retry. Retry attempt <{retryCount}>");
};
services.AddHttpClient("5000_exponentialbackoff").ConfigureHttpClient(httpClient => {
httpClient.BaseAddress = new Uri(baseUrl);
})
.AddTransientHttpErrorPolicy(builder =>
{
//builder.OrResult(msg => msg.StatusCode == HttpStatusCode.Continue);
return(builder.WaitAndRetryAsync(backOffs, onRetry));
});
services.AddHttpClient("5000_circuitbreaker").ConfigureHttpClient(httpClient => {
httpClient.BaseAddress = new Uri(baseUrl);
})
.AddTransientHttpErrorPolicy(builder =>
{
//builder.OrResult(msg => msg.StatusCode == HttpStatusCode.Continue);
return(builder.CircuitBreakerAsync(2, TimeSpan.FromSeconds(30), (response, timeSpan) => {
Log.Information($"Circuit is broken and will remain broken for <{timeSpan}>");
}, () => {
Log.Information("Circuit has been reset");
}));
});
services.AddHttpClient("5000_jitter").ConfigureHttpClient(httpClient => {
httpClient.BaseAddress = new Uri(baseUrl);
})
.AddTransientHttpErrorPolicy(builder =>
{
Random jitterer = new Random();
return(builder.WaitAndRetryAsync(5,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)) // exponential back-off: 2, 4, 8 etc
+ TimeSpan.FromMilliseconds(jitterer.Next(0, 1000)), // plus some jitter: up to 1 second
onRetry));
});
services.AddTransient <BulkheadExecutor>();
services.AddSingleton(provider => new Func <BulkheadExecutor>(() => provider.GetService <BulkheadExecutor>()));
services.AddHttpClient("5000_bulkhead").ConfigureHttpClient(httpClient => {
httpClient.BaseAddress = new Uri(baseUrl);
});
services.AddHttpClient("5000_fallback").ConfigureHttpClient(httpClient => {
httpClient.BaseAddress = new Uri(baseUrl);
})
.AddTransientHttpErrorPolicy(builder =>
{
return(builder.FallbackAsync(new HttpResponseMessage
{
Content = new StringContent("Fallback value returned")
}, result =>
{
Log.Error($"Fallback value substituted, due to: {result.Exception}.");
return Task.CompletedTask;
}));
});
services.AddHttpClient("5000_cache").ConfigureHttpClient(httpClient => {
httpClient.BaseAddress = new Uri(baseUrl);
});
services.AddHttpClient("5000_timeout").ConfigureHttpClient(httpClient => {
httpClient.BaseAddress = new Uri(baseUrl);
});
services.AddSingleton <PolicyRegistryExecutor>();
services.AddMemoryCache();
services.AddSingleton <IAsyncCacheProvider, MemoryCacheProvider>();
CachePolicy <HttpResponseMessage> cachePolicy = Policy.CacheAsync <HttpResponseMessage>(services.BuildServiceProvider().GetRequiredService <IAsyncCacheProvider>(), TimeSpan.FromMinutes(1),
(context, value) => Log.Information("OnCacheGet"),
(context, value) => Log.Information("OnCacheMiss"),
(context, value) => Log.Information("OnCachePut"),
(context, value, exception) => Log.Information("OnCacheGetError"),
(context, value, exception) => Log.Information("OnCachePutError"));
//TimeoutPolicy<HttpResponseMessage> timeoutPolicy = Policy.TimeoutAsync<HttpResponseMessage>(TimeSpan.FromSeconds(2).Seconds, TimeoutStrategy.Optimistic, (context, t, executedTask) =>
//{
// Log.Information("Timeout occurred");
// return Task.CompletedTask;
//});
TimeoutPolicy <HttpResponseMessage> timeoutPolicy = Policy.TimeoutAsync <HttpResponseMessage>(TimeSpan.FromSeconds(2).Seconds, TimeoutStrategy.Pessimistic, (context, t, executedTask) =>
{
Log.Information("Timeout occurred");
return(Task.CompletedTask);
});
services.AddSingleton <IPolicyRegistry <string>, PolicyRegistry>(serviceProvider =>
{
PolicyRegistry registry = new PolicyRegistry();
registry.Add("cachePolicy", cachePolicy);
registry.Add("timeoutPolicy", timeoutPolicy);
return(registry);
});
}
private async Task <BigInteger> IndexBlocksAsync(string indexerId, BigInteger currentBlockNumber,
Func <BigInteger, bool> checkDelegate)
{
try
{
if (_firstRun && _indexingSettings.From == currentBlockNumber)
{
await _logger.WriteInfoAsync
(
"BlockIndexingJob",
"IndexBlocksAsync",
indexerId,
$"Indexing begins from block-{currentBlockNumber}",
DateTime.UtcNow
);
var blockContent = await _rpcBlockReader.ReadBlockAsync(currentBlockNumber);
var blockContext = new BlockContext(Id, Version, indexerId, blockContent);
await _indexingService.IndexBlockAsync(blockContext);
currentBlockNumber++;
_firstRun = false;
}
var iterationVector = 0;
while (checkDelegate(currentBlockNumber))
{
BlockContent blockContent = null;
var transactionCount = 0;
await _logger.WriteInfoAsync
(
"BlockIndexingJob",
"IndexBlocksAsync",
indexerId,
$"Indexing block-{currentBlockNumber}, Vector:{iterationVector}",
DateTime.UtcNow
);
//Would throw on time out
await RetryPolicy.ExecuteAsync(async() =>
{
await _logger.WriteInfoAsync
(
"BlockIndexingJob",
"IndexBlocksAsync",
indexerId,
$"Block-{currentBlockNumber}, Vector:{iterationVector} Reading info",
DateTime.UtcNow
);
//Throws Timeout Exception
await TimeoutPolicy.ExecuteAsync(async() =>
{
blockContent = blockContent ?? await _rpcBlockReader.ReadBlockAsync(currentBlockNumber);
}, TimeSpan.FromMinutes(7));
await _logger.WriteInfoAsync
(
"BlockIndexingJob",
"IndexBlocksAsync",
indexerId,
$"Checking existence of the parent-{currentBlockNumber}",
DateTime.UtcNow
);
var blockContext = new BlockContext(Id, Version, indexerId, blockContent);
var blockExists = await _blockService.DoesBlockExist(blockContent.BlockModel.ParentHash);
transactionCount = blockContent.Transactions.Count;
iterationVector = blockExists ? 1 : -1; //That is how we deal with forks
await _logger.WriteInfoAsync
(
"BlockIndexingJob",
"IndexBlocksAsync",
indexerId,
$"Indexing block in DB -{currentBlockNumber}",
DateTime.UtcNow
);
// Throws Timeout Exception
await TimeoutPolicy.ExecuteAsync(async() =>
{
await _indexingService.IndexBlockAsync(blockContext);
}, TimeSpan.FromMinutes(5));
}, 5, 100);
await _logger.WriteInfoAsync
(
"BlockIndexingJob",
"IndexBlocksAsync",
indexerId,
$"Indexing completed for block-{currentBlockNumber}, Vector:{iterationVector}, transaction count - {transactionCount}",
DateTime.UtcNow
);
currentBlockNumber += iterationVector;
}
}
catch (Exception e)
{
if (e is BlockIsNotYetMinedException)
{
throw;
}
await _logger.WriteErrorAsync
(
"BlockIndexingJob",
"RunAsync",
$"Indexing failed for block-{currentBlockNumber}",
e,
DateTime.UtcNow
);
throw;
}
return(currentBlockNumber);
}
public void CombinePolicyConfigsTest()
{
var type = typeof(PolicyFactory);
PolicyFactory factory = new PolicyFactory();
string methodName = "CombinePolicyConfigs";
var configs = (List <IPolicyConfig>)Helper.RunInstanceMethod(type, methodName, factory, new object[3] {
null, null, null
});
Assert.Empty(configs);
var factoryPolicies = new List <IPolicyConfig>()
{
};
var apiPolicies = new List <IPolicyConfig>()
{
};
var sortIndexs = new List <int>();
var configs2 = (List <IPolicyConfig>)Helper.RunInstanceMethod(type, methodName, factory, new object[3] {
factoryPolicies, apiPolicies, sortIndexs
});
Assert.Empty(configs2);
RetryPolicy retryPolicy = new RetryPolicy(5);
factoryPolicies.Add(retryPolicy);
var configs3 = (List <IPolicyConfig>)Helper.RunInstanceMethod(type, methodName, factory, new object[3] {
factoryPolicies, apiPolicies, sortIndexs
});
Assert.NotEmpty(configs3);
Assert.Single(configs3);
TimeoutPolicy timeoutPolicy = new TimeoutPolicy(300);
apiPolicies.Add(timeoutPolicy);
var configs4 = (List <IPolicyConfig>)Helper.RunInstanceMethod(type, methodName, factory, new object[3] {
factoryPolicies, apiPolicies, sortIndexs
});
Assert.NotEmpty(configs4);
Assert.Equal(2, configs4.Count);
Assert.Equal(retryPolicy, configs4[0]);
RetryPolicy apiRetryPolicy = new RetryPolicy(8);
apiPolicies.Add(apiRetryPolicy);
var configs5 = (List <IPolicyConfig>)Helper.RunInstanceMethod(type, methodName, factory, new object[3] {
factoryPolicies, apiPolicies, sortIndexs
});
Assert.Equal(2, configs5.Count);
Assert.Equal(apiRetryPolicy, configs5[0]);
sortIndexs = new List <int>()
{
5, 4, 3, 2, 1
};
var configs6 = (List <IPolicyConfig>)Helper.RunInstanceMethod(type, methodName, factory, new object[3] {
factoryPolicies, apiPolicies, sortIndexs
});
Assert.NotEmpty(configs6);
Assert.Equal(timeoutPolicy, configs6[0]);
}
public CircuitBreaker(CircuitBreakerPolicy circuitBreakerPolicy, TimeoutPolicy timeoutPolicy)
{
CircuitBreakerPolicy = circuitBreakerPolicy;
TimeoutPolicy = timeoutPolicy;
}