protected Processor(AsyncCircuitBreakerPolicy circuitBreakerPolicy,
ClusterOptions clusterOptions,
ILogger logger)
{
Logger = logger;
CircuitBreakerPolicy = circuitBreakerPolicy;
ClusterOptions = clusterOptions;
var interval = new Subject<Unit>();
var scheduler = Scheduler.Default;
_timerSubscription = interval.Select(_ => Observable.Interval(TimeSpan.FromSeconds(5)))
.Switch()
.Select(duration => Observable.FromAsync(ComputeMetrics))
.Switch()
.ObserveOn(scheduler).Subscribe();
interval.OnNext(Unit.Default);
switch (ClusterOptions.LimitCpuUsage)
{
case int limit when limit >= 80:
ThreadPriority = ThreadPriority.Normal;
break;
case int limit when limit >= 40:
ThreadPriority = ThreadPriority.BelowNormal;
break;
case int limit when limit >= 0:
ThreadPriority = ThreadPriority.Lowest;
break;
default:
ThreadPriority = ThreadPriority.Normal;
break;
}
EvictedItemsSubject = new Subject<int>();
_evictedItemsSubscription = EvictedItemsSubject.Subscribe(itemEvicted => { _itemsEvicted = itemEvicted; });
}
private void InitializePollyIfRequired(IServiceScope scope)
{
if (circuitBreaker == null)
{
ServiceDetailTracker.WriteStatus(false, DateTime.Now, "Normal start");
var configurationProvider = scope.ServiceProvider.GetService <IRocketConfigurationProvider>();
circuitBreaker = Policy
.Handle <Exception>()
.CircuitBreakerAsync(
1,
TimeSpan.FromMilliseconds(configurationProvider.TaskRunnerSettings.CircuitBreakerDelayMilliSeconds),
(exception, timeSpan, context) =>
{
onDemandQueueManager.DeQueueAll();
_ = timeSpan;
_ = context;
var message = $"Failure reason: '{exception.Message}'";
ServiceDetailTracker.WriteStatus(true, DateTime.Now, message);
},
(context) =>
{
_ = context;
ServiceDetailTracker.WriteStatus(false, DateTime.Now, "Resumed after failure");
}
);
}
}
public ProxyController(IHttpClientFactory httpClientFactory)
{
_fallbackPolicy = Policy <IActionResult>
.Handle <Exception>()
.FallbackAsync(Content("Sorry, we are currently experiencing issues. Please try again later"));
_retryPolicy = Policy <IActionResult>
.Handle <Exception>()
.RetryAsync();
if (_circuitBreakerPolicy == null)
{
_circuitBreakerPolicy = Policy
.Handle <Exception>()
.CircuitBreakerAsync(2, TimeSpan.FromMinutes(1));
}
_policy = Policy <IActionResult>
.Handle <Exception>()
.FallbackAsync(Content("Sorry, we are currently experiencing issues. Please try again later"))
.WrapAsync(_retryPolicy)
.WrapAsync(_circuitBreakerPolicy);
_httpClient = httpClientFactory.CreateClient();
}
private Weather GetWeatheryWithPolly(string city)
{
AsyncCircuitBreakerPolicy breaker = Policy
.Handle <OpenWeatherMap.OpenWeatherMapException>()
.CircuitBreakerAsync(
exceptionsAllowedBeforeBreaking: 2,
durationOfBreak: TimeSpan.FromMinutes(1)
);
var timeoutPolicy = Policy
.TimeoutAsync(new TimeSpan(0, 0, 2), TimeoutStrategy.Pessimistic); // Pessimistic strategy as openWeatherClient doesn't support CancellationToken
var fallBackPolicy = Policy <OpenWeatherMap.CurrentWeatherResponse>
.Handle <Exception>(IsFallbackPolicyEnabled)
.FallbackAsync <OpenWeatherMap.CurrentWeatherResponse>((ct) => DoFallbackAction(ct, city));
var simmyPolicy = GetSimmyLatencyPolicy();
var policyWrap = fallBackPolicy // fallbackPolicy must be the innermost to work for all cases!!
.WrapAsync(breaker)
.WrapAsync(timeoutPolicy)
.WrapAsync(simmyPolicy); // simmyPolicy must be the outermost to trigger all internal policies!
var result = policyWrap.ExecuteAsync(async()
=> await openWeatherClient.CurrentWeather.GetByName(city, OpenWeatherMap.MetricSystem.Metric)).Result;
var weather = WeatherConverter.GetWeatherFromResponse(result);
return(weather);
}
internal static AsyncPolicyWrap <object> GetAsyncFallBackPolicy(
INHibernateLogger logger,
AsyncCircuitBreakerPolicy circuitBreaker)
{
return(Policy <object>
.Handle <BrokenCircuitException>()
.Or <Alachisoft.NCache.Runtime.Exceptions.CacheException>(ex =>
ex.ErrorCode == NCacheErrorCodes.NO_SERVER_AVAILABLE)
.FallbackAsync(
fallbackAction:
(ctx, ct) =>
{
return Task.FromResult((object)null);
},
onFallbackAsync:
(res, ctx) =>
{
if (logger.IsDebugEnabled())
{
logger.Debug(
$"AsyncFallback policy:\n" +
$"\tRegion: {ctx["region"]}" +
$"\tOperation: {ctx["operation"]}\n" +
$"\tFallback result: {res.Result ?? "null"}");
}
return Task.CompletedTask;
}
)
.WrapAsync(circuitBreaker));
}
public PrimeController(
IMapper mapper,
IMediator mediator,
ILinksService linksService,
ILoggingManager loggingManager,
PolicyConfiguration policyConfiguration)
{
_mapper = mapper;
_mediator = mediator;
_linksService = linksService;
_loggingManager = loggingManager;
_policyConfiguration = policyConfiguration;
_circuitBreakerPolicy = Policy
.Handle <TaskCanceledException>()
.CircuitBreakerAsync(policyConfiguration.CircuitBreakerAllowedExceptions, TimeSpan.FromMinutes(_policyConfiguration.CircuitBreakerShutdownDurationInMinutes),
(ex, t) =>
{
_loggingManager.LogFatal("Circuit broken!");
},
() =>
{
_loggingManager.LogInformation("Circuit Reset!");
});
}
/// <summary>
/// <see cref="AsyncProcessor{TInput,TOutput,TAsync}"/>
/// </summary>
/// <param name="circuitBreakerPolicy"><see cref="CircuitBreakerPolicy"/></param>
/// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
/// <param name="cts"><see cref="CancellationTokenSource"/></param>
/// <param name="logger"><see cref="ILogger"/></param>
protected AsyncProcessor(AsyncCircuitBreakerPolicy circuitBreakerPolicy,
ClusterOptions clusterOptions,
CancellationTokenSource cts,
ILogger logger) : base(circuitBreakerPolicy, clusterOptions, logger)
{
// We observe new items on an EventLoopScheduler which is backed by a dedicated background thread
// Then we process items asynchronously, with a circuit breaker policy
ItemsSubjectSubscription = SynchronizedItemsSubject
.ObserveOn(new EventLoopScheduler(ts => new Thread(ts)
{
IsBackground = true, Priority = ThreadPriority
}))
.Select(item =>
{
return(Observable.FromAsync(() =>
{
// ExecuteAndCaptureAsync let items to be "captured" in a way they never throw any exception, but are gracefully handled by a circuit breaker policy on non-success attempt
return CircuitBreakerPolicy.ExecuteAndCaptureAsync(
ct => Process(item, ct), cts.Token);
}));
})
.Merge()
.Subscribe(unit =>
{
if (unit.Outcome == OutcomeType.Failure)
{
Logger.LogCritical(
unit.FinalException != null
? $"Could not process bulk: {unit.FinalException.Message}."
: "An error has occured while processing the bulk.");
}
},
ex => Logger.LogError(ex.Message));
}
public HomeController(ILogger <HomeController> logger)
{
_logger = logger;
_retryPolicy = Policy.HandleResult <HttpResponseMessage>(result =>
!result.IsSuccessStatusCode).RetryAsync(5, (a, b) => {
_logger.LogInformation("Retry");
});
_circuitBreakerPolicy = Policy.HandleResult <HttpResponseMessage>
(r => !r.IsSuccessStatusCode).Or <HttpRequestException>()
.CircuitBreakerAsync(2, TimeSpan.FromSeconds(10),
(d, c) => {
string a = "Break";
},
() => {
string a = "Rest";
},
() => {
string a = "Half";
}
);
_fallbackPolicy = Policy.HandleResult <HttpResponseMessage>(r => !r.IsSuccessStatusCode)
.Or <BrokenCircuitException>()
.FallbackAsync(new HttpResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = new ObjectContent(typeof(Message), new Message
{
Id = 100,
Text = "Default Text"
}, new JsonMediaTypeFormatter())
});
}
public virtual void AddPolicies(IServiceCollection services)
{
services.AddSingleton <IReadOnlyPolicyRegistry <string>, PolicyRegistry>((serviceProvider) =>
{
PolicyRegistry registry = new PolicyRegistry();
Action <Exception, TimeSpan> onBreak = (exception, timespan) =>
{
serviceProvider.GetRequiredService <ILogger>().Error($"Utilizando CircuitBreaker durante {timespan.TotalSeconds} segundo(s) tras el error: {exception}");
};
Action onReset = () =>
{
serviceProvider.GetRequiredService <ILogger>().Info($"Finalizando CircuitBreaker.");
};
AsyncCircuitBreakerPolicy breaker = Policy
.Handle <Exception>()
.CircuitBreakerAsync(2, TimeSpan.FromSeconds(30), onBreak, onReset);
AsyncPolicy policyCache = Policy
.Handle <Exception>()
.RetryAsync(3, onRetry: (exception, retryCount) =>
{
serviceProvider.GetRequiredService <ILogger>().Warn($"Error en el intento {retryCount}: {exception}");
})
.WrapAsync(breaker);
registry.Add(Claves.CLAVE_POLITICA_CACHE, policyCache);
return(registry);
});
}
/// <summary>
/// <see cref="DirectSequentialProcessor{TInput}"/>
/// </summary>
/// <param name="circuitBreakerPolicy"><see cref="CircuitBreakerPolicy"/></param>
/// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
/// <param name="progress">Progress of the current bulk</param>
/// <param name="cts"><see cref="CancellationTokenSource"/></param>
/// <param name="logger"><see cref="ILogger"/></param>
protected DirectSequentialProcessor(AsyncCircuitBreakerPolicy circuitBreakerPolicy,
ClusterOptions clusterOptions,
IProgress <double> progress,
CancellationTokenSource cts,
ILogger logger) : base(circuitBreakerPolicy, clusterOptions, logger)
{
ItemsSubjectSubscription = SynchronizedItemsSubject
.ObserveOn(new EventLoopScheduler(ts => new Thread(ts)
{
IsBackground = true, Priority = ThreadPriority
}))
.Select(item =>
{
return(Observable.FromAsync(() =>
{
return CircuitBreakerPolicy.ExecuteAndCaptureAsync(
ct => Process(item, progress, ct), cts.Token);
}));
})
// Dequeue sequentially
.Concat()
.Subscribe(unit =>
{
if (unit.Outcome == OutcomeType.Failure)
{
Logger.LogCritical(
unit.FinalException != null
? $"Could not process item: {unit.FinalException.Message}."
: "An error has occured while processing the item.");
}
},
ex => Logger.LogError(ex.Message));
ItemsExecutorSubjectSubscription = SynchronizedItemsExecutorSubject
.ObserveOn(new EventLoopScheduler(ts => new Thread(ts)
{
IsBackground = true, Priority = ThreadPriority
}))
.Select(item =>
{
return(Observable.FromAsync(() =>
{
return CircuitBreakerPolicy.ExecuteAndCaptureAsync(
ct => Process(item, progress, ct), cts.Token);
}));
})
// Dequeue sequentially
.Concat()
.Subscribe(unit =>
{
if (unit.Outcome == OutcomeType.Failure)
{
Logger.LogCritical(
unit.FinalException != null
? $"Could not process item: {unit.FinalException.Message}."
: "An error has occured while processing the item.");
}
},
ex => Logger.LogError(ex.Message));
}
public void Should_retry_until_threshold_is_reached()
{
AsyncCircuitBreakerPolicy breaker = Policy
.Handle <Exception>()
.CircuitBreakerAsync(3,
TimeSpan.FromMilliseconds(1000),
OnBreak, OnReset);
AsyncPolicyWrap retryPolicy = Policy.Handle <Exception>().RetryAsync(3)
.WrapAsync(breaker);
//var policy = Policy
// .Handle<TaskCanceledException>()
// .Or<FailedConnectionToBankException>()
// .FallbackAsync(cancel => ReturnWillHandleLater(payment.GatewayPaymentId, payment.RequestId))
// .WrapAsync(breaker);
int count = 0;
retryPolicy.ExecuteAsync(async() =>
{
Console.WriteLine(count++);
await Task.FromException(new Exception());
});
}
public HAPublisher(IPublisher primaryPublisher, IPublisher secondaryPublisher)
{
this.primaryPublisher = primaryPublisher;
this.secondaryPublisher = secondaryPublisher;
this.exponentialRetryPolicy = Policy
.Handle <TimeoutException>()
.WaitAndRetryAsync(
2,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, timeSpan, context) => {
Logger.Log(string.Format("Retry for ex={0}", exception.GetType().Name), LoggerState.Warning);
});
this.circuitBreakerPolicy = Policy.Handle <Exception>().CircuitBreakerAsync(
2,
TimeSpan.FromSeconds(20),
(context, ts) => { Logger.Log("CircuitBreaker Closed to Open", LoggerState.Warning); },
() => { Logger.Log("CircuitBreaker Reset", LoggerState.Warning); });
this.fallbackPolicy = Policy
.Handle <Exception>()
.FallbackAsync(async(ctx, ct) => {
await this.PublishSecondaryAsync(ctx["message"].ToString());
}, async(ex, ctx) => {
Logger.Log(string.Format("Executing fallback for Ex={0}", ex.GetType().Name), LoggerState.Warning);
await Task.FromResult(0);
});
}
public CatalogController(HttpClient httpClient, AsyncCircuitBreakerPolicy <HttpResponseMessage> breakerPolicy)
{
_httpClient = httpClient;
_breakerPolicy = breakerPolicy;
_httpRetryPolicy = Policy.HandleResult <HttpResponseMessage>(r => !r.IsSuccessStatusCode).RetryAsync(2);
}
public DemoController(HttpClient httpClient, AsyncCircuitBreakerPolicy <HttpResponseMessage> breakerPolicy)
{
_httpClient = httpClient;
_breakerPolicy = breakerPolicy;
_httpRetryPolicy =
Policy.HandleResult <HttpResponseMessage>(r => !r.IsSuccessStatusCode)
.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)));
}
public Producer(IAsyncCollector <EventData> messagesCollector, ILogger logger)
{
MessagesCollector = messagesCollector;
Logger = logger;
circuitBreakerPolicy = CreateCircuitBreakerPolicy();
retryPolicy = CreateRetryPolicy();
}
public CacheService(IDistributedCache cache, IOptions <PollySettings> pollySettings)
{
_cache = cache;
_cachingPolicy = Policy.Handle <RedisConnectionException>()
.Or <RedisTimeoutException>()
.CircuitBreakerAsync(pollySettings.Value.RepeatedTimes,
TimeSpan.FromSeconds(pollySettings.Value.RepeatedDelay));
}
protected AsyncAbstractQueueProcessor(AsyncCircuitBreakerPolicy circuitBreakerPolicy,
ClusterOptions clusterOptions,
ILogger logger) : base(circuitBreakerPolicy, clusterOptions, logger)
{
ISubject <TAsync> itemsSubject = new Subject <TAsync>();
// SynchronizedItemsSubject is a thread-safe object in which we can push items concurrently
SynchronizedItemsSubject = Subject.Synchronize(itemsSubject);
}
public ExceptionMiddleware(RequestDelegate next, ILogger <ExceptionMiddleware> logger)
{
_next = next;
_logger = logger;
_policy = Policy
.Handle <Exception>()
.CircuitBreakerAsync(5, TimeSpan.FromSeconds(30));
}
public HttpClientFactoryService(IHttpClientFactory httpClientFactory, AsyncCircuitBreakerPolicy <HttpResponseMessage> circuitBreaker)
{
//add in Service Configuration
_httpClientFactory = httpClientFactory;
_circuitBreaker = circuitBreaker;
//init policy
_retryPolicy = Policy.Handle <HttpRequestException>().RetryAsync(MaxRetries);
}
public AdvancedCircuitBreakerHealth(IReadOnlyPolicyRegistry <string> registry)
{
if (registry == null)
{
throw new ArgumentNullException(nameof(registry));
}
_circuitBreakerPolicy = registry.Get <AsyncCircuitBreakerPolicy <HttpResponseMessage> >(PolicyConstants.ADVANCED_CIRCUITBREAKER_POLICY_NAME);
}
public Policies()
{
RequestTimeoutRetryPolicy = HttpPolicyExtensions
.HandleTransientHttpError()
.WaitAndRetryAsync(3, ra => TimeSpan.FromSeconds(Math.Pow(0.2, ra)));
RequestTimeoutCircuitBreaker = HttpPolicyExtensions
.HandleTransientHttpError()
.CircuitBreakerAsync(5, TimeSpan.FromSeconds(30));
}
protected DirectAbstractProcessor(AsyncCircuitBreakerPolicy circuitBreakerPolicy,
ClusterOptions clusterOptions,
ILogger logger) : base(circuitBreakerPolicy, clusterOptions, logger)
{
ISubject <TInput> itemsSubject = new Subject <TInput>();
ISubject <Func <TInput> > itemsExecutorSubject = new Subject <Func <TInput> >();
// SynchronizedItems are thread-safe objects in which we can push items concurrently
SynchronizedItemsSubject = Subject.Synchronize(itemsSubject);
SynchronizedItemsExecutorSubject = Subject.Synchronize(itemsExecutorSubject);
}
private AsyncPolicyWrap <U> GetFallBackPolicy <U>(
AsyncCircuitBreakerPolicy circuitBreakerPolicy) where U : class
{
return(Policy <U>
.Handle <BrokenCircuitException>()
.Or <CacheException>(ex =>
ex.ErrorCode == NCacheErrorCodes.NO_SERVER_AVAILABLE
)
.FallbackAsync((U)null)
.WrapAsync(circuitBreakerPolicy));
}
/// <summary>
/// Circuit Breaker Policy
/// </summary>
/// <param name="maxExceptions">Max exceptions allowed</param>
/// <param name="durationOfBreak">Duration of break in seconds</param>
/// <returns></returns>
public ArtesianPolicyConfig CircuitBreakerPolicyConfig(int maxExceptions = MaxExceptionsDefault, int durationOfBreak = DurationOfBreakDefault)
{
_circuitBreakerPolicy = Policy
.Handle <Exception>(x => x.InnerException is HttpRequestException)
.CircuitBreakerAsync(
exceptionsAllowedBeforeBreaking: maxExceptions,
durationOfBreak: TimeSpan.FromSeconds(durationOfBreak)
);
return(this);
}
private AsyncPolicyWrap <string> GetFallBackPolicy(
AsyncCircuitBreakerPolicy circuitBreakerPolicy)
{
return(Policy <string>
.Handle <BrokenCircuitException>()
.Or <CacheException>(ex =>
ex.ErrorCode == NCacheErrorCodes.NO_SERVER_AVAILABLE
)
.FallbackAsync(CACHE_PROBLEMS)
.WrapAsync(circuitBreakerPolicy));
}
private AsyncPolicyWrap <bool> SetFallBackPolicy(
AsyncCircuitBreakerPolicy circuitBreakerPolicy)
{
return(Policy <bool>
.Handle <BrokenCircuitException>()
.Or <CacheException>(ex =>
ex.ErrorCode == NCacheErrorCodes.NO_SERVER_AVAILABLE
)
.FallbackAsync(false)
.WrapAsync(circuitBreakerPolicy));
}
public MessageService(IMessageRepository messageRepository)
{
Trace.WriteLine("MessageService running");
_messageRepository = messageRepository;
// Политика повторителя.
_retryPolicy = Policy
.Handle <Exception>()
// retryCount - указывает, сколько раз вы хотите повторить попытку.
// sleepDurationProvider - делегат, который определяет, как долго ждать перед повторной попыткой.
.WaitAndRetryAsync(2, retryAttempt =>
{
// Экспоненциальное время ожидания.
var timeToWait = TimeSpan.FromSeconds(Math.Pow(2, retryAttempt));
Trace.WriteLine($"Waiting {timeToWait.TotalSeconds} seconds");
return(timeToWait);
});
// Политика выключателя.
// Обращаемся к репозиторию, через объединение политик.
// В случае ошибки, выключатель перейдет в открытое состояние, на указанное время.
_circuitBreakerPolicy = Policy <string>
.Handle <Exception>()
// exceptionsAllowedBeforeBreaking - указывает, сколько исключений подряд вызовет разрыв цепи.
// durationOfBreak - указывает, как долго цепь будет оставаться разорванной.
.CircuitBreakerAsync(1, TimeSpan.FromSeconds(30),
// onBreak - является делегатом, позволяет выполнить какое-то действие, когда цепь разорвана.
(exception, timeSpan) =>
{
Trace.WriteLine($"Circuit broken! {timeSpan}");
},
// onReset - является делегатом, позволяет выполнить какое-либо действие, когда канал сброшен
() =>
{
Trace.WriteLine("Circuit Reset!");
});
// Политика подмены исключения на ответ.
// Игнорируем ошибку и выводим статические данные.
var fallbackPolicy = Policy <string>
.Handle <Exception>()
.FallbackAsync(async token =>
{
Trace.WriteLine("Return Fallback");
return(await new ValueTask <string>("Bypassing a request to the repository."));
});
// Объединение политик в одну.
_policyWrap = Policy.WrapAsync(fallbackPolicy, _circuitBreakerPolicy);
}
private void SetupCircuitBreaker()
{
_circuitPolicy = Policy.Handle <Exception>()
.CircuitBreakerAsync(
3,
TimeSpan.FromSeconds(10),
(exception, span) =>
Console.WriteLine(String.Format("Circuit breaking for {0} ms due to {1}",
span.TotalMilliseconds, exception.Message)),
() => _logger.LogInformation($"Close circuit and allow calls again"),
() => _logger.LogInformation($"HalfOpen circuit allow test call again"));
}
public PurchaseCommandHandler(IPurchaseRepostiory purchaseRepository, IEventBus serviceBus, IEmailService emailService,
IHttpClientFactory httpClientFactory)
{
this.purchaseRepository = purchaseRepository;
this.serviceBus = serviceBus;
this.emailService = emailService;
this.httpClientFactory = httpClientFactory;
// Circuit breaker policy defined
circuitAsyncBreakerPolicy = Policy
.Handle <RpcException>().CircuitBreakerAsync(2, TimeSpan.FromSeconds(15));
}
private void CreateCircuitBreakerSimplePolicy()
=> CircuitBreakerPolicy = Policy
.Handle <Exception>()
.CircuitBreakerAsync(
exceptionsAllowedBeforeBreaking: ConfigurationSection.CircuitBreakerConfiguration.ExceptionsAllowedBeforeBreaking,
durationOfBreak: TimeSpan.FromMilliseconds(ConfigurationSection.CircuitBreakerConfiguration.DurationOfBreaking),
onBreak: (exception, timeOfBreak) =>
{
_metricService.CircuitBreakerMetric.IncrementBreakCount();
_metricService.CircuitBreakerMetric.IncrementBreakTime((long)timeOfBreak.TotalMilliseconds);
Console.WriteLine($"\t[{DateTime.Now}] Break for [{timeOfBreak}]");
},
onReset: () => Console.WriteLine($"\tReseted"),
() => Console.WriteLine($"\t[{DateTime.Now}] HalfOpen"));