public static async Task Retry(this IRetryPolicy retryPolicy, Func <Task> retryMethod, CancellationToken cancellationToken = default)
{
var inlinePipeContext = new InlinePipeContext(cancellationToken);
RetryPolicyContext <InlinePipeContext> policyContext = retryPolicy.CreatePolicyContext(inlinePipeContext);
try
{
await retryMethod().ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exception)
{
if (!policyContext.CanRetry(exception, out RetryContext <InlinePipeContext> retryContext))
{
throw;
}
try
{
await Attempt(inlinePipeContext, retryContext, retryMethod).ConfigureAwait(false);
return;
}
catch (OperationCanceledException ex) when(ex.CancellationToken == cancellationToken)
{
}
throw;
}
finally
{
policyContext.Dispose();
}
}
Task IRetryObserver.PostCreate <T>(RetryPolicyContext <T> context)
{
return(TaskUtil.Completed);
}
public async Task Send(ConsumeContext <T> context, IPipe <ConsumeContext <T> > next)
{
RetryPolicyContext <ConsumeContext <T> > policyContext = _retryPolicy.CreatePolicyContext(context);
await _observers.PostCreate(policyContext).ConfigureAwait(false);
try
{
await next.Send(policyContext.Context).ConfigureAwait(false);
}
catch (Exception exception)
{
if (context.CancellationToken.IsCancellationRequested)
{
if (exception is OperationCanceledException canceledException && canceledException.CancellationToken == context.CancellationToken)
{
throw;
}
context.CancellationToken.ThrowIfCancellationRequested();
}
if (policyContext.Context.TryGetPayload(out RetryContext <ConsumeContext <T> > payloadRetryContext))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(payloadRetryContext).ConfigureAwait(false);
context.GetOrAddPayload(() => payloadRetryContext);
throw;
}
if (policyContext.Context.TryGetPayload(out RetryContext genericRetryContext))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(genericRetryContext).ConfigureAwait(false);
context.GetOrAddPayload(() => genericRetryContext);
throw;
}
if (!policyContext.CanRetry(exception, out RetryContext <ConsumeContext <T> > retryContext))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(retryContext).ConfigureAwait(false);
if (_retryPolicy.IsHandled(exception))
{
context.GetOrAddPayload(() => retryContext);
}
throw;
}
int previousDeliveryCount = context.Headers.Get(MessageHeaders.RedeliveryCount, default(int?)) ?? 0;
for (int retryIndex = 0; retryIndex < previousDeliveryCount; retryIndex++)
{
if (!retryContext.CanRetry(exception, out retryContext))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(retryContext).ConfigureAwait(false);
if (_retryPolicy.IsHandled(exception))
{
context.GetOrAddPayload(() => retryContext);
}
throw;
}
}
await _observers.PostFault(retryContext).ConfigureAwait(false);
try
{
if (!context.TryGetPayload(out MessageRedeliveryContext redeliveryContext))
{
throw new ContextException("The message redelivery context was not available to delay the message", exception);
}
var delay = retryContext.Delay ?? TimeSpan.Zero;
await redeliveryContext.ScheduleRedelivery(delay).ConfigureAwait(false);
await context.NotifyConsumed(context, context.ReceiveContext.ElapsedTime, TypeMetadataCache <RedeliveryRetryFilter <T> > .ShortName).ConfigureAwait(false);
}
catch (Exception redeliveryException)
{
throw new ContextException("The message delivery could not be rescheduled", new AggregateException(redeliveryException, exception));
}
}
}
public CommandContextRetryPolicyContext(RetryPolicyContext <CommandContext> policyContext, RetryCommandContext context)
{
_policyContext = policyContext;
_context = context;
}
public Task PostCreate <T>(RetryPolicyContext <T> context) where T : class, PipeContext => Task.CompletedTask;
public virtual HttpResponse DoAction <T>(AcsRequest <T> request, bool autoRetry, int maxRetryNumber,
string regionId,
AlibabaCloudCredentials credentials, Signer signer, FormatType?format, List <Endpoint> endpoints)
where T : AcsResponse
{
var httpStatusCode = "";
var retryAttemptTimes = 0;
ClientException exception;
RetryPolicyContext retryPolicyContext;
do
{
try
{
var watch = Stopwatch.StartNew();
FormatType?requestFormatType = request.AcceptFormat;
format = requestFormatType;
var domain = request.ProductDomain ??
Endpoint.FindProductDomain(regionId, request.Product, endpoints);
if (null == domain)
{
throw new ClientException("SDK.InvalidRegionId", "Can not find endpoint to access.");
}
request.Headers["User-Agent"] =
UserAgent.Resolve(request.GetSysUserAgentConfig(), userAgentConfig);
var httpRequest = request.SignRequest(signer, credentials, format, domain);
ResolveTimeout(httpRequest);
SetHttpsInsecure(IgnoreCertificate);
ResolveProxy(httpRequest, request);
var response = GetResponse(httpRequest);
httpStatusCode = response.Status.ToString();
PrintHttpDebugMsg(request, response);
watch.Stop();
CommonLog.ExecuteTime = watch.ElapsedMilliseconds;
return(response);
}
catch (ClientException ex)
{
retryPolicyContext = new RetryPolicyContext(ex, httpStatusCode, retryAttemptTimes, request.Product,
request.Version,
request.ActionName, RetryCondition.BlankStatus);
CommonLog.LogException(ex, ex.ErrorCode, ex.ErrorMessage);
exception = ex;
}
Thread.Sleep(retryPolicy.GetDelayTimeBeforeNextRetry(retryPolicyContext));
} while ((retryPolicy.ShouldRetry(retryPolicyContext) & RetryCondition.NoRetry) != RetryCondition.NoRetry);
if (exception != null)
{
CommonLog.LogException(exception, exception.ErrorCode, exception.ErrorMessage);
throw new ClientException(exception.ErrorCode, exception.ErrorMessage);
}
return(null);
}
public async Task Send(ConsumeContext <T> context, IPipe <ConsumeContext <T> > next)
{
RetryPolicyContext <ConsumeContext <T> > policyContext = _retryPolicy.CreatePolicyContext(context);
await _observers.PostCreate(policyContext).ConfigureAwait(false);
try
{
await next.Send(policyContext.Context).ConfigureAwait(false);
}
catch (Exception exception)
{
RetryContext <ConsumeContext <T> > payloadRetryContext;
if (context.TryGetPayload(out payloadRetryContext))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(payloadRetryContext).ConfigureAwait(false);
throw;
}
RetryContext genericRetryContext;
if (context.TryGetPayload(out genericRetryContext))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(genericRetryContext).ConfigureAwait(false);
throw;
}
RetryContext <ConsumeContext <T> > retryContext;
if (!policyContext.CanRetry(exception, out retryContext))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(retryContext).ConfigureAwait(false);
context.GetOrAddPayload(() => retryContext);
throw;
}
int previousDeliveryCount = context.Headers.Get(MessageHeaders.RedeliveryCount, default(int?)) ?? 0;
for (int retryIndex = 0; retryIndex < previousDeliveryCount; retryIndex++)
{
if (!retryContext.CanRetry(exception, out retryContext))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(retryContext).ConfigureAwait(false);
context.GetOrAddPayload(() => retryContext);
throw;
}
}
await _observers.PostFault(retryContext).ConfigureAwait(false);
try
{
MessageRedeliveryContext redeliveryContext;
if (!context.TryGetPayload(out redeliveryContext))
{
throw new ContextException("The message redelivery context was not available to delay the message", exception);
}
var delay = retryContext.Delay ?? TimeSpan.Zero;
await redeliveryContext.ScheduleRedelivery(delay).ConfigureAwait(false);
}
catch (Exception redeliveryException)
{
throw new ContextException("The message delivery could not be rescheduled", new AggregateException(redeliveryException, exception));
}
}
}
public RetryCondition ShouldRetry(RetryPolicyContext retryPolicyContext)
{
return(RetryCondition.NoRetry);
}
public RetryCondition ShouldRetry(RetryPolicyContext retryPolicyContext)
{
return(defaultRetryHttpStatusCodeList.Contains(retryPolicyContext.HttpStatusCode)
? RetryCondition.ShouldRetry
: RetryCondition.NoRetry);
}
public static async Task Retry(this IHostConfiguration hostConfiguration, Func <Task> factory, ISupervisor supervisor,
CancellationToken cancellationToken)
{
var description = hostConfiguration.HostAddress;
using var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, supervisor.Stopping);
var stoppingContext = new SupervisorStoppingContext(tokenSource.Token);
RetryPolicyContext <SupervisorStoppingContext> policyContext = hostConfiguration.ReceiveTransportRetryPolicy.CreatePolicyContext(stoppingContext);
try
{
RetryContext <SupervisorStoppingContext> retryContext = null;
while (!tokenSource.Token.IsCancellationRequested)
{
try
{
if (retryContext?.Delay != null)
{
await Task.Delay(retryContext.Delay.Value, tokenSource.Token).ConfigureAwait(false);
}
if (tokenSource.Token.IsCancellationRequested)
{
throw new ConnectionException($"The connection is stopping and cannot be used: {description}", retryContext?.Exception);
}
await factory().ConfigureAwait(false);
return;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exception)
{
if (retryContext != null)
{
retryContext = retryContext.CanRetry(exception, out RetryContext <SupervisorStoppingContext> nextRetryContext)
? nextRetryContext
: null;
}
if (retryContext == null && !policyContext.CanRetry(exception, out retryContext))
{
throw;
}
}
if (tokenSource.Token.IsCancellationRequested)
{
break;
}
try
{
await Task.Delay(TimeSpan.FromSeconds(1), tokenSource.Token).ConfigureAwait(false);
}
catch
{
// just a little breather before reconnecting the receive transport
}
}
throw new ConnectionException($"The connection is stopping and cannot be used: {description}", retryContext?.Exception);
}
finally
{
policyContext.Dispose();
}
}
public int GetDelayTimeBeforeNextRetry(RetryPolicyContext retryPolicyContext)
{
return(0);
}
public virtual async Task <HttpResponse> DoActionAsync <T>(AcsRequest <T> request, bool autoRetry, int maxRetryNumber,
string regionId,
AlibabaCloudCredentials credentials, Signer signer, FormatType?format, List <Endpoint> endpoints,
CancellationToken cancellationToken)
where T : AcsResponse
{
var httpStatusCode = "";
var retryAttemptTimes = 0;
ClientException exception;
RetryPolicyContext retryPolicyContext;
do
{
try
{
var watch = Stopwatch.StartNew();
FormatType?requestFormatType = request.AcceptFormat;
var domain = request.ProductDomain ??
Endpoint.FindProductDomain(regionId, request.Product, endpoints);
if (null == domain)
{
throw new ClientException("SDK.InvalidRegionId", "Can not find endpoint to access.");
}
var userAgent = UserAgent.Resolve(request.GetSysUserAgentConfig(), userAgentConfig);
DictionaryUtil.Add(request.Headers, "User-Agent", userAgent);
DictionaryUtil.Add(request.Headers, "x-acs-version", request.Version);
if (!string.IsNullOrWhiteSpace(request.ActionName))
{
DictionaryUtil.Add(request.Headers, "x-acs-action", request.ActionName);
}
var httpRequest = request.SignRequest(signer, credentials, requestFormatType, domain);
ResolveTimeout(httpRequest, request.Product, request.Version, request.ActionName);
SetHttpsInsecure(IgnoreCertificate);
ResolveProxy(httpRequest, request);
var response = await GetResponseAsync(httpRequest, cancellationToken).ConfigureAwait(false);
httpStatusCode = response.Status.ToString();
PrintHttpDebugMsg(request, response);
watch.Stop();
CommonLog.ExecuteTime = watch.ElapsedMilliseconds;
return(response);
}
catch (ClientException ex)
{
retryPolicyContext = new RetryPolicyContext(ex, httpStatusCode, retryAttemptTimes, request.Product,
request.Version,
request.ActionName, RetryCondition.BlankStatus);
CommonLog.LogException(ex, ex.ErrorCode, ex.ErrorMessage);
exception = ex;
}
await Task.Delay(retryPolicy.GetDelayTimeBeforeNextRetry(retryPolicyContext), cancellationToken).ConfigureAwait(false);
} while ((retryPolicy.ShouldRetry(retryPolicyContext) & RetryCondition.NoRetry) != RetryCondition.NoRetry);
if (exception != null)
{
CommonLog.LogException(exception, exception.ErrorCode, exception.ErrorMessage);
throw new ClientException(exception.ErrorCode, exception.ErrorMessage);
}
return(null);
}
async Task RunJob(PipeContext context, IJobConsumer <TJob> jobConsumer)
{
var jobContext = context.GetPayload <JobContext <TJob> >();
RetryPolicyContext <JobContext <TJob> > policyContext = _retryPolicy.CreatePolicyContext(jobContext);
try
{
await jobContext.NotifyStarted().ConfigureAwait(false);
await jobConsumer.Run(jobContext).ConfigureAwait(false);
await jobContext.NotifyCompleted().ConfigureAwait(false);
}
catch (OperationCanceledException exception)
{
if (jobContext.CancellationToken == exception.CancellationToken)
{
await jobContext.NotifyCanceled("Operation canceled").ConfigureAwait(false);
}
}
catch (Exception exception)
{
if (!policyContext.CanRetry(exception, out RetryContext <JobContext <TJob> > retryContext))
{
if (_retryPolicy.IsHandled(exception))
{
context.GetOrAddPayload(() => retryContext);
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
}
await jobContext.NotifyFaulted(exception).ConfigureAwait(false);
return;
}
var currentRetryAttempt = jobContext.RetryAttempt;
for (var retryIndex = 0; retryIndex < currentRetryAttempt; retryIndex++)
{
if (!retryContext.CanRetry(exception, out retryContext))
{
if (_retryPolicy.IsHandled(exception))
{
context.GetOrAddPayload(() => retryContext);
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
}
await jobContext.NotifyFaulted(exception).ConfigureAwait(false);
return;
}
}
var delay = retryContext.Delay ?? TimeSpan.Zero;
await jobContext.NotifyFaulted(exception, delay).ConfigureAwait(false);
}
finally
{
policyContext.Dispose();
}
}
public ConsumeContextRetryPolicyContext(RetryPolicyContext <ConsumeContext> policyContext, RetryConsumeContext context, CancellationToken cancellationToken)
{
_policyContext = policyContext;
_context = context;
_cancellationToken = cancellationToken;
}
public override HttpResponse DoAction <T>(AcsRequest <T> request, bool autoRetry, int maxRetryNumber,
string regionId,
AlibabaCloudCredentials credentials, Signer signer, FormatType?format, List <Endpoint> endpoints)
{
var httpStatusCode = "";
var retryAttemptTimes = 0;
ClientException exception;
RetryPolicyContext retryPolicyContext;
do
{
try
{
var watch = Stopwatch.StartNew();
FormatType?requestFormatType = request.AcceptFormat;
format = requestFormatType;
ProductDomain domain;
if (!string.IsNullOrEmpty(this.endpoint))
{
domain = new ProductDomain("", this.endpoint);
}
else
{
domain = request.ProductDomain ??
Aliyun.Acs.Core.Regions.Endpoint.FindProductDomain(regionId, request.Product, endpoints);
}
if (null == domain)
{
throw new ClientException("SDK.InvalidRegionId", "Can not find endpoint to access.");
}
var userAgent = UserAgent.Resolve(request.GetSysUserAgentConfig(), userAgentConfig);
DictionaryUtil.Add(request.Headers, "User-Agent", userAgent);
DictionaryUtil.Add(request.Headers, "x-acs-version", request.Version);
if (!string.IsNullOrWhiteSpace(request.ActionName))
{
DictionaryUtil.Add(request.Headers, "x-acs-action", request.ActionName);
}
var httpRequest = SignRequest(request, signer, credentials, format, domain);
Helper.RunInstanceMethod(typeof(DefaultAcsClient), "ResolveTimeout", this, new object[] { httpRequest, request.Product, request.Version, request.ActionName });
SetHttpsInsecure(IgnoreCertificate);
ResolveProxy(httpRequest, request);
var response = GetResponse(httpRequest);
httpStatusCode = response.Status.ToString();
Helper.RunInstanceMethod(typeof(DefaultAcsClient), "PrintHttpDebugMsg", this, new object[] { request, response });
watch.Stop();
return(response);
}
catch (ClientException ex)
{
retryPolicyContext = new RetryPolicyContext(ex, httpStatusCode, retryAttemptTimes, request.Product,
request.Version,
request.ActionName, RetryCondition.BlankStatus);
exception = ex;
}
Thread.Sleep(retryPolicy.GetDelayTimeBeforeNextRetry(retryPolicyContext));
} while ((retryPolicy.ShouldRetry(retryPolicyContext) & RetryCondition.NoRetry) != RetryCondition.NoRetry);
if (exception != null)
{
throw new ClientException(exception.ErrorCode, exception.ErrorMessage);
}
return(null);
}
public Task PostCreate <T>(RetryPolicyContext <T> context)
where T : class, PipeContext
{
return(TaskUtil.Completed);
}
public Task PostCreate <T>(RetryPolicyContext <T> context)
where T : class, PipeContext
{
return(ForEachAsync(x => x.PostCreate(context)));
}
public async Task Send(TContext context, IPipe <TContext> next)
{
using (RetryPolicyContext <TContext> policyContext = _retryPolicy.CreatePolicyContext(context))
{
if (_observers.Count > 0)
{
await _observers.PostCreate(policyContext).ConfigureAwait(false);
}
try
{
await next.Send(policyContext.Context).ConfigureAwait(false);
}
catch (OperationCanceledException exception)
when(exception.CancellationToken == context.CancellationToken || exception.CancellationToken == policyContext.Context.CancellationToken)
{
throw;
}
catch (Exception exception)
{
if (policyContext.Context.CancellationToken.IsCancellationRequested)
{
policyContext.Context.CancellationToken.ThrowIfCancellationRequested();
}
if (!policyContext.CanRetry(exception, out RetryContext <TContext> retryContext))
{
if (_retryPolicy.IsHandled(exception))
{
context.GetOrAddPayload(() => retryContext);
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
if (_observers.Count > 0)
{
await _observers.RetryFault(retryContext).ConfigureAwait(false);
}
}
throw;
}
var previousDeliveryCount = context.GetRedeliveryCount();
for (var retryIndex = 0; retryIndex < previousDeliveryCount; retryIndex++)
{
if (!retryContext.CanRetry(exception, out retryContext))
{
if (_retryPolicy.IsHandled(exception))
{
context.GetOrAddPayload(() => retryContext);
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
if (_observers.Count > 0)
{
await _observers.RetryFault(retryContext).ConfigureAwait(false);
}
}
throw;
}
}
if (_observers.Count > 0)
{
await _observers.PostFault(retryContext).ConfigureAwait(false);
}
try
{
var redeliveryContext = context.GetPayload <MessageRedeliveryContext>();
var delay = retryContext.Delay ?? TimeSpan.Zero;
await redeliveryContext.ScheduleRedelivery(delay).ConfigureAwait(false);
await context.NotifyConsumed(context, context.ReceiveContext.ElapsedTime,
TypeMetadataCache <RedeliveryRetryFilter <TContext, TMessage> > .ShortName).ConfigureAwait(false);
}
catch (Exception redeliveryException)
{
throw new TransportException(context.ReceiveContext.InputAddress, "The message delivery could not be rescheduled",
new AggregateException(redeliveryException, exception));
}
}
}
}
async Task IFilter <TContext> .Send(TContext context, IPipe <TContext> next)
{
using (RetryPolicyContext <TContext> policyContext = _retryPolicy.CreatePolicyContext(context))
{
await _observers.PostCreate(policyContext).ConfigureAwait(false);
try
{
await next.Send(policyContext.Context).ConfigureAwait(false);
}
catch (OperationCanceledException exception)
when(exception.CancellationToken == policyContext.Context.CancellationToken || exception.CancellationToken == context.CancellationToken)
{
throw;
}
catch (Exception exception)
{
if (policyContext.Context.CancellationToken.IsCancellationRequested)
{
policyContext.Context.CancellationToken.ThrowIfCancellationRequested();
}
if (policyContext.Context.TryGetPayload(out RetryContext <TContext> payloadRetryContext))
{
if (_retryPolicy.IsHandled(exception))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(payloadRetryContext).ConfigureAwait(false);
}
context.GetOrAddPayload(() => payloadRetryContext);
throw;
}
if (policyContext.Context.TryGetPayload(out RetryContext genericRetryContext))
{
if (_retryPolicy.IsHandled(exception))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(genericRetryContext).ConfigureAwait(false);
}
context.GetOrAddPayload(() => genericRetryContext);
throw;
}
if (!policyContext.CanRetry(exception, out RetryContext <TContext> retryContext))
{
if (_retryPolicy.IsHandled(exception))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(retryContext).ConfigureAwait(false);
context.GetOrAddPayload(() => retryContext);
}
throw;
}
await _observers.PostFault(retryContext).ConfigureAwait(false);
await Attempt(context, retryContext, next).ConfigureAwait(false);
}
}
}
async Task Run()
{
var stoppingContext = new TransportStoppingContext(Stopping);
RetryPolicyContext <TransportStoppingContext> policyContext = _retryPolicy.CreatePolicyContext(stoppingContext);
try
{
RetryContext <TransportStoppingContext> retryContext = null;
while (!IsStopping)
{
try
{
if (retryContext?.Delay != null)
{
await Task.Delay(retryContext.Delay.Value, Stopping).ConfigureAwait(false);
}
if (!IsStopping)
{
await RunTransport().ConfigureAwait(false);
}
}
catch (OperationCanceledException exception)
{
if (retryContext == null)
{
await NotifyFaulted(exception).ConfigureAwait(false);
}
throw;
}
catch (Exception exception)
{
if (retryContext != null)
{
retryContext = retryContext.CanRetry(exception, out RetryContext <TransportStoppingContext> nextRetryContext)
? nextRetryContext
: null;
}
if (retryContext == null && !policyContext.CanRetry(exception, out retryContext))
{
break;
}
}
if (IsStopping)
{
break;
}
try
{
await Task.Delay(TimeSpan.FromSeconds(1), Stopping).ConfigureAwait(false);
}
catch
{
// just a little breather before reconnecting the receive transport
}
}
}
catch (OperationCanceledException exception)
{
if (exception.CancellationToken != Stopping)
{
LogContext.Debug?.Log(exception, "ReceiveTransport Operation Cancelled: {InputAddress}", _context.InputAddress);
}
}
catch (Exception exception)
{
LogContext.Debug?.Log(exception, "ReceiveTransport Run Exception: {InputAddress}", _context.InputAddress);
}
finally
{
policyContext.Dispose();
}
}