/// <summary>
/// Should the caller retry the operation.
/// </summary>
/// <param name="exception">Exception that occurred when the operation was tried</param>
/// <param name="cancellationToken"></param>
/// <returns>True indicates caller should retry, False otherwise</returns>
public async Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
this.retryContext = null;
// Received Connection error (HttpRequestException), initiate the endpoint rediscovery
if (exception is HttpRequestException _)
{
DefaultTrace.TraceWarning("Endpoint not reachable. Refresh cache and retry");
return(await this.ShouldRetryOnEndpointFailureAsync(
isReadRequest : this.isReadRequest,
forceRefresh : false,
retryOnPreferredLocations : true));
}
DocumentClientException clientException = exception as DocumentClientException;
ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync(
clientException?.StatusCode,
clientException?.GetSubStatus());
if (shouldRetryResult != null)
{
return(shouldRetryResult);
}
return(await this.throttlingRetry.ShouldRetryAsync(exception, cancellationToken));
}
async Task Attempt(RetryContext <TContext> retryContext, IPipe <TContext> next)
{
await retryContext.PreRetry().ConfigureAwait(false);
try
{
await next.Send(retryContext.Context).ConfigureAwait(false);
}
catch (Exception exception)
{
RetryContext <TContext> payloadRetryContext;
if (retryContext.Context.TryGetPayload(out payloadRetryContext) && !payloadRetryContext.CanRetry(exception, out payloadRetryContext))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
throw;
}
RetryContext <TContext> nextRetryContext;
if (!retryContext.CanRetry(exception, out nextRetryContext))
{
await nextRetryContext.RetryFaulted(exception).ConfigureAwait(false);
retryContext.Context.GetOrAddPayload(() => nextRetryContext);
throw;
}
if (nextRetryContext.Delay.HasValue)
{
await Task.Delay(nextRetryContext.Delay.Value).ConfigureAwait(false);
}
await Attempt(nextRetryContext, next).ConfigureAwait(false);
}
}
public ConsumeContextRetryContext(RetryContext <CommandContext> retryContext, RetryCommandContext context)
{
_retryContext = retryContext;
_context = context;
_context.RetryAttempt = retryContext.RetryCount;
}
bool RetryContext <TContext> .CanRetry(Exception exception, out RetryContext <TContext> retryContext)
{
retryContext = new IncrementalRetryContext <TContext>(_policy, Context, Exception, RetryCount + 1, _delay + _delayIncrement, _delayIncrement,
CancellationToken);
return(RetryAttempt < _policy.RetryLimit && _policy.IsHandled(exception));
}
private static async Task ReadFileAsync(RetryContext context)
{
try
{
using (FileStream stream = new FileStream("test.txt", FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 256, true))
{
byte[] buffer = new byte[4];
int bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length);
if (bytesRead == buffer.Length)
{
string text = Encoding.ASCII.GetString(buffer);
Log("ReadFileAsync read '{0}'", text);
context.Add("Text", text);
}
else
{
Log("ReadFileAsync read only {0} bytes.", bytesRead);
}
}
}
catch (Exception e)
{
Log("ReadFileAsync error: {0}: {1}", e.GetType().Name, e.Message);
throw;
}
}
static async Task Attempt <T>(RetryContext <ConsumeContext <T> > retryContext, Func <ConsumeContext <T>, Task> retryMethod)
where T : class
{
try
{
await retryMethod(retryContext.Context).ConfigureAwait(false);
}
catch (Exception exception)
{
if (retryContext.Context.TryGetPayload(out RetryContext <ConsumeContext <T> > nextRetryContext))
{
throw;
}
if (!retryContext.CanRetry(exception, out nextRetryContext))
{
retryContext.Context.GetOrAddPayload(() => nextRetryContext);
throw;
}
if (nextRetryContext.Delay.HasValue)
{
await Task.Delay(nextRetryContext.Delay.Value).ConfigureAwait(false);
}
await Attempt(nextRetryContext, retryMethod).ConfigureAwait(false);
}
}
static async Task <TResult> Attempt <T, TResult>(RetryContext <T> retryContext, Func <Task <TResult> > retryMethod, CancellationToken cancellationToken)
where T : class
{
cancellationToken.ThrowIfCancellationRequested();
try
{
return(await retryMethod().ConfigureAwait(false));
}
catch (Exception exception)
{
RetryContext <T> nextRetryContext;
if (!retryContext.CanRetry(exception, out nextRetryContext))
{
throw;
}
if (nextRetryContext.Delay.HasValue)
{
await Task.Delay(nextRetryContext.Delay.Value, cancellationToken).ConfigureAwait(false);
}
return(await Attempt(nextRetryContext, retryMethod, cancellationToken).ConfigureAwait(false));
}
}
/// <summary>
/// Should the caller retry the operation.
/// </summary>
/// <param name="exception">Exception that occurred when the operation was tried</param>
/// <param name="cancellationToken"></param>
/// <returns>True indicates caller should retry, False otherwise</returns>
public async Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
this.retryContext = null;
// Received Connection error (HttpRequestException), initiate the endpoint rediscovery
if (exception is HttpRequestException _)
{
DefaultTrace.TraceWarning("ClientRetryPolicy: Gateway HttpRequestException Endpoint not reachable. Failed Location: {0}; ResourceAddress: {1}",
this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
this.documentServiceRequest?.ResourceAddress ?? string.Empty);
// Mark both read and write requests because it gateway exception.
// This means all requests going to the region will fail.
return(await this.ShouldRetryOnEndpointFailureAsync(
isReadRequest : this.isReadRequest,
markBothReadAndWriteAsUnavailable : true,
forceRefresh : false,
retryOnPreferredLocations : true));
}
if (exception is DocumentClientException clientException)
{
ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync(
clientException?.StatusCode,
clientException?.GetSubStatus());
if (shouldRetryResult != null)
{
return(shouldRetryResult);
}
}
return(await this.throttlingRetry.ShouldRetryAsync(exception, cancellationToken));
}
public void Can_add_and_get_results_from_context()
{
RetryContext context = new RetryContext();
context.Add("a", 1);
context.Add("b", "Two");
object c1 = new object();
context.Add("c", c1);
int a2 = context.Get <int>("a");
string b2 = context.Get <string>("b");
object c2 = context.Get <object>("c");
Assert.Equal(1, a2);
Assert.Equal("Two", b2);
Assert.Same(c2, c1);
object x = context.Get <object>("x");
int y = context.Get <int>("y");
string z = context.Get <string>("z");
Assert.Null(x);
Assert.Equal(0, y);
Assert.Null(z);
}
async Task IRetryObserver.RetryFault <T>(RetryContext <T> context)
{
if (context.Context.TryGetPayload(out Span span))
{
Console.WriteLine("RetryFault:", span.Id);
}
}
public void InvokeBasicFunctionWithTriggerMetadataAndTraceContextAndRetryContextWorks()
{
string path = Path.Join(s_funcDirectory, "testBasicFunctionWithTriggerMetadataAndRetryContext.ps1");
var(functionInfo, testManager) = PrepareFunction(path, string.Empty);
Hashtable triggerMetadata = new Hashtable(StringComparer.OrdinalIgnoreCase)
{
{ TestInputBindingName, TestStringData }
};
RetryContext retryContext = new RetryContext(TestRetryCount, TestMaxRetryCount, TestException);
try
{
FunctionMetadata.RegisterFunctionMetadata(testManager.InstanceId, functionInfo.OutputBindings);
Hashtable result = InvokeFunction(testManager, functionInfo, triggerMetadata, retryContext);
// The outputBinding hashtable for the runspace should be cleared after 'InvokeFunction'
Hashtable outputBindings = FunctionMetadata.GetOutputBindingHashtable(testManager.InstanceId);
Assert.Empty(outputBindings);
// A PowerShell function should be created fro the Az function.
string expectedResult = $"{TestStringData},{functionInfo.DeployedPSFuncName}:{TestRetryCount},{TestMaxRetryCount},{TestMessage}";
Assert.Equal(expectedResult, result[TestOutputBindingName]);
}
finally
{
FunctionMetadata.UnregisterFunctionMetadata(testManager.InstanceId);
}
}
public TimeSpan?NextRetryDelay(RetryContext retryContext)
{
var retryDelay = retryContext.ElapsedTime < TimeSpan.FromSeconds(60) ? _random.Next(2, 5) : _random.Next(5, 20);
Console.WriteLine($"Delaying next reconnect by {retryDelay} seconds");
return(TimeSpan.FromSeconds(retryDelay));
}
public TimeSpan?NextRetryDelay(RetryContext retryContext)
{
var nextRetryDelay = TimeSpan.FromSeconds(5) + (TimeSpan.FromSeconds(5) * _random.NextDouble());
LogDebug($"Reconnecting in {nextRetryDelay:g}");
return(nextRetryDelay);
}
public async Task<RetryContext> ExecuteAsync()
{
RetryContext context = new RetryContext();
bool shouldRetry = false;
TimeSpan startTime = this.Timer.Elapsed;
do
{
try
{
if (shouldRetry)
{
await this.BeforeRetry(context);
}
context.Exception = null;
context.ElapsedTime = this.Timer.Elapsed - startTime;
await this.func(context);
}
catch (Exception e)
{
context.Exception = new AggregateException(e);
}
context.ElapsedTime = this.Timer.Elapsed - startTime;
context.Succeeded = this.Succeeded(context);
shouldRetry = this.ShouldRetry(context);
++context.Iteration;
}
while (!context.Succeeded && shouldRetry);
return context;
}
public TimeSpan?NextRetryDelay(RetryContext retryContext)
{
SnTrace.TaskManagement.Write($"Connection retry attempt failed. Elapsed time: {retryContext.ElapsedTime}");
// retry infinitely
return(TimeSpan.FromSeconds(10));
}
async Task IRetryObserver.PreRetry <T>(RetryContext <T> context)
{
var span = new Span(context.RetryAttempt.ToString());
context.Context.AddOrUpdatePayload(() => span, _ => span);
Console.WriteLine("PreRetry: ", span);
}
public void Execute_invokes_before_retry_func_after_iteration()
{
RetryLoop loop = new RetryLoop(r => Task.FromResult(false));
loop.Succeeded = r => false;
loop.ShouldRetry = r => r.Iteration < 1;
int count = 0;
int iteration = 0;
loop.BeforeRetry = delegate(RetryContext r)
{
++count;
iteration = r.Iteration;
return(Task.FromResult(false));
};
Task <RetryContext> task = loop.ExecuteAsync();
Assert.Equal(TaskStatus.RanToCompletion, task.Status);
RetryContext context = task.Result;
Assert.Equal(2, context.Iteration);
Assert.False(context.Succeeded);
Assert.Equal(1, count);
Assert.Equal(1, iteration);
}
public Task PostFault <T>(RetryContext <T> context)
where T : class, PipeContext
{
_completionSource.TrySetResult(context);
return(TaskUtil.Completed);
}
private async Task <ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
bool isReadRequest,
bool forceRefresh,
bool retryOnPreferredLocations)
{
if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount)
{
DefaultTrace.TraceInformation("ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}", this.failoverRetryCount);
return(ShouldRetryResult.NoRetry());
}
this.failoverRetryCount++;
if (this.locationEndpoint != null)
{
if (isReadRequest)
{
this.globalEndpointManager.MarkEndpointUnavailableForRead(this.locationEndpoint);
}
else
{
this.globalEndpointManager.MarkEndpointUnavailableForWrite(this.locationEndpoint);
}
}
TimeSpan retryDelay = TimeSpan.Zero;
if (!isReadRequest)
{
DefaultTrace.TraceInformation("Failover happening. retryCount {0}", this.failoverRetryCount);
if (this.failoverRetryCount > 1)
{
//if retried both endpoints, follow regular retry interval.
retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS);
}
}
else
{
retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS);
}
await this.globalEndpointManager.RefreshLocationAsync(null, forceRefresh);
int retryLocationIndex = this.failoverRetryCount; // Used to generate a round-robin effect
if (retryOnPreferredLocations)
{
retryLocationIndex = 0; // When the endpoint is marked as unavailable, it is moved to the bottom of the preferrence list
}
this.retryContext = new RetryContext
{
RetryLocationIndex = retryLocationIndex,
RetryRequestOnPreferredLocations = retryOnPreferredLocations,
};
return(ShouldRetryResult.RetryAfter(retryDelay));
}
async Task IRetryObserver.PostFault <T>(RetryContext <T> context)
{
if (context.Context.TryGetPayload(out Span span))
{
this.counter.Increment();
Console.WriteLine("PostFault:", span.Id);
}
}
private static Hashtable InvokeFunction(
PowerShellManager powerShellManager,
AzFunctionInfo functionInfo,
Hashtable triggerMetadata = null,
RetryContext retryContext = null)
{
return(powerShellManager.InvokeFunction(functionInfo, triggerMetadata, null, retryContext, s_testInputData, new FunctionInvocationPerformanceStopwatch()));
}
Task IRetryObserver.PostFault <T>(RetryContext <T> context)
{
Interlocked.Increment(ref _postFaultCount);
_postFault.SetCompleted();
return(TaskUtil.Completed);
}
public TimeSpan?NextRetryDelay(RetryContext ctx)
{
if (ctx.PreviousRetryCount < 10)
{
return(span);
}
return(null);
}
Task IRetryObserver.RetryFault <T>(RetryContext <T> context)
{
Interlocked.Increment(ref _retryFaultCount);
_retryFault.TrySetResult(context);
return(TaskUtil.Completed);
}
public bool CanRetry(Exception exception, out RetryContext <CommandContext> retryContext)
{
var canRetry = _policyContext.CanRetry(exception, out RetryContext <CommandContext> policyRetryContext);
retryContext = new ConsumeContextRetryContext(policyRetryContext, canRetry ? _context.CreateNext() : _context);
return(canRetry);
}
async Task Run()
{
var stoppingContext = new TransportStoppingContext(Stopping);
RetryPolicyContext <TransportStoppingContext> policyContext = _retryPolicy.CreatePolicyContext(stoppingContext);
try
{
while (!IsStopping)
{
RetryContext <TransportStoppingContext> retryContext = null;
try
{
if (retryContext != null)
{
LogContext.Warning?.Log(retryContext.Exception, "Retrying {Delay}: {Message}", retryContext.Delay,
retryContext.Exception.Message);
if (retryContext.Delay.HasValue)
{
await Task.Delay(retryContext.Delay.Value, Stopping).ConfigureAwait(false);
}
}
if (!IsStopping)
{
await RunTransport().ConfigureAwait(false);
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exception)
{
if (retryContext != null)
{
retryContext = retryContext.CanRetry(exception, out RetryContext <TransportStoppingContext> nextRetryContext)
? nextRetryContext
: null;
}
if (retryContext == null && !policyContext.CanRetry(exception, out retryContext))
{
break;
}
}
}
}
catch (Exception exception)
{
LogContext.Debug?.Log(exception, "ReceiveTransport Run Exception: {InputAddress}", _context.InputAddress);
}
finally
{
policyContext.Dispose();
}
}
public Hashtable InvokeFunction(
AzFunctionInfo functionInfo,
Hashtable triggerMetadata,
TraceContext traceContext,
RetryContext retryContext,
IList <ParameterBinding> inputData,
FunctionInvocationPerformanceStopwatch stopwatch)
{
var outputBindings = FunctionMetadata.GetOutputBindingHashtable(_pwsh.Runspace.InstanceId);
var durableController = new DurableController(functionInfo.DurableFunctionInfo, _pwsh);
try
{
durableController.BeforeFunctionInvocation(inputData);
AddEntryPointInvocationCommand(functionInfo);
stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.FunctionCodeReady);
SetInputBindingParameterValues(functionInfo, inputData, durableController, triggerMetadata, traceContext, retryContext);
stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.InputBindingValuesReady);
if (!durableController.ShouldSuppressPipelineTraces())
{
_pwsh.AddCommand("Microsoft.Azure.Functions.PowerShellWorker\\Trace-PipelineObject");
}
stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.InvokingFunctionCode);
Logger.Log(isUserOnlyLog: false, LogLevel.Trace, CreateInvocationPerformanceReportMessage(functionInfo.FuncName, stopwatch));
try
{
return(durableController.TryInvokeOrchestrationFunction(out var result)
? result
: InvokeNonOrchestrationFunction(durableController, outputBindings));
}
catch (RuntimeException e)
{
ErrorAnalysisLogger.Log(Logger, e.ErrorRecord, isException: true);
Logger.Log(isUserOnlyLog: true, LogLevel.Error, GetFunctionExceptionMessage(e));
throw;
}
catch (OrchestrationFailureException e)
{
if (e.InnerException is IContainsErrorRecord inner)
{
Logger.Log(isUserOnlyLog: true, LogLevel.Error, GetFunctionExceptionMessage(inner));
}
throw;
}
}
finally
{
durableController.AfterFunctionInvocation();
outputBindings.Clear();
ResetRunspace();
}
}
public Task RetryFault <T>(RetryContext <T> context)
where T : class, PipeContext
{
var payload = context.Context.GetOrAddPayload(() => new RetryPayload());
payload.RetryFaultCount++;
return(TaskUtil.Completed);
}
private ResponseObject RetryMethodContextDisableBackoffOnceImplementation(RetryContext context)
{
if (context.Iteration == 0)
{
context.DisableBackoff();
}
return(new ResponseObject(ResponseCode.Error));
}
public bool CanRetry(Exception exception, out RetryContext <ConsumeContext> retryContext)
{
RetryContext <ConsumeContext> policyRetryContext;
var canRetry = _retryContext.CanRetry(exception, out policyRetryContext);
retryContext = new ConsumeContextRetryContext(policyRetryContext, _context);
return(canRetry);
}
public TimeSpan?NextRetryDelay(RetryContext retryContext)
{
return(retryContext.PreviousRetryCount switch
{
0 => TimeSpan.FromSeconds(5),
1 => TimeSpan.FromSeconds(10),
2 => TimeSpan.FromSeconds(20),
<= 10 => TimeSpan.FromSeconds(30),
_ => TimeSpan.FromSeconds(60),
});
private ShouldRetryResult ShouldRetryOnSessionNotAvailable()
{
this.sessionTokenRetryCount++;
if (!this.enableEndpointDiscovery)
{
// if endpoint discovery is disabled, the request cannot be retried anywhere else
return(ShouldRetryResult.NoRetry());
}
else
{
if (this.canUseMultipleWriteLocations)
{
ReadOnlyCollection <Uri> endpoints = this.isReadRequest ? this.globalEndpointManager.ReadEndpoints : this.globalEndpointManager.WriteEndpoints;
if (this.sessionTokenRetryCount > endpoints.Count)
{
// When use multiple write locations is true and the request has been tried
// on all locations, then don't retry the request
return(ShouldRetryResult.NoRetry());
}
else
{
this.retryContext = new RetryContext()
{
RetryCount = this.sessionTokenRetryCount - 1,
RetryRequestOnPreferredLocations = this.sessionTokenRetryCount > 1,
ClearSessionTokenOnSessionNotAvailable = (this.sessionTokenRetryCount == endpoints.Count) // clear on last attempt
};
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
}
else
{
if (this.sessionTokenRetryCount > 1)
{
// When cannot use multiple write locations, then don't retry the request if
// we have already tried this request on the write location
return(ShouldRetryResult.NoRetry());
}
else
{
this.retryContext = new RetryContext
{
RetryCount = this.sessionTokenRetryCount - 1,
RetryRequestOnPreferredLocations = false,
ClearSessionTokenOnSessionNotAvailable = true,
};
return(ShouldRetryResult.RetryAfter(TimeSpan.Zero));
}
}
}
}
private static Task BackoffAsync(RetryContext context)
{
TimeSpan delay = TimeSpan.FromMilliseconds(10 * context.Iteration);
Log("Backing off for {0:0.000} seconds.", delay.TotalSeconds);
return Task.Delay(delay);
}
public void Can_add_and_get_results_from_context()
{
RetryContext context = new RetryContext();
context.Add("a", 1);
context.Add("b", "Two");
object c1 = new object();
context.Add("c", c1);
int a2 = context.Get<int>("a");
string b2 = context.Get<string>("b");
object c2 = context.Get<object>("c");
Assert.Equal(1, a2);
Assert.Equal("Two", b2);
Assert.Same(c2, c1);
object x = context.Get<object>("x");
int y = context.Get<int>("y");
string z = context.Get<string>("z");
Assert.Null(x);
Assert.Equal(0, y);
Assert.Null(z);
}
