/// <summary> /// Should the caller retry the operation. /// </summary> /// <param name="exception">Exception that occurred when the operation was tried</param> /// <param name="cancellationToken"></param> /// <returns>True indicates caller should retry, False otherwise</returns> public async Task <ShouldRetryResult> ShouldRetryAsync( Exception exception, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); this.retryContext = null; // Received Connection error (HttpRequestException), initiate the endpoint rediscovery if (exception is HttpRequestException _) { DefaultTrace.TraceWarning("Endpoint not reachable. Refresh cache and retry"); return(await this.ShouldRetryOnEndpointFailureAsync( isReadRequest : this.isReadRequest, forceRefresh : false, retryOnPreferredLocations : true)); } DocumentClientException clientException = exception as DocumentClientException; ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync( clientException?.StatusCode, clientException?.GetSubStatus()); if (shouldRetryResult != null) { return(shouldRetryResult); } return(await this.throttlingRetry.ShouldRetryAsync(exception, cancellationToken)); }
async Task Attempt(RetryContext <TContext> retryContext, IPipe <TContext> next) { await retryContext.PreRetry().ConfigureAwait(false); try { await next.Send(retryContext.Context).ConfigureAwait(false); } catch (Exception exception) { RetryContext <TContext> payloadRetryContext; if (retryContext.Context.TryGetPayload(out payloadRetryContext) && !payloadRetryContext.CanRetry(exception, out payloadRetryContext)) { await retryContext.RetryFaulted(exception).ConfigureAwait(false); throw; } RetryContext <TContext> nextRetryContext; if (!retryContext.CanRetry(exception, out nextRetryContext)) { await nextRetryContext.RetryFaulted(exception).ConfigureAwait(false); retryContext.Context.GetOrAddPayload(() => nextRetryContext); throw; } if (nextRetryContext.Delay.HasValue) { await Task.Delay(nextRetryContext.Delay.Value).ConfigureAwait(false); } await Attempt(nextRetryContext, next).ConfigureAwait(false); } }
public ConsumeContextRetryContext(RetryContext <CommandContext> retryContext, RetryCommandContext context) { _retryContext = retryContext; _context = context; _context.RetryAttempt = retryContext.RetryCount; }
bool RetryContext <TContext> .CanRetry(Exception exception, out RetryContext <TContext> retryContext) { retryContext = new IncrementalRetryContext <TContext>(_policy, Context, Exception, RetryCount + 1, _delay + _delayIncrement, _delayIncrement, CancellationToken); return(RetryAttempt < _policy.RetryLimit && _policy.IsHandled(exception)); }
private static async Task ReadFileAsync(RetryContext context) { try { using (FileStream stream = new FileStream("test.txt", FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 256, true)) { byte[] buffer = new byte[4]; int bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length); if (bytesRead == buffer.Length) { string text = Encoding.ASCII.GetString(buffer); Log("ReadFileAsync read '{0}'", text); context.Add("Text", text); } else { Log("ReadFileAsync read only {0} bytes.", bytesRead); } } } catch (Exception e) { Log("ReadFileAsync error: {0}: {1}", e.GetType().Name, e.Message); throw; } }
static async Task Attempt <T>(RetryContext <ConsumeContext <T> > retryContext, Func <ConsumeContext <T>, Task> retryMethod) where T : class { try { await retryMethod(retryContext.Context).ConfigureAwait(false); } catch (Exception exception) { if (retryContext.Context.TryGetPayload(out RetryContext <ConsumeContext <T> > nextRetryContext)) { throw; } if (!retryContext.CanRetry(exception, out nextRetryContext)) { retryContext.Context.GetOrAddPayload(() => nextRetryContext); throw; } if (nextRetryContext.Delay.HasValue) { await Task.Delay(nextRetryContext.Delay.Value).ConfigureAwait(false); } await Attempt(nextRetryContext, retryMethod).ConfigureAwait(false); } }
static async Task <TResult> Attempt <T, TResult>(RetryContext <T> retryContext, Func <Task <TResult> > retryMethod, CancellationToken cancellationToken) where T : class { cancellationToken.ThrowIfCancellationRequested(); try { return(await retryMethod().ConfigureAwait(false)); } catch (Exception exception) { RetryContext <T> nextRetryContext; if (!retryContext.CanRetry(exception, out nextRetryContext)) { throw; } if (nextRetryContext.Delay.HasValue) { await Task.Delay(nextRetryContext.Delay.Value, cancellationToken).ConfigureAwait(false); } return(await Attempt(nextRetryContext, retryMethod, cancellationToken).ConfigureAwait(false)); } }
/// <summary> /// Should the caller retry the operation. /// </summary> /// <param name="exception">Exception that occurred when the operation was tried</param> /// <param name="cancellationToken"></param> /// <returns>True indicates caller should retry, False otherwise</returns> public async Task <ShouldRetryResult> ShouldRetryAsync( Exception exception, CancellationToken cancellationToken) { this.retryContext = null; // Received Connection error (HttpRequestException), initiate the endpoint rediscovery if (exception is HttpRequestException _) { DefaultTrace.TraceWarning("ClientRetryPolicy: Gateway HttpRequestException Endpoint not reachable. Failed Location: {0}; ResourceAddress: {1}", this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty, this.documentServiceRequest?.ResourceAddress ?? string.Empty); // Mark both read and write requests because it gateway exception. // This means all requests going to the region will fail. return(await this.ShouldRetryOnEndpointFailureAsync( isReadRequest : this.isReadRequest, markBothReadAndWriteAsUnavailable : true, forceRefresh : false, retryOnPreferredLocations : true)); } if (exception is DocumentClientException clientException) { ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync( clientException?.StatusCode, clientException?.GetSubStatus()); if (shouldRetryResult != null) { return(shouldRetryResult); } } return(await this.throttlingRetry.ShouldRetryAsync(exception, cancellationToken)); }
public void Can_add_and_get_results_from_context() { RetryContext context = new RetryContext(); context.Add("a", 1); context.Add("b", "Two"); object c1 = new object(); context.Add("c", c1); int a2 = context.Get <int>("a"); string b2 = context.Get <string>("b"); object c2 = context.Get <object>("c"); Assert.Equal(1, a2); Assert.Equal("Two", b2); Assert.Same(c2, c1); object x = context.Get <object>("x"); int y = context.Get <int>("y"); string z = context.Get <string>("z"); Assert.Null(x); Assert.Equal(0, y); Assert.Null(z); }
async Task IRetryObserver.RetryFault <T>(RetryContext <T> context) { if (context.Context.TryGetPayload(out Span span)) { Console.WriteLine("RetryFault:", span.Id); } }
public void InvokeBasicFunctionWithTriggerMetadataAndTraceContextAndRetryContextWorks() { string path = Path.Join(s_funcDirectory, "testBasicFunctionWithTriggerMetadataAndRetryContext.ps1"); var(functionInfo, testManager) = PrepareFunction(path, string.Empty); Hashtable triggerMetadata = new Hashtable(StringComparer.OrdinalIgnoreCase) { { TestInputBindingName, TestStringData } }; RetryContext retryContext = new RetryContext(TestRetryCount, TestMaxRetryCount, TestException); try { FunctionMetadata.RegisterFunctionMetadata(testManager.InstanceId, functionInfo.OutputBindings); Hashtable result = InvokeFunction(testManager, functionInfo, triggerMetadata, retryContext); // The outputBinding hashtable for the runspace should be cleared after 'InvokeFunction' Hashtable outputBindings = FunctionMetadata.GetOutputBindingHashtable(testManager.InstanceId); Assert.Empty(outputBindings); // A PowerShell function should be created fro the Az function. string expectedResult = $"{TestStringData},{functionInfo.DeployedPSFuncName}:{TestRetryCount},{TestMaxRetryCount},{TestMessage}"; Assert.Equal(expectedResult, result[TestOutputBindingName]); } finally { FunctionMetadata.UnregisterFunctionMetadata(testManager.InstanceId); } }
public TimeSpan?NextRetryDelay(RetryContext retryContext) { var retryDelay = retryContext.ElapsedTime < TimeSpan.FromSeconds(60) ? _random.Next(2, 5) : _random.Next(5, 20); Console.WriteLine($"Delaying next reconnect by {retryDelay} seconds"); return(TimeSpan.FromSeconds(retryDelay)); }
public TimeSpan?NextRetryDelay(RetryContext retryContext) { var nextRetryDelay = TimeSpan.FromSeconds(5) + (TimeSpan.FromSeconds(5) * _random.NextDouble()); LogDebug($"Reconnecting in {nextRetryDelay:g}"); return(nextRetryDelay); }
public async Task<RetryContext> ExecuteAsync() { RetryContext context = new RetryContext(); bool shouldRetry = false; TimeSpan startTime = this.Timer.Elapsed; do { try { if (shouldRetry) { await this.BeforeRetry(context); } context.Exception = null; context.ElapsedTime = this.Timer.Elapsed - startTime; await this.func(context); } catch (Exception e) { context.Exception = new AggregateException(e); } context.ElapsedTime = this.Timer.Elapsed - startTime; context.Succeeded = this.Succeeded(context); shouldRetry = this.ShouldRetry(context); ++context.Iteration; } while (!context.Succeeded && shouldRetry); return context; }
public TimeSpan?NextRetryDelay(RetryContext retryContext) { SnTrace.TaskManagement.Write($"Connection retry attempt failed. Elapsed time: {retryContext.ElapsedTime}"); // retry infinitely return(TimeSpan.FromSeconds(10)); }
async Task IRetryObserver.PreRetry <T>(RetryContext <T> context) { var span = new Span(context.RetryAttempt.ToString()); context.Context.AddOrUpdatePayload(() => span, _ => span); Console.WriteLine("PreRetry: ", span); }
public void Execute_invokes_before_retry_func_after_iteration() { RetryLoop loop = new RetryLoop(r => Task.FromResult(false)); loop.Succeeded = r => false; loop.ShouldRetry = r => r.Iteration < 1; int count = 0; int iteration = 0; loop.BeforeRetry = delegate(RetryContext r) { ++count; iteration = r.Iteration; return(Task.FromResult(false)); }; Task <RetryContext> task = loop.ExecuteAsync(); Assert.Equal(TaskStatus.RanToCompletion, task.Status); RetryContext context = task.Result; Assert.Equal(2, context.Iteration); Assert.False(context.Succeeded); Assert.Equal(1, count); Assert.Equal(1, iteration); }
public Task PostFault <T>(RetryContext <T> context) where T : class, PipeContext { _completionSource.TrySetResult(context); return(TaskUtil.Completed); }
private async Task <ShouldRetryResult> ShouldRetryOnEndpointFailureAsync( bool isReadRequest, bool forceRefresh, bool retryOnPreferredLocations) { if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount) { DefaultTrace.TraceInformation("ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}", this.failoverRetryCount); return(ShouldRetryResult.NoRetry()); } this.failoverRetryCount++; if (this.locationEndpoint != null) { if (isReadRequest) { this.globalEndpointManager.MarkEndpointUnavailableForRead(this.locationEndpoint); } else { this.globalEndpointManager.MarkEndpointUnavailableForWrite(this.locationEndpoint); } } TimeSpan retryDelay = TimeSpan.Zero; if (!isReadRequest) { DefaultTrace.TraceInformation("Failover happening. retryCount {0}", this.failoverRetryCount); if (this.failoverRetryCount > 1) { //if retried both endpoints, follow regular retry interval. retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS); } } else { retryDelay = TimeSpan.FromMilliseconds(ClientRetryPolicy.RetryIntervalInMS); } await this.globalEndpointManager.RefreshLocationAsync(null, forceRefresh); int retryLocationIndex = this.failoverRetryCount; // Used to generate a round-robin effect if (retryOnPreferredLocations) { retryLocationIndex = 0; // When the endpoint is marked as unavailable, it is moved to the bottom of the preferrence list } this.retryContext = new RetryContext { RetryLocationIndex = retryLocationIndex, RetryRequestOnPreferredLocations = retryOnPreferredLocations, }; return(ShouldRetryResult.RetryAfter(retryDelay)); }
async Task IRetryObserver.PostFault <T>(RetryContext <T> context) { if (context.Context.TryGetPayload(out Span span)) { this.counter.Increment(); Console.WriteLine("PostFault:", span.Id); } }
private static Hashtable InvokeFunction( PowerShellManager powerShellManager, AzFunctionInfo functionInfo, Hashtable triggerMetadata = null, RetryContext retryContext = null) { return(powerShellManager.InvokeFunction(functionInfo, triggerMetadata, null, retryContext, s_testInputData, new FunctionInvocationPerformanceStopwatch())); }
Task IRetryObserver.PostFault <T>(RetryContext <T> context) { Interlocked.Increment(ref _postFaultCount); _postFault.SetCompleted(); return(TaskUtil.Completed); }
public TimeSpan?NextRetryDelay(RetryContext ctx) { if (ctx.PreviousRetryCount < 10) { return(span); } return(null); }
Task IRetryObserver.RetryFault <T>(RetryContext <T> context) { Interlocked.Increment(ref _retryFaultCount); _retryFault.TrySetResult(context); return(TaskUtil.Completed); }
public bool CanRetry(Exception exception, out RetryContext <CommandContext> retryContext) { var canRetry = _policyContext.CanRetry(exception, out RetryContext <CommandContext> policyRetryContext); retryContext = new ConsumeContextRetryContext(policyRetryContext, canRetry ? _context.CreateNext() : _context); return(canRetry); }
async Task Run() { var stoppingContext = new TransportStoppingContext(Stopping); RetryPolicyContext <TransportStoppingContext> policyContext = _retryPolicy.CreatePolicyContext(stoppingContext); try { while (!IsStopping) { RetryContext <TransportStoppingContext> retryContext = null; try { if (retryContext != null) { LogContext.Warning?.Log(retryContext.Exception, "Retrying {Delay}: {Message}", retryContext.Delay, retryContext.Exception.Message); if (retryContext.Delay.HasValue) { await Task.Delay(retryContext.Delay.Value, Stopping).ConfigureAwait(false); } } if (!IsStopping) { await RunTransport().ConfigureAwait(false); } } catch (OperationCanceledException) { throw; } catch (Exception exception) { if (retryContext != null) { retryContext = retryContext.CanRetry(exception, out RetryContext <TransportStoppingContext> nextRetryContext) ? nextRetryContext : null; } if (retryContext == null && !policyContext.CanRetry(exception, out retryContext)) { break; } } } } catch (Exception exception) { LogContext.Debug?.Log(exception, "ReceiveTransport Run Exception: {InputAddress}", _context.InputAddress); } finally { policyContext.Dispose(); } }
public Hashtable InvokeFunction( AzFunctionInfo functionInfo, Hashtable triggerMetadata, TraceContext traceContext, RetryContext retryContext, IList <ParameterBinding> inputData, FunctionInvocationPerformanceStopwatch stopwatch) { var outputBindings = FunctionMetadata.GetOutputBindingHashtable(_pwsh.Runspace.InstanceId); var durableController = new DurableController(functionInfo.DurableFunctionInfo, _pwsh); try { durableController.BeforeFunctionInvocation(inputData); AddEntryPointInvocationCommand(functionInfo); stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.FunctionCodeReady); SetInputBindingParameterValues(functionInfo, inputData, durableController, triggerMetadata, traceContext, retryContext); stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.InputBindingValuesReady); if (!durableController.ShouldSuppressPipelineTraces()) { _pwsh.AddCommand("Microsoft.Azure.Functions.PowerShellWorker\\Trace-PipelineObject"); } stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.InvokingFunctionCode); Logger.Log(isUserOnlyLog: false, LogLevel.Trace, CreateInvocationPerformanceReportMessage(functionInfo.FuncName, stopwatch)); try { return(durableController.TryInvokeOrchestrationFunction(out var result) ? result : InvokeNonOrchestrationFunction(durableController, outputBindings)); } catch (RuntimeException e) { ErrorAnalysisLogger.Log(Logger, e.ErrorRecord, isException: true); Logger.Log(isUserOnlyLog: true, LogLevel.Error, GetFunctionExceptionMessage(e)); throw; } catch (OrchestrationFailureException e) { if (e.InnerException is IContainsErrorRecord inner) { Logger.Log(isUserOnlyLog: true, LogLevel.Error, GetFunctionExceptionMessage(inner)); } throw; } } finally { durableController.AfterFunctionInvocation(); outputBindings.Clear(); ResetRunspace(); } }
public Task RetryFault <T>(RetryContext <T> context) where T : class, PipeContext { var payload = context.Context.GetOrAddPayload(() => new RetryPayload()); payload.RetryFaultCount++; return(TaskUtil.Completed); }
private ResponseObject RetryMethodContextDisableBackoffOnceImplementation(RetryContext context) { if (context.Iteration == 0) { context.DisableBackoff(); } return(new ResponseObject(ResponseCode.Error)); }
public bool CanRetry(Exception exception, out RetryContext <ConsumeContext> retryContext) { RetryContext <ConsumeContext> policyRetryContext; var canRetry = _retryContext.CanRetry(exception, out policyRetryContext); retryContext = new ConsumeContextRetryContext(policyRetryContext, _context); return(canRetry); }
public TimeSpan?NextRetryDelay(RetryContext retryContext) { return(retryContext.PreviousRetryCount switch { 0 => TimeSpan.FromSeconds(5), 1 => TimeSpan.FromSeconds(10), 2 => TimeSpan.FromSeconds(20), <= 10 => TimeSpan.FromSeconds(30), _ => TimeSpan.FromSeconds(60), });
private ShouldRetryResult ShouldRetryOnSessionNotAvailable() { this.sessionTokenRetryCount++; if (!this.enableEndpointDiscovery) { // if endpoint discovery is disabled, the request cannot be retried anywhere else return(ShouldRetryResult.NoRetry()); } else { if (this.canUseMultipleWriteLocations) { ReadOnlyCollection <Uri> endpoints = this.isReadRequest ? this.globalEndpointManager.ReadEndpoints : this.globalEndpointManager.WriteEndpoints; if (this.sessionTokenRetryCount > endpoints.Count) { // When use multiple write locations is true and the request has been tried // on all locations, then don't retry the request return(ShouldRetryResult.NoRetry()); } else { this.retryContext = new RetryContext() { RetryCount = this.sessionTokenRetryCount - 1, RetryRequestOnPreferredLocations = this.sessionTokenRetryCount > 1, ClearSessionTokenOnSessionNotAvailable = (this.sessionTokenRetryCount == endpoints.Count) // clear on last attempt }; return(ShouldRetryResult.RetryAfter(TimeSpan.Zero)); } } else { if (this.sessionTokenRetryCount > 1) { // When cannot use multiple write locations, then don't retry the request if // we have already tried this request on the write location return(ShouldRetryResult.NoRetry()); } else { this.retryContext = new RetryContext { RetryCount = this.sessionTokenRetryCount - 1, RetryRequestOnPreferredLocations = false, ClearSessionTokenOnSessionNotAvailable = true, }; return(ShouldRetryResult.RetryAfter(TimeSpan.Zero)); } } } }
private static Task BackoffAsync(RetryContext context) { TimeSpan delay = TimeSpan.FromMilliseconds(10 * context.Iteration); Log("Backing off for {0:0.000} seconds.", delay.TotalSeconds); return Task.Delay(delay); }
public void Can_add_and_get_results_from_context() { RetryContext context = new RetryContext(); context.Add("a", 1); context.Add("b", "Two"); object c1 = new object(); context.Add("c", c1); int a2 = context.Get<int>("a"); string b2 = context.Get<string>("b"); object c2 = context.Get<object>("c"); Assert.Equal(1, a2); Assert.Equal("Two", b2); Assert.Same(c2, c1); object x = context.Get<object>("x"); int y = context.Get<int>("y"); string z = context.Get<string>("z"); Assert.Null(x); Assert.Equal(0, y); Assert.Null(z); }