private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container) { string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\""; string serviceHealth = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerId, options : healthCheck); if (string.IsNullOrEmpty(serviceHealth)) { // Container has no HEALTHCHECK return; } var retryCount = 0; while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase)) { TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff(retryCount, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(2)); executionContext.Output($"{container.ContainerNetworkAlias} service is starting, waiting {backoff.Seconds} seconds before checking again."); await Task.Delay(backoff, executionContext.CancellationToken); serviceHealth = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerId, options : healthCheck); retryCount++; } if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase)) { executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); } else { throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}."); } }
public async Task ConnectAsync(VssConnection jobConnection) { _connection = jobConnection; int totalAttempts = 5; int attemptCount = totalAttempts; var configurationStore = HostContext.GetService <IConfigurationStore>(); var runnerSettings = configurationStore.GetSettings(); while (!_connection.HasAuthenticated && attemptCount-- > 0) { try { await _connection.ConnectAsync(); break; } catch (Exception ex) when(attemptCount > 0) { Trace.Info($"Catch exception during connect. {attemptCount} attempts left."); Trace.Error(ex); if (runnerSettings.IsHostedServer) { await CheckNetworkEndpointsAsync(attemptCount); } } int attempt = totalAttempts - attemptCount; TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff(attempt, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(3.2), TimeSpan.FromMilliseconds(100)); await Task.Delay(backoff); } _taskClient = _connection.GetClient <TaskHttpClient>(); _hasConnection = true; }
protected override async Task <HttpResponseMessage> SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { ArgUtil.NotNull(request, nameof(request)); if (PlatformUtil.RunningOnMacOS || PlatformUtil.RunningOnLinux) { request.Version = HttpVersion.Version11; } // Potential improvements: // 1. Calculate a max time across attempts, to avoid retries (this class) on top of retries (VssHttpRetryMessageHandler) // causing more time to pass than expected in degenerative cases. // 2. Increase the per-attempt timeout on each attempt. Instead of 5 minutes on each attempt, start low and build to 10-20 minutes. HttpResponseMessage response = null; // We can safely retry on timeout if the request isn't one that changes state. Boolean canRetry = (request.Method == HttpMethod.Get || request.Method == HttpMethod.Head || request.Method == HttpMethod.Options); if (canRetry) { Int32 attempt = 1; TimeoutException exception = null; Int32 maxAttempts = _retryOptions.MaxRetries + 1; while (attempt <= maxAttempts) { // Reset the exception so we don't have a lingering variable exception = null; Stopwatch watch = Stopwatch.StartNew(); try { response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); break; } catch (TimeoutException ex) { exception = ex; } TimeSpan backoff; if (attempt < maxAttempts) { backoff = BackoffTimerHelper.GetExponentialBackoff( attempt, _retryOptions.MinBackoff, _retryOptions.MaxBackoff, _retryOptions.BackoffCoefficient); } else { break; } string message = StringUtil.Loc("RMContainerItemRequestTimedOut", (int)watch.Elapsed.TotalSeconds, backoff.TotalSeconds, request.Method, request.RequestUri); _logger.Warning(message); attempt++; await Task.Delay(backoff, cancellationToken).ConfigureAwait(false); } if (exception != null) { throw exception; } } else { // No retries. Just pipe the request through to the other handlers. response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); } return(response); }