private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container)
{
string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\"";
string serviceHealth = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerId, options : healthCheck);
if (string.IsNullOrEmpty(serviceHealth))
{
// Container has no HEALTHCHECK
return;
}
var retryCount = 0;
while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase))
{
TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff(retryCount, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(2));
executionContext.Output($"{container.ContainerNetworkAlias} service is starting, waiting {backoff.Seconds} seconds before checking again.");
await Task.Delay(backoff, executionContext.CancellationToken);
serviceHealth = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerId, options : healthCheck);
retryCount++;
}
if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase))
{
executionContext.Output($"{container.ContainerNetworkAlias} service is healthy.");
}
else
{
throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}.");
}
}
public async Task ConnectAsync(VssConnection jobConnection)
{
_connection = jobConnection;
int totalAttempts = 5;
int attemptCount = totalAttempts;
var configurationStore = HostContext.GetService <IConfigurationStore>();
var runnerSettings = configurationStore.GetSettings();
while (!_connection.HasAuthenticated && attemptCount-- > 0)
{
try
{
await _connection.ConnectAsync();
break;
}
catch (Exception ex) when(attemptCount > 0)
{
Trace.Info($"Catch exception during connect. {attemptCount} attempts left.");
Trace.Error(ex);
if (runnerSettings.IsHostedServer)
{
await CheckNetworkEndpointsAsync(attemptCount);
}
}
int attempt = totalAttempts - attemptCount;
TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff(attempt, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(3.2), TimeSpan.FromMilliseconds(100));
await Task.Delay(backoff);
}
_taskClient = _connection.GetClient <TaskHttpClient>();
_hasConnection = true;
}
protected override async Task <HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
ArgUtil.NotNull(request, nameof(request));
if (PlatformUtil.RunningOnMacOS || PlatformUtil.RunningOnLinux)
{
request.Version = HttpVersion.Version11;
}
// Potential improvements:
// 1. Calculate a max time across attempts, to avoid retries (this class) on top of retries (VssHttpRetryMessageHandler)
// causing more time to pass than expected in degenerative cases.
// 2. Increase the per-attempt timeout on each attempt. Instead of 5 minutes on each attempt, start low and build to 10-20 minutes.
HttpResponseMessage response = null;
// We can safely retry on timeout if the request isn't one that changes state.
Boolean canRetry = (request.Method == HttpMethod.Get || request.Method == HttpMethod.Head || request.Method == HttpMethod.Options);
if (canRetry)
{
Int32 attempt = 1;
TimeoutException exception = null;
Int32 maxAttempts = _retryOptions.MaxRetries + 1;
while (attempt <= maxAttempts)
{
// Reset the exception so we don't have a lingering variable
exception = null;
Stopwatch watch = Stopwatch.StartNew();
try
{
response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
break;
}
catch (TimeoutException ex)
{
exception = ex;
}
TimeSpan backoff;
if (attempt < maxAttempts)
{
backoff = BackoffTimerHelper.GetExponentialBackoff(
attempt,
_retryOptions.MinBackoff,
_retryOptions.MaxBackoff,
_retryOptions.BackoffCoefficient);
}
else
{
break;
}
string message = StringUtil.Loc("RMContainerItemRequestTimedOut", (int)watch.Elapsed.TotalSeconds, backoff.TotalSeconds, request.Method, request.RequestUri);
_logger.Warning(message);
attempt++;
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
}
if (exception != null)
{
throw exception;
}
}
else
{
// No retries. Just pipe the request through to the other handlers.
response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
}
return(response);
}
