public VssHttpRetryMessageHandler(
VssHttpRetryOptions options,
HttpMessageHandler innerHandler)
: base(innerHandler)
{
m_retryOptions = options;
}
/// <summary>
/// Heuristic used to determine whether an exception is a transient network
/// failure that should be retried.
/// </summary>
public static bool IsTransientNetworkException(
Exception ex,
VssHttpRetryOptions options,
out HttpStatusCode?httpStatusCode,
out WebExceptionStatus?webExceptionStatus,
out SocketError?socketErrorCode,
out WinHttpErrorCode?winHttpErrorCode,
out CurlErrorCode?curlErrorCode)
{
httpStatusCode = null;
webExceptionStatus = null;
socketErrorCode = null;
winHttpErrorCode = null;
curlErrorCode = null;
while (ex != null)
{
if (IsTransientNetworkExceptionHelper(ex, options, out httpStatusCode, out webExceptionStatus, out socketErrorCode, out winHttpErrorCode, out curlErrorCode))
{
return(true);
}
ex = ex.InnerException;
}
return(false);
}
/// <summary>
/// Heuristic used to determine whether an exception is a transient network
/// failure that should be retried.
/// </summary>
public static bool IsTransientNetworkException(
Exception ex,
VssHttpRetryOptions options)
{
HttpStatusCode? httpStatusCode;
WebExceptionStatus?webExceptionStatus;
SocketError? socketErrorCode;
WinHttpErrorCode? winHttpErrorCode;
CurlErrorCode? curlErrorCode;
return(IsTransientNetworkException(ex, options, out httpStatusCode, out webExceptionStatus, out socketErrorCode, out winHttpErrorCode, out curlErrorCode));
}
protected override async Task <HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
Int32 attempt = 1;
HttpResponseMessage response = null;
HttpRequestException exception = null;
VssTraceActivity traceActivity = VssTraceActivity.Current;
// Allow overriding default retry options per request
VssHttpRetryOptions retryOptions = m_retryOptions;
object retryOptionsObject;
if (request.Options.TryGetValue(HttpRetryOptionsKey, out retryOptionsObject)) // NETSTANDARD compliant, TryGetValue<T> is not
{
// Fallback to default options if object of unexpected type was passed
retryOptions = retryOptionsObject as VssHttpRetryOptions ?? m_retryOptions;
}
TimeSpan minBackoff = retryOptions.MinBackoff;
Int32 maxAttempts = retryOptions.MaxRetries + 1;
IVssHttpRetryInfo retryInfo = null;
object retryInfoObject;
if (request.Options.TryGetValue(HttpRetryInfoKey, out retryInfoObject)) // NETSTANDARD compliant, TryGetValue<T> is not
{
retryInfo = retryInfoObject as IVssHttpRetryInfo;
}
if (IsLowPriority(request))
{
// Increase the backoff and retry count, low priority requests can be retried many times if the server is busy.
minBackoff = TimeSpan.FromSeconds(minBackoff.TotalSeconds * 2);
maxAttempts = maxAttempts * 10;
}
TimeSpan backoff = minBackoff;
while (attempt <= maxAttempts)
{
// Reset the exception so we don't have a lingering variable
exception = null;
Boolean canRetry = false;
SocketError? socketError = null;
HttpStatusCode? statusCode = null;
WebExceptionStatus?webExceptionStatus = null;
WinHttpErrorCode? winHttpErrorCode = null;
CurlErrorCode? curlErrorCode = null;
string afdRefInfo = null;
try
{
if (attempt == 1)
{
retryInfo?.InitialAttempt(request);
}
response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (attempt > 1)
{
TraceHttpRequestSucceededWithRetry(traceActivity, response, attempt);
}
// Verify the response is successful or the status code is one that may be retried.
if (response.IsSuccessStatusCode)
{
break;
}
else
{
statusCode = response.StatusCode;
afdRefInfo = response.Headers.TryGetValues(HttpHeaders.AfdResponseRef, out var headers) ? headers.First() : null;
canRetry = m_retryOptions.IsRetryableResponse(response);
}
}
catch (HttpRequestException ex)
{
exception = ex;
canRetry = VssNetworkHelper.IsTransientNetworkException(exception, m_retryOptions, out statusCode, out webExceptionStatus, out socketError, out winHttpErrorCode, out curlErrorCode);
}
catch (TimeoutException)
{
throw;
}
if (attempt < maxAttempts && canRetry)
{
backoff = BackoffTimerHelper.GetExponentialBackoff(attempt, minBackoff, m_retryOptions.MaxBackoff, m_retryOptions.BackoffCoefficient);
retryInfo?.Retry(backoff);
TraceHttpRequestRetrying(traceActivity, request, attempt, backoff, statusCode, webExceptionStatus, socketError, winHttpErrorCode, curlErrorCode, afdRefInfo);
}
else
{
if (attempt < maxAttempts)
{
if (exception == null)
{
TraceHttpRequestFailed(traceActivity, request, statusCode != null ? statusCode.Value : (HttpStatusCode)0, afdRefInfo);
}
else
{
TraceHttpRequestFailed(traceActivity, request, exception);
}
}
else
{
TraceHttpRequestFailedMaxAttempts(traceActivity, request, attempt, statusCode, webExceptionStatus, socketError, winHttpErrorCode, curlErrorCode, afdRefInfo);
}
break;
}
// Make sure to dispose of this so we don't keep the connection open
if (response != null)
{
response.Dispose();
}
attempt++;
TraceRaw(request, 100011, TraceLevel.Error,
"{{ \"Client\":\"{0}\", \"Endpoint\":\"{1}\", \"Attempt\":{2}, \"MaxAttempts\":{3}, \"Backoff\":{4} }}",
m_clientName,
request.RequestUri.Host,
attempt,
maxAttempts,
backoff.TotalMilliseconds);
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
}
if (exception != null)
{
throw exception;
}
return(response);
}
public VssHttpRetryMessageHandler(VssHttpRetryOptions options)
{
m_retryOptions = options;
}
/// <summary>
/// Helper which checks a particular Exception instance (non-recursive).
/// </summary>
private static bool IsTransientNetworkExceptionHelper(
Exception ex,
VssHttpRetryOptions options,
out HttpStatusCode?httpStatusCode,
out WebExceptionStatus?webExceptionStatus,
out SocketError?socketErrorCode,
out WinHttpErrorCode?winHttpErrorCode,
out CurlErrorCode?curlErrorCode)
{
ArgumentUtility.CheckForNull(ex, "ex");
httpStatusCode = null;
webExceptionStatus = null;
socketErrorCode = null;
winHttpErrorCode = null;
curlErrorCode = null;
if (ex is WebException)
{
WebException webEx = (WebException)ex;
if (webEx.Response != null && webEx.Response is HttpWebResponse)
{
var httpResponse = (HttpWebResponse)webEx.Response;
httpStatusCode = httpResponse.StatusCode;
// If the options include this status code as a retryable error then we report the exception
// as transient to the caller
if (options.RetryableStatusCodes.Contains(httpResponse.StatusCode))
{
return(true);
}
}
webExceptionStatus = webEx.Status;
if (webEx.Status == WebExceptionStatus.ConnectFailure ||
webEx.Status == WebExceptionStatus.ConnectionClosed ||
webEx.Status == WebExceptionStatus.KeepAliveFailure ||
webEx.Status == WebExceptionStatus.NameResolutionFailure ||
webEx.Status == WebExceptionStatus.ReceiveFailure ||
webEx.Status == WebExceptionStatus.SendFailure ||
webEx.Status == WebExceptionStatus.Timeout)
{
return(true);
}
}
else if (ex is SocketException)
{
SocketException sockEx = (SocketException)ex;
socketErrorCode = sockEx.SocketErrorCode;
if (sockEx.SocketErrorCode == SocketError.Interrupted ||
sockEx.SocketErrorCode == SocketError.NetworkDown ||
sockEx.SocketErrorCode == SocketError.NetworkUnreachable ||
sockEx.SocketErrorCode == SocketError.NetworkReset ||
sockEx.SocketErrorCode == SocketError.ConnectionAborted ||
sockEx.SocketErrorCode == SocketError.ConnectionReset ||
sockEx.SocketErrorCode == SocketError.TimedOut ||
sockEx.SocketErrorCode == SocketError.HostDown ||
sockEx.SocketErrorCode == SocketError.HostUnreachable ||
sockEx.SocketErrorCode == SocketError.TryAgain)
{
return(true);
}
}
else if (ex is Win32Exception) // WinHttpException when use WinHttp (dotnet core)
{
Win32Exception winHttpEx = (Win32Exception)ex;
Int32 errorCode = winHttpEx.NativeErrorCode;
if (errorCode > (Int32)WinHttpErrorCode.WINHTTP_ERROR_BASE &&
errorCode <= (Int32)WinHttpErrorCode.WINHTTP_ERROR_LAST)
{
winHttpErrorCode = (WinHttpErrorCode)errorCode;
if (winHttpErrorCode == WinHttpErrorCode.ERROR_WINHTTP_CANNOT_CONNECT ||
winHttpErrorCode == WinHttpErrorCode.ERROR_WINHTTP_CONNECTION_ERROR ||
winHttpErrorCode == WinHttpErrorCode.ERROR_WINHTTP_INTERNAL_ERROR ||
winHttpErrorCode == WinHttpErrorCode.ERROR_WINHTTP_NAME_NOT_RESOLVED ||
winHttpErrorCode == WinHttpErrorCode.ERROR_WINHTTP_TIMEOUT)
{
return(true);
}
}
}
else if (ex is IOException)
{
if (null != ex.InnerException &&
ex.InnerException is Win32Exception)
{
String stackTrace = ex.StackTrace;
if (null != stackTrace &&
stackTrace.IndexOf("System.Net.Security._SslStream.StartWriting(", StringComparison.Ordinal) >= 0)
{
// HACK: There is an underlying HRESULT code for this error which is not set on the exception which
// bubbles from the underlying stack. The top of the stack trace will be in the _SslStream class
// and will have an exception chain of HttpRequestException -> IOException -> Win32Exception.
// Check for SEC_E_CONTEXT_EXPIRED as this occurs at random in the underlying stack. Retrying the
// request should get a new connection and work correctly, so we ignore this particular error.
return(true);
}
}
}
else if (ex.GetType().Name == "CurlException") // CurlException when use libcurl (dotnet core)
{
// Valid curl error code should in range (0, 93]
if (ex.HResult > 0 && ex.HResult < 94)
{
curlErrorCode = (CurlErrorCode)ex.HResult;
if (curlErrorCode == CurlErrorCode.CURLE_COULDNT_RESOLVE_PROXY ||
curlErrorCode == CurlErrorCode.CURLE_COULDNT_RESOLVE_HOST ||
curlErrorCode == CurlErrorCode.CURLE_COULDNT_CONNECT ||
curlErrorCode == CurlErrorCode.CURLE_HTTP2 ||
curlErrorCode == CurlErrorCode.CURLE_PARTIAL_FILE ||
curlErrorCode == CurlErrorCode.CURLE_WRITE_ERROR ||
curlErrorCode == CurlErrorCode.CURLE_UPLOAD_FAILED ||
curlErrorCode == CurlErrorCode.CURLE_READ_ERROR ||
curlErrorCode == CurlErrorCode.CURLE_OPERATION_TIMEDOUT ||
curlErrorCode == CurlErrorCode.CURLE_INTERFACE_FAILED ||
curlErrorCode == CurlErrorCode.CURLE_GOT_NOTHING ||
curlErrorCode == CurlErrorCode.CURLE_SEND_ERROR ||
curlErrorCode == CurlErrorCode.CURLE_RECV_ERROR)
{
return(true);
}
}
}
return(false);
}