/// <summary>
/// Given a service request, retries the request up to 10 times using an exponential
/// backoff strategy. Also takes an enumeration of status codes for which requests should be
/// retried (that is, HTTP errors that are expected to occur transiently).
/// </summary>
internal TResponse ExecuteWithRetries <TResponse>(
IClientServiceRequest <TResponse> request, IEnumerable <HttpStatusCode> statusCodes)
{
var backOff = new Google.Apis.Util.ExponentialBackOff();
int numRetries = 0;
while (true)
{
try
{
return(request.Execute());
}
catch (Google.GoogleApiException e)
{
if (!statusCodes.Contains(e.HttpStatusCode))
{
throw e;
}
numRetries++;
var nextSpan = backOff.GetNextBackOff(numRetries);
if (nextSpan == TimeSpan.MinValue)
{
throw e;
}
Console.WriteLine("Attempt {0} failed, retrying in {1}.", numRetries, nextSpan);
System.Threading.Thread.Sleep(nextSpan);
}
}
}
private static TResponse SafeExecuteSync <TResponse>(IClientServiceRequest <TResponse> request, int attemptOrderNumber)
{
// Attempt to execute provided request.
try { return(request.Execute()); }
// On request operation canceled
catch (OperationCanceledException ex)
{
// Increase attempt order number by one.
attemptOrderNumber++;
// Validate whether request reattempt limit hasn't been exceeded
if (RequestsExecutor._cancellationErrorAttemptsLimit < attemptOrderNumber)
{
// ... and throw appropriate exception if has. attempt
throw new RequestExecutionUnexpectedException(request, ex);
}
// Await appropriate cooldown period.
Thread.Sleep(RequestsExecutor._cancellationErrorCooldown);
// Reattempt.
return(RequestsExecutor.SafeExecuteSync(request, attemptOrderNumber));
}
// On google internal error encounter, retry
catch (Google.GoogleApiException ex) when(ex.Message.Contains("Internal error encountered. [500]"))
{
// Increase attempt order number by one.
attemptOrderNumber++;
// Validate whether request reattempt limit hasn't been exceeded
if (RequestsExecutor._internalErrorAttemptsLimit < attemptOrderNumber)
{
// ... and throw appropriate exception if has. attempt
throw new RequestExecutionUnexpectedException(request, ex);
}
// Await appropriate cooldown period.
Thread.Sleep(RequestsExecutor._internalErrorCooldown);
// Reattempt.
return(RequestsExecutor.SafeExecuteSync <TResponse>(request, attemptOrderNumber));
}
// On used all available requests quota
catch (Google.GoogleApiException ex) when(ex.Message.Contains("Quota exceeded for quota group"))
{
// Set session request limiter state to indicate no available google requests quota at this time.
SessionRequestsLimiter.Instance.ClearAvailableQuota();
// Reattempt.
return(RequestsExecutor.SafeExecuteSync <TResponse>(request, attemptOrderNumber));
}
// On any other unexpected database error
catch (Google.GoogleApiException ex) { throw new RequestExecutionUnexpectedException(request, ex); }
}