private RetryParams ComputeAdalRetry(Exception ex)
{
if (ex is AdalServiceException)
{
AdalServiceException adalServiceException = (AdalServiceException)ex;
// When the Service Token Server (STS) is too busy because of “too many requests”,
// it returns an HTTP error 429 with a hint about when you can try again (Retry-After response field) as a delay in seconds
if (adalServiceException.ErrorCode == AdalError.ServiceUnavailable || adalServiceException.StatusCode == 429)
{
RetryConditionHeaderValue retryAfter = adalServiceException.Headers.RetryAfter;
// Depending on the service, the recommended retry time may be in retryAfter.Delta or retryAfter.Date. Check both.
if (retryAfter != null && retryAfter.Delta.HasValue)
{
return(new RetryParams(retryAfter.Delta.Value));
}
else if (retryAfter != null && retryAfter.Date.HasValue)
{
return(new RetryParams(retryAfter.Date.Value.Offset));
}
// We got a 429 but didn't get a specific back-off time. Use the default
return(RetryParams.DefaultBackOff(0));
}
}
return(RetryParams.DefaultBackOff(0));
}
public static async Task <TResult> Run <TResult>(Func <Task <TResult> > task, Func <Exception, int, RetryParams> retryExceptionHandler)
{
RetryParams retry = RetryParams.StopRetrying;
List <Exception> exceptions = new List <Exception>();
int currentRetryCount = 0;
do
{
try
{
return(await task().ConfigureAwait(false));
}
catch (Exception ex)
{
exceptions.Add(ex);
retry = retryExceptionHandler(ex, currentRetryCount);
}
if (retry.ShouldRetry)
{
currentRetryCount++;
await Task.Delay(retry.RetryAfter.WithRandomNoise()).ConfigureAwait(false);
}
} while (retry.ShouldRetry);
throw new AggregateException("Failed to acquire token for client credentials.", exceptions);
}
private RetryParams HandleAdalException(Exception ex, int currentRetryCount)
{
if (IsAdalServiceUnavailable(ex))
{
return(ComputeAdalRetry(ex));
}
else if (ex is ThrottleException)
{
// This is an exception that we threw, with knowledge that
// one of our threads is trying to acquire a token from the server
// Use the retry parameters recommended in the exception
ThrottleException throttlException = (ThrottleException)ex;
return(throttlException.RetryParams ?? RetryParams.DefaultBackOff(currentRetryCount));
}
else
{
// We end up here is the exception is not an ADAL exception. An example, is under high traffic
// where we could have a timeout waiting to acquire a token, waiting on the semaphore.
// If we hit a timeout, we want to retry a reasonable number of times.
return(RetryParams.DefaultBackOff(currentRetryCount));
}
}
private async Task <AuthenticationResult> AcquireTokenAsync()
{
bool acquired = false;
try
{
// The ADAL client team recommends limiting concurrency of calls. When the Token is in cache there is never
// contention on this semaphore, but when tokens expire there is some. However, after measuring performance
// with and without the semaphore (and different configs for the semaphore), not limiting concurrency actually
// results in higher response times overall. Without the use of this semaphore calls to AcquireTokenAsync can take up
// to 5 seconds under high concurrency scenarios.
acquired = await authContextSemaphore.WaitAsync(SemaphoreTimeout).ConfigureAwait(false);
// If we are allowed to enter the semaphore, acquire the token.
if (acquired)
{
// Acquire token async using ADAL.NET
// https://github.com/AzureAD/azure-activedirectory-library-for-dotnet
// Given that this is a ClientCredential scenario, it will use the cache without the
// need to call AcquireTokenSilentAsync (which is only for user credentials).
var res = await authContext.AcquireTokenAsync(JwtConfig.OAuthResourceUri, clientCredential).ConfigureAwait(false);
// This means we acquired a valid token successfully. We can make our retry policy null.
// Note that the retry policy is set under the semaphore so no additional synchronization is needed.
if (currentRetryPolicy != null)
{
currentRetryPolicy = null;
}
return(res);
}
else
{
// If the token is taken, it means that one thread is trying to acquire a token from the server.
// If we already received information about how much to throttle, it will be in the currentRetryPolicy.
// Use that to inform our next delay before trying.
throw new ThrottleException()
{
RetryParams = currentRetryPolicy
};
}
}
catch (Exception ex)
{
// If we are getting throttled, we set the retry policy according to the RetryAfter headers
// that we receive from the auth server.
// Note that the retry policy is set under the semaphore so no additional synchronization is needed.
if (IsAdalServiceUnavailable(ex))
{
currentRetryPolicy = ComputeAdalRetry(ex);
}
throw ex;
}
finally
{
// Always release the semaphore if we acquired it.
if (acquired)
{
authContextSemaphore.Release();
}
}
}
