private RetryParams ComputeAdalRetry(Exception ex)
{
if (ex is AdalServiceException)
{
AdalServiceException adalServiceException = (AdalServiceException)ex;
// When the Service Token Server (STS) is too busy because of “too many requests”,
// it returns an HTTP error 429 with a hint about when you can try again (Retry-After response field) as a delay in seconds
if (adalServiceException.ErrorCode == msalTemporarilyUnavailable || adalServiceException.StatusCode == 429)
{
RetryConditionHeaderValue retryAfter = adalServiceException.Headers.RetryAfter;
// Depending on the service, the recommended retry time may be in retryAfter.Delta or retryAfter.Date. Check both.
if (retryAfter != null && retryAfter.Delta.HasValue)
{
return(new RetryParams(retryAfter.Delta.Value));
}
else if (retryAfter != null && retryAfter.Date.HasValue)
{
return(new RetryParams(retryAfter.Date.Value.Offset));
}
// We got a 429 but didn't get a specific back-off time. Use the default
return(RetryParams.DefaultBackOff(0));
}
}
return(RetryParams.DefaultBackOff(0));
}
private static RetryParams HandleAdalException(Exception ex, int currentRetryCount)
{
if (IsAdalServiceUnavailable(ex))
{
return(ComputeAdalRetry(ex));
}
else if (ex is ThrottleException)
{
// This is an exception that we threw, with knowledge that
// one of our threads is trying to acquire a token from the server
// Use the retry parameters recommended in the exception
ThrottleException throttlException = (ThrottleException)ex;
return(throttlException.RetryParams ?? RetryParams.DefaultBackOff(currentRetryCount));
}
else if (IsAdalServiceInvalidRequest(ex))
{
return(RetryParams.StopRetrying);
}
else
{
// We end up here is the exception is not an ADAL exception. An example, is under high traffic
// where we could have a timeout waiting to acquire a token, waiting on the semaphore.
// If we hit a timeout, we want to retry a reasonable number of times.
return(RetryParams.DefaultBackOff(currentRetryCount));
}
}
public static async Task <TResult> Run <TResult>(Func <Task <TResult> > task, Func <Exception, int, RetryParams> retryExceptionHandler)
{
RetryParams retry = RetryParams.StopRetrying;
List <Exception> exceptions = new List <Exception>();
int currentRetryCount = 0;
do
{
try
{
return(await task().ConfigureAwait(false));
}
catch (Exception ex)
{
exceptions.Add(ex);
retry = retryExceptionHandler(ex, currentRetryCount);
}
if (retry.ShouldRetry)
{
currentRetryCount++;
await Task.Delay(retry.RetryAfter.WithJitter()).ConfigureAwait(false);
}
}while (retry.ShouldRetry);
throw new AggregateException("Failed to acquire token for client credentials.", exceptions);
}
private RetryParams HandleTokenProviderException(Exception e, int retryCount)
{
_logger.LogError(e, "Exception when trying to acquire token using MSI!");
return(e is AzureServiceTokenProviderException // BadRequest
? RetryParams.StopRetrying
: RetryParams.DefaultBackOff(retryCount));
}
private RetryParams HandleMsalException(Exception ex, int ct)
{
_logger?.LogError(ex, "Exception acquiring token through MSAL.");
if (ex is MsalServiceException msalException)
{
_logger?.LogWarning(msalException, $"MSAL service error code: {msalException.ErrorCode}.");
// Service error with status code "temporarily_unavailable" is retryable.
// Spec and reference: https://docs.microsoft.com/en-us/azure/active-directory/develop/reference-aadsts-error-codes.
if (msalException.ErrorCode == "temporarily_unavailable")
{
return(RetryParams.DefaultBackOff(ct));
}
}
return(RetryParams.StopRetrying);
}
private async Task <AuthenticationResult> AcquireTokenAsync(bool forceRefresh = false)
{
bool acquired = false;
if (forceRefresh)
{
authContext.TokenCache.Clear();
}
try
{
// The ADAL client team recommends limiting concurrency of calls. When the Token is in cache there is never
// contention on this semaphore, but when tokens expire there is some. However, after measuring performance
// with and without the semaphore (and different configs for the semaphore), not limiting concurrency actually
// results in higher response times overall. Without the use of this semaphore calls to AcquireTokenAsync can take up
// to 5 seconds under high concurrency scenarios.
acquired = tokenRefreshSemaphore.WaitOne(SemaphoreTimeout);
// If we are allowed to enter the semaphore, acquire the token.
if (acquired)
{
// Acquire token async using MSAL.NET
// https://github.com/AzureAD/azure-activedirectory-library-for-dotnet
// Given that this is a ClientCredential scenario, it will use the cache without the
// need to call AcquireTokenSilentAsync (which is only for user credentials).
// Scenario details: https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/wiki/Client-credential-flows#it-uses-the-application-token-cache
var res = await authContext.AcquireTokenAsync(oAuthConfig.Scope, this.clientCredential).ConfigureAwait(false);
// This means we acquired a valid token successfully. We can make our retry policy null.
// Note that the retry policy is set under the semaphore so no additional synchronization is needed.
if (currentRetryPolicy != null)
{
currentRetryPolicy = null;
}
return(res);
}
else
{
// If the token is taken, it means that one thread is trying to acquire a token from the server.
// If we already received information about how much to throttle, it will be in the currentRetryPolicy.
// Use that to inform our next delay before trying.
throw new ThrottleException()
{
RetryParams = currentRetryPolicy
};
}
}
catch (Exception ex)
{
// If we are getting throttled, we set the retry policy according to the RetryAfter headers
// that we receive from the auth server.
// Note that the retry policy is set under the semaphore so no additional synchronization is needed.
if (IsAdalServiceUnavailable(ex))
{
currentRetryPolicy = ComputeAdalRetry(ex);
}
throw;
}
finally
{
// Always release the semaphore if we acquired it.
if (acquired)
{
tokenRefreshSemaphore.Release();
}
}
}
private async Task <AuthenticatorResult> AcquireTokenAsync(bool forceRefresh = false)
{
if (_clientApplication == null)
{
throw new InvalidOperationException("AcquireTokenAsync should not be called for empty credentials.");
}
bool acquired = false;
try
{
// Limiting concurrency on MSAL token acquisitions. When the Token is in cache there is never
// contention on this semaphore, but when tokens expire there is some. However, after measuring performance
// with and without the semaphore (and different configs for the semaphore), not limiting concurrency actually
// results in higher response times, more throttling and more contention.
// Without the use of this semaphore calls to AcquireTokenAsync can take tens of seconds under high concurrency scenarios.
#pragma warning disable VSTHRD103 // Call async methods when in an async method
acquired = tokenRefreshSemaphore.Wait(SemaphoreTimeout);
#pragma warning restore VSTHRD103 // Call async methods when in an async method
// If we are allowed to enter the semaphore, acquire the token.
if (acquired)
{
// Note that in MSAL, we dont pass resources anymore, and we instead pass scopes. To be recognized by MSAL, we append the '/.default' to the scope.
// Scope requirements described in MSAL migration spec: https://docs.microsoft.com/en-us/azure/active-directory/develop/msal-net-migration.
const string scopePostFix = "/.default";
var scope = _scope ?? OAuthScope;
if (!scope.EndsWith(scopePostFix, StringComparison.OrdinalIgnoreCase))
{
scope = $"{scope}{scopePostFix}";
}
// Acquire token async using MSAL.NET
// This will use the cache from the application cache of the MSAL library, no external caching is needed.
var msalResult = await _clientApplication
.AcquireTokenForClient(new[] { scope })
.WithAuthority(_authority ?? OAuthEndpoint, _validateAuthority)
.WithForceRefresh(forceRefresh)
.ExecuteAsync().ConfigureAwait(false);
// This means we acquired a valid token successfully. We can make our retry policy null.
return(new AuthenticatorResult()
{
AccessToken = msalResult.AccessToken,
ExpiresOn = msalResult.ExpiresOn
});
}
else
{
// If the token is taken, it means that one thread is trying to acquire a token from the server.
// Throttle this request to allow the currently running request to fulfill and then let this one get the result from the cache.
throw new ThrottleException()
{
RetryParams = RetryParams.DefaultBackOff(0)
};
}
}
finally
{
// Always release the semaphore if we acquired it.
if (acquired)
{
ReleaseSemaphore();
}
}
}
