private async ValueTask RefreshCachedTokenWithRetryHelperAsync(ITrace trace) { // A different thread is already updating the access token. Count starts off at 1. bool skipRefreshBecause = this.backgroundRefreshLock.CurrentCount != 1; await this.backgroundRefreshLock.WaitAsync(); try { // Token was already refreshed successfully from another thread. if (skipRefreshBecause && this.cachedAccessToken.ExpiresOn > DateTime.UtcNow) { return; } Exception lastException = null; const int totalRetryCount = 3; for (int retry = 0; retry < totalRetryCount; retry++) { if (this.cancellationToken.IsCancellationRequested) { DefaultTrace.TraceInformation( "Stop RefreshTokenWithIndefiniteRetries because cancellation is requested"); break; } using (ITrace getTokenTrace = trace.StartChild( name: nameof(this.RefreshCachedTokenWithRetryHelperAsync), component: TraceComponent.Authorization, level: Tracing.TraceLevel.Info)) { try { await this.ExecuteGetTokenWithRequestTimeoutAsync(); return; } catch (RequestFailedException requestFailedException) { lastException = requestFailedException; getTokenTrace.AddDatum( $"RequestFailedException at {DateTime.UtcNow.ToString(CultureInfo.InvariantCulture)}", requestFailedException); DefaultTrace.TraceError($"TokenCredential.GetToken() failed with RequestFailedException. scope = {string.Join(";", this.tokenRequestContext.Scopes)}, retry = {retry}, Exception = {lastException}"); // Don't retry on auth failures if (requestFailedException.Status == (int)HttpStatusCode.Unauthorized || requestFailedException.Status == (int)HttpStatusCode.Forbidden) { this.cachedAccessToken = default; throw; } } catch (OperationCanceledException operationCancelled) { lastException = operationCancelled; getTokenTrace.AddDatum( $"OperationCanceledException at {DateTime.UtcNow.ToString(CultureInfo.InvariantCulture)}", operationCancelled); DefaultTrace.TraceError( $"TokenCredential.GetTokenAsync() failed. scope = {string.Join(";", this.tokenRequestContext.Scopes)}, retry = {retry}, Exception = {lastException}"); throw CosmosExceptionFactory.CreateRequestTimeoutException( message: ClientResources.FailedToGetAadToken, headers: new Headers() { SubStatusCode = SubStatusCodes.FailedToGetAadToken, }, innerException: lastException, trace: getTokenTrace); } catch (Exception exception) { lastException = exception; getTokenTrace.AddDatum( $"Exception at {DateTime.UtcNow.ToString(CultureInfo.InvariantCulture)}", exception); DefaultTrace.TraceError( $"TokenCredential.GetTokenAsync() failed. scope = {string.Join(";", this.tokenRequestContext.Scopes)}, retry = {retry}, Exception = {lastException}"); } } DefaultTrace.TraceError( $"TokenCredential.GetTokenAsync() failed. scope = {string.Join(";", this.tokenRequestContext.Scopes)}, retry = {retry}, Exception = {lastException}"); } throw CosmosExceptionFactory.CreateUnauthorizedException( message: ClientResources.FailedToGetAadToken, headers: new Headers() { SubStatusCode = SubStatusCodes.FailedToGetAadToken, }, innerException: lastException, trace: trace); } finally { this.backgroundRefreshLock.Release(); } }