protected override bool ShouldRetryImpl(RetryState retryStateObj)
{
Contract.Assert(retryStateObj is RetryStateEx);
RetryStateEx retryState = (RetryStateEx)retryStateObj;
// Calculate the delay as exponential value based on the number of retries.
retryState.Delay =
RetryPolicyUtils.CalcExponentialRetryDelay(
retryState.RetryCount,
_intervalFactor,
_minInterval,
_maxInterval);
// Add the delay to the total retry time
retryState.TotalRetryTime = retryState.TotalRetryTime + retryState.Delay;
// Calculate the maximum total retry time depending on how long ago was the task (this retry policy) started.
// Longer running tasks are less eager to abort since, more work is has been done.
TimeSpan totalRetryTimeLimit = checked(TimeSpan.FromMilliseconds(
Math.Max(
Math.Min(
_stopwatch.ElapsedMilliseconds * _totalRetryTimeLimitRate,
_maxTotalRetryTimeLimit.TotalMilliseconds),
_minTotalRetryTimeLimit.TotalMilliseconds)));
if (retryState.TotalRetryTime <= totalRetryTimeLimit)
{
return true;
}
retryState.Delay = TimeSpan.Zero;
return false;
}
/// <summary>
/// Notifies the subscribers whenever an error is ignored on retry.
/// </summary>
/// <param name="retryState">The state of current retry attempt.</param>
protected virtual void OnIgnoreErrorOccurred(RetryState retryState)
{
var ignoreErrorOccurred = IgnoreErrorOccurred;
if (ignoreErrorOccurred != null)
{
ignoreErrorOccurred(retryState);
}
}
/// <summary>
/// Notifies the subscribers whenever a retry condition is encountered.
/// </summary>
/// <param name="retryState">The state of current retry attempt.</param>
protected virtual void OnRetryOccurred(RetryState retryState)
{
var retryOccurred = RetryOccurred;
if (retryOccurred != null)
{
retryOccurred(retryState);
}
}
internal static void DataConnectionFailureRetry(RetryState retryState)
{
Logger.Instance.Write(LogLevel.Normal, string.Format(CultureInfo.InvariantCulture,
"Connection retry number {0}. Delaying {1} ms before retry. Exception: {2}",
retryState.RetryCount,
retryState.Delay.TotalMilliseconds.ToString(CultureInfo.InvariantCulture),
retryState.LastError.ToString()));
RetryPolicyUtils.RaiseAmbientRetryMessage(retryState, SqlSchemaModelErrorCodes.ServiceActions.ConnectionRetry);
}
internal static void CommandFailureIgnore(RetryState retryState, string commandKeyword)
{
Logger.Instance.Write(LogLevel.Normal, string.Format(
CultureInfo.InvariantCulture,
"{0} retry number {1}. Ignoring failure. Exception: {2}",
commandKeyword,
retryState.RetryCount,
retryState.LastError.ToString()));
RetryPolicyUtils.RaiseAmbientIgnoreMessage(retryState, SqlSchemaModelErrorCodes.ServiceActions.CommandRetry);
}
internal static void CommandFailureRetry(RetryState retryState, string commandKeyword)
{
Logger.Write(TraceEventType.Information, string.Format(
CultureInfo.InvariantCulture,
"{0} retry number {1}. Delaying {2} ms before retry. Exception: {3}",
commandKeyword,
retryState.RetryCount,
retryState.Delay.TotalMilliseconds.ToString(CultureInfo.InvariantCulture),
retryState.LastError.ToString()));
RetryPolicyUtils.RaiseAmbientRetryMessage(retryState, SqlSchemaModelErrorCodes.ServiceActions.CommandRetry);
}
protected override bool ShouldRetryImpl(RetryState retryState)
{
Contract.Assert(retryState != null);
if (IsLessThanMaxRetryCount(retryState.RetryCount, _maxRetryCount))
{
retryState.Delay = _intervalBetweenRetries;
return true;
}
retryState.Delay = TimeSpan.Zero;
return false;
}
protected override bool ShouldRetryImpl(RetryState retryState)
{
Contract.Assert(retryState != null);
if (IsLessThanMaxRetryCount(retryState.RetryCount, _maxRetryCount))
{
retryState.Delay = RetryPolicyUtils.CalcExponentialRetryDelay(retryState.RetryCount, _intervalFactor, _minInterval, _maxInterval);
return true;
}
retryState.Delay = TimeSpan.Zero;
return false;
}
protected override bool ShouldRetryImpl(RetryState retryState)
{
Contract.Assert(retryState != null);
if (IsLessThanMaxRetryCount(retryState.RetryCount, _maxRetryCount))
{
retryState.Delay = TimeSpan.FromMilliseconds(_initialInterval.TotalMilliseconds + (_increment.TotalMilliseconds * (retryState.RetryCount - 1)));
return true;
}
retryState.Delay = TimeSpan.Zero;
return false;
}
/// <summary>
/// Traces the Schema retry information before raising the retry message
/// </summary>
/// <param name="retryState"></param>
/// <param name="errorCode"></param>
/// <param name="azureSessionId"></param>
internal static void RaiseSchemaAmbientRetryMessage(RetryState retryState, int errorCode, Guid azureSessionId)
{
if (azureSessionId != Guid.Empty)
{
Logger.Write(TraceEventType.Warning, string.Format(
"Retry occurred: session: {0}; attempt - {1}; delay - {2}; exception - \"{3}\"",
azureSessionId,
retryState.RetryCount,
retryState.Delay,
retryState.LastError
));
RaiseAmbientRetryMessage(retryState, errorCode);
}
}
public bool ShouldIgnoreError(RetryState retryState)
{
bool shouldIgnoreError = ErrorDetectionStrategy.ShouldIgnoreError(retryState.LastError);
Logger.Instance.Write(LogLevel.Error,
string.Format(
CultureInfo.InvariantCulture,
"Ignore Error requested: Retry count = {0}. Delay = {1}, SQL Error Number = {2}, Should Ignore Error = {3}",
retryState.RetryCount,
retryState.Delay,
GetErrorNumber(retryState.LastError),
shouldIgnoreError));
return shouldIgnoreError;
}
internal static void RaiseAmbientIgnoreMessage(RetryState retryState, int errorCode)
{
Action <SqlServerRetryError> retryMsgHandler = AmbientSettings.ConnectionRetryMessageHandler;
if (retryMsgHandler != null)
{
string msg = SqlServerRetryError.FormatIgnoreMessage(
retryState.RetryCount,
retryState.LastError);
retryMsgHandler(new SqlServerRetryError(
msg,
retryState.LastError,
retryState.RetryCount,
errorCode,
ErrorSeverity.Warning));
}
}
public bool ShouldRetry(RetryState retryState)
{
bool canRetry = ErrorDetectionStrategy.CanRetry(retryState.LastError);
bool shouldRetry = canRetry
&& ShouldRetryImpl(retryState);
Logger.Instance.Write(LogLevel.Error,
string.Format(
CultureInfo.InvariantCulture,
"Retry requested: Retry count = {0}. Delay = {1}, SQL Error Number = {2}, Can retry error = {3}, Will retry = {4}",
retryState.RetryCount,
retryState.Delay,
GetErrorNumber(retryState.LastError),
canRetry,
shouldRetry));
// Perform an extra check in the delay interval. Should prevent from accidentally ending up with the value of -1 which will block a thread indefinitely.
// In addition, any other negative numbers will cause an ArgumentOutOfRangeException fault which will be thrown by Thread.Sleep.
if (retryState.Delay.TotalMilliseconds < 0)
{
retryState.Delay = TimeSpan.Zero;
}
return shouldRetry;
}
internal static void ElementCommandFailureIgnore(RetryState retryState)
{
CommandFailureIgnore(retryState, "Element Command");
}
internal static void CreateDatabaseCommandFailureIgnore(RetryState retryState)
{
CommandFailureIgnore(retryState, "Database Command");
}
internal static void CommandFailureIgnore(RetryState retryState)
{
CommandFailureIgnore(retryState, "Command");
}
internal static void CommandFailureRetry(RetryState retryState)
{
CommandFailureRetry(retryState, "Command");
}
/// <summary>
/// Repetitively executes the specified action while it satisfies the current retry policy.
/// </summary>
/// <typeparam name="R">The type of result expected from the executable action.</typeparam>
/// <param name="func">A delegate representing the executable action which returns the result of type R.</param>
/// <param name="token">Cancellation token to cancel action between retries.</param>
/// <returns>The result from the action.</returns>
public R ExecuteAction<R>(Func<RetryState, R> func, CancellationToken? token = null)
{
RetryState retryState = CreateRetryState();
if (token != null)
{
token.Value.ThrowIfCancellationRequested();
}
while (true)
{
try
{
return func(retryState);
}
catch (RetryLimitExceededException limitExceededEx)
{
// The user code can throw a RetryLimitExceededException to force the exit from the retry loop.
// The RetryLimitExceeded exception can have an inner exception attached to it. This is the exception
// which we will have to throw up the stack so that callers can handle it.
if (limitExceededEx.InnerException != null)
{
throw limitExceededEx.InnerException;
}
return default(R);
}
catch (Exception ex)
{
retryState.LastError = ex;
if (retryState.RetryCount > 0 || this.ShouldIgnoreOnFirstTry)
{
// If we can ignore this error, then break out of the loop and consider this execution as passing
// We return the default value for the type R
if (ShouldIgnoreError(retryState))
{
OnIgnoreErrorOccurred(retryState);
return default(R);
}
}
retryState.RetryCount++;
if (!ShouldRetry(retryState))
{
throw;
}
}
OnRetryOccurred(retryState);
if ((retryState.RetryCount > 1 || !FastFirstRetry) && !retryState.IsDelayDisabled)
{
Thread.Sleep(retryState.Delay);
}
// check for cancellation after delay.
if (token != null)
{
token.Value.ThrowIfCancellationRequested();
}
}
}
private void RetryConnectionCallback(RetryState retryState)
{
RetryPolicyUtils.RaiseSchemaAmbientRetryMessage(retryState, SqlSchemaModelErrorCodes.ServiceActions.ConnectionRetry, _azureSessionId);
}
protected abstract bool ShouldRetryImpl(RetryState retryState);
