public override void HandleAfter(Context context)
{
var retryPolicy = context.ClientConfig.RetryPolicy;
while (true)
{
OTSException exceptionForRetry = null;
try
{
InnerHandler.HandleAfter(context);
}
catch (OTSClientException exception)
{
exceptionForRetry = exception;
}
catch (OTSServerException exception)
{
exceptionForRetry = exception;
}
if (OTSClientTestHelper.RetryTimesAndBackOffRecordSwith)
{
if (OTSClientTestHelper.RetryExceptions.Count() > OTSClientTestHelper.RetryTimes)
{
exceptionForRetry = OTSClientTestHelper.RetryExceptions[OTSClientTestHelper.RetryTimes];
}
}
if (ShouldRetry(retryPolicy, context, exceptionForRetry))
{
int retryDelay = retryPolicy.DelayBeforeNextRetry(context, exceptionForRetry);
Thread.Sleep(retryDelay);
ResetRetry(context);
context.RetryTimes += 1;
if (OTSClientTestHelper.RetryTimesAndBackOffRecordSwith)
{
OTSClientTestHelper.RetryTimes += 1;
OTSClientTestHelper.RetryDelays.Add(retryDelay);
}
continue;
}
if (exceptionForRetry != null)
{
throw exceptionForRetry;
}
// TODO handle retry in BatchWriteRow & BatchGetRow
return;
}
}
private bool ShouldRetry(RetryPolicy retryPolicy, Context context, OTSException exception)
{
if (retryPolicy.MaxRetryTimeReached(context, exception))
{
return(false);
}
if (retryPolicy.CanRetry(context, exception))
{
return(true);
}
return(false);
}
public static bool IsServerThrottlingException(OTSException exception)
{
var e = exception as OTSServerException;
if (e != null)
{
if (e.ErrorCode == "OTSServerBusy" ||
e.ErrorCode == "OTSNotEnoughCapacityUnit" ||
(e.ErrorCode == "OTSQuotaExhausted" && e.ErrorMessage == "Too frequent table operations."))
{
return(true);
}
}
return(false);
}
public override bool CanRetry(Context context, OTSException exception)
{
if (exception == null)
{
// No exception ocurred
return(false);
}
if (RetryUtil.ShouldRetryNoMatterWhichAPI(exception))
{
return(true);
}
if (RetryUtil.IsRepeatableAPI(context.APIName) &&
RetryUtil.ShouldRetryWhenAPIRepeatable(exception))
{
return(true);
}
return(false);
}
public static bool ShouldRetryWhenAPIRepeatable(OTSException exception)
{
var e = exception as OTSServerException;
if (e != null)
{
if (e.ErrorCode == "OTSTimeout" ||
e.ErrorCode == "OTSInternalServerError" ||
e.ErrorCode == "OTSServerUnavailable")
{
return(true);
}
int code = (int)e.HttpStatusCode;
if (code == 500 || code == 502 || code == 503)
{
return(true);
}
// TODO handle network error & timeout
}
return(false);
}
public static bool ShouldRetryNoMatterWhichAPI(OTSException exception)
{
var e = exception as OTSServerException;
if (e != null)
{
if (e.ErrorCode == "OTSRowOperationConflict" ||
e.ErrorCode == "OTSNotEnoughCapacityUnit" ||
e.ErrorCode == "OTSTableNotReady" ||
e.ErrorCode == "OTSPartitionUnavailable" ||
e.ErrorCode == "OTSServerBusy")
{
return(true);
}
if (e.ErrorCode == "OTSQuotaExhausted" &&
e.ErrorMessage == "Too frequent table operations.")
{
return(true);
}
}
return(false);
}
public override int DelayBeforeNextRetry(Context context, OTSException exception)
{
int delayFactor;
if (RetryUtil.IsServerThrottlingException(exception))
{
delayFactor = ServerThrottlingExceptionDelayFactor;
}
else
{
delayFactor = StabilityExceptionDelayFactor;
}
int delayLimit = delayFactor * (int)Math.Pow(ScaleFactor, context.RetryTimes);
if (delayLimit >= MaxDelay)
{
delayLimit = MaxDelay;
}
int realDelay = RandomGenerator.Next(delayLimit / 2, delayLimit);
return(realDelay);
}
public override bool MaxRetryTimeReached(Context context, OTSException exception)
{
return(context.RetryTimes >= MaxRetryTimes);
}
public override bool CanRetry(Context context, OTSException exception)
{
return(false);
}
public override int DelayBeforeNextRetry(Context context, OTSException exception)
{
return(0);
}
public abstract int DelayBeforeNextRetry(Context context, OTSException exception);
public abstract bool CanRetry(Context context, OTSException exception);
public abstract bool MaxRetryTimeReached(Context context, OTSException exception);