public void TestRetryBackOffWithOtherExceptions()
{
OTSClientTestHelper.TurnOnRetryTimesAndBackOffRecording();
var e1 = new OTSServerException("/ListTable",
HttpStatusCode.ServiceUnavailable,
"OTSRowOperationConflict",
"Data is being modified by the other request.");
var e2 = new OTSServerException("/ListTable",
HttpStatusCode.ServiceUnavailable,
"OTSTimeout",
"Operation timeout.");
testRetryWithException(new OTSServerException[] { e1, e1, e1, e1 });
assertRetryDelay(0, 100, 200);
assertRetryDelay(1, 200, 400);
assertRetryDelay(2, 400, 800);
OTSClientTestHelper.Reset();
OTSClientTestHelper.TurnOnRetryTimesAndBackOffRecording();
testRetryWithException(new OTSServerException[] { e2, e2, e2, e2 });
assertRetryDelay(0, 100, 200);
assertRetryDelay(1, 200, 400);
assertRetryDelay(2, 400, 800);
OTSClientTestHelper.Reset();
OTSClientTestHelper.TurnOnRetryTimesAndBackOffRecording();
}
private void throwOTSServerException(Context context)
{
var exception = new OTSServerException(context.APIName, context.HttpResponseStatusCode);
if (context.ClientConfig.OTSErrorLogHandler != null)
{
context.ClientConfig.OTSErrorLogHandler(exception.ToString() + "\n");
}
throw exception;
}
public override void HandleAfter(Context context)
{
OTSServerException exception;
InnerHandler.HandleAfter(context);
var statusCode = context.HttpResponseStatusCode;
if ((int)statusCode >= 200 && (int)statusCode < 300)
{
return;
}
var builder = PB.Error.CreateBuilder();
string errorCode = null, errorMessage = null;
try
{
builder.MergeFrom(context.HttpResponseBody);
var message = builder.Build();
errorCode = message.Code;
errorMessage = message.Message;
} catch (Google.ProtocolBuffers.InvalidProtocolBufferException) {
throwOTSServerException(context);
} catch (Google.ProtocolBuffers.UninitializedMessageException) {
throwOTSServerException(context);
}
string requestID;
if (context.HttpResponseHeaders.ContainsKey("x-ots-requestid"))
{
requestID = context.HttpResponseHeaders["x-ots-requestid"];
}
else
{
requestID = null;
}
exception = new OTSServerException(
context.APIName,
statusCode,
errorCode,
errorMessage,
requestID
);
if (context.ClientConfig.OTSErrorLogHandler != null)
{
context.ClientConfig.OTSErrorLogHandler.Invoke(exception.ToString());
}
throw exception;
}
public static void AssertOTSServerException(OTSServerException expect, OTSServerException actual)
{
if (expect.APIName != actual.APIName ||
expect.HttpStatusCode != actual.HttpStatusCode ||
expect.ErrorCode != actual.ErrorCode ||
expect.ErrorMessage != actual.ErrorMessage)
{
throw new AssertionException(String.Format(
"OTSServerException Assert Failed. expect: {0} actual {1}",
expect.Message, actual.Message
));
}
}
public void TestRetryWithExceptionChanged()
{
var e1 = new OTSServerException("/ListTable",
HttpStatusCode.ServiceUnavailable,
"OTSRowOperationConflict",
"Data is being modified by the other request.");
var e3 = new OTSServerException("/ListTable",
HttpStatusCode.ServiceUnavailable,
"OTSQuotaExhausted",
"Too frequent table operations.");
OTSClientTestHelper.TurnOnRetryTimesAndBackOffRecording();
testRetry(new OTSServerException[] { e1, e3, e3 });
assertRetryDelay(0, 100, 200);
assertRetryDelay(1, 500, 1000);
assertRetryDelay(2, 1000, 2000);
OTSClientTestHelper.Reset();
}
void TestRetryCondition(int[][] retryConditionTable, HttpStatusCode httpCode)
{
for (int i = 0; i < apiNames.Count(); i++)
{
for (int j = 0; j < errors.Count(); j++)
{
var apiName = "/" + apiNames[i];
var errorCode = errors[j][0];
var errorMessage = errors[j][1];
var expect = retryConditionTable[j][i] == 1;
var exception = new OTSServerException(apiName, httpCode, errorCode, errorMessage);
var context = new Context();
context.APIName = apiName;
Assert.AreEqual(expect, retryPolicy.CanRetry(context, exception),
"Retry Condition Not Match: API Name {0}, Error Code: {1}",
apiName, errorCode);
}
}
}
public void TestRetryBackOffWithServerThrottlingException()
{
OTSClientTestHelper.TurnOnRetryTimesAndBackOffRecording();
var e1 = new OTSServerException("/ListTable",
HttpStatusCode.ServiceUnavailable,
"OTSServerBusy",
"Server is busy.");
var e2 = new OTSServerException("/ListTable",
HttpStatusCode.ServiceUnavailable,
"OTSNotEnoughCapacityUnit",
"Remaining capacity unit is not enough.");
var e3 = new OTSServerException("/ListTable",
HttpStatusCode.ServiceUnavailable,
"OTSQuotaExhausted",
"Too frequent table operations.");
testRetryWithException(new OTSServerException[] { e1, e1, e1, e1 });
assertRetryDelay(0, 250, 500);
assertRetryDelay(1, 500, 1000);
assertRetryDelay(2, 1000, 2000);
OTSClientTestHelper.Reset();
OTSClientTestHelper.TurnOnRetryTimesAndBackOffRecording();
testRetryWithException(new OTSServerException[] { e2, e2, e2, e2 });
assertRetryDelay(0, 250, 500);
assertRetryDelay(1, 500, 1000);
assertRetryDelay(2, 1000, 2000);
OTSClientTestHelper.Reset();
OTSClientTestHelper.TurnOnRetryTimesAndBackOffRecording();
testRetryWithException(new OTSServerException[] { e3, e3, e3, e3 });
assertRetryDelay(0, 250, 500);
assertRetryDelay(1, 500, 1000);
assertRetryDelay(2, 1000, 2000);
OTSClientTestHelper.Reset();
}
public void TestRetryTwice()
{
OTSClientTestHelper.TurnOnRetryTimesAndBackOffRecording();
var e1 = new OTSServerException("/ListTable",
HttpStatusCode.ServiceUnavailable,
"OTSRowOperationConflict",
"Data is being modified by the other request.");
var e2 = new OTSServerException("/ListTable",
HttpStatusCode.ServiceUnavailable,
"OTSTimeout",
"Operation timeout.");
var e3 = new OTSServerException("/ListTable",
HttpStatusCode.ServiceUnavailable,
"OTSQuotaExhausted",
"Too frequent table operations.");
TestRetry(new OTSServerException[] { e1, e1 });
AssertRetryDelay(0, 99, 200);
AssertRetryDelay(1, 199, 400);
OTSClientTestHelper.Reset();
OTSClientTestHelper.TurnOnRetryTimesAndBackOffRecording();
TestRetry(new OTSServerException[] { e2, e2 });
AssertRetryDelay(0, 99, 200);
AssertRetryDelay(1, 199, 400);
OTSClientTestHelper.Reset();
OTSClientTestHelper.TurnOnRetryTimesAndBackOffRecording();
TestRetry(new OTSServerException[] { e3, e3, e3 });
AssertRetryDelay(0, 249, 500);
AssertRetryDelay(1, 499, 1000);
OTSClientTestHelper.Reset();
}
