bool IExceptionHandler.TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is FabricNotPrimaryException)
{
if (exceptionInformation.TargetReplica == TargetReplicaSelector.Default)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, int.MaxValue);
return(true);
}
//ServiceTrace.Source.WriteInfo(ServiceRemotingExceptionHandler.TraceType, "{0} Got exception {1} which does not match the replica target : {2}", (object) this._traceId, (object) exceptionInformation.Exception, (object) exceptionInformation.TargetReplica);
result = null;
return(false);
}
if (exceptionInformation.Exception is FabricNotReadableException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, int.MaxValue);
return(true);
}
if (exceptionInformation.Exception is FabricTransientException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, true, retrySettings, int.MaxValue);
return(true);
}
result = null;
return(false);
}
/// <summary>
/// Method that examines the exception and determines how that exception can be handled.
/// </summary>
/// <param name="exceptionInformation">Information about the exception</param>
/// <param name="retrySettings">The operation retry preferences.</param>
/// <param name="result">Result of the exception handling</param>
/// <returns>true if the exception is handled, false otherwise</returns>
bool IExceptionHandler.TryHandleException(
ExceptionInformation exceptionInformation,
OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
var e = exceptionInformation.Exception;
if (e is ActorConcurrencyLockTimeoutException)
{
if (ActorLogicalCallContext.IsPresent())
{
result = new ExceptionHandlingThrowResult()
{
ExceptionToThrow = e
};
return(true);
}
result = new ExceptionHandlingRetryResult(
e,
true,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
result = null;
return(false);
}
private static bool TryHandleFabricException(
FabricException fabricException,
OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
if ((fabricException is FabricCannotConnectException) ||
(fabricException is FabricEndpointNotFoundException)
)
{
result = new ExceptionHandlingRetryResult(
fabricException,
false,
retrySettings,
int.MaxValue);
return(true);
}
if (fabricException.ErrorCode.Equals(FabricErrorCode.ServiceTooBusy))
{
result = new ExceptionHandlingRetryResult(
fabricException,
true,
retrySettings,
int.MaxValue);
return(true);
}
result = null;
return(false);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
this.logger.LogError("retry {RetryAttempt} for {replica}: {exception}", retrySettings.RetryPolicy.TotalNumberOfRetries, exceptionInformation.TargetReplica.ToString(), exceptionInformation.Exception.Message);
if (IsNotTransient(exceptionInformation))
{
this.logger.LogError(exceptionInformation.Exception, "A known non-transient exception occurred on retry {RetryAttempt} for {replica}: {exception}", retrySettings.RetryPolicy.TotalNumberOfRetries, exceptionInformation.TargetReplica.ToString(), exceptionInformation.Exception.Message);
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, TransientException.IsNotTransient, TimeSpan.FromSeconds(1), retrySettings.DefaultMaxRetryCountForNonTransientErrors);
return(true);
}
// if exceptionInformation.Exception is known and is transient (can be retried without re-resolving)
if (IsTransient(exceptionInformation))
{
this.logger.LogError(exceptionInformation.Exception, "A known transient exception occurred on retry {RetryAttempt} for {replica}: {exception}", retrySettings.RetryPolicy.TotalNumberOfRetries, exceptionInformation.TargetReplica.ToString(), exceptionInformation.Exception.Message);
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, TransientException.IsTransient, TimeSpan.FromSeconds(1), retrySettings.DefaultMaxRetryCountForTransientErrors);
return(true);
}
// if exceptionInformation.Exception is unknown (let the next IExceptionHandler attempt to handle it)
this.logger.LogError(exceptionInformation.Exception, "A unknown exception occurred on retry {RetryAttempt} for {replica}: {exception}", retrySettings.RetryPolicy.TotalNumberOfRetries, exceptionInformation.TargetReplica.ToString(), exceptionInformation.Exception.Message);
result = null;
return(false);
}
protected bool HandleRemoteException(Exception e, out ExceptionHandlingResult result)
{
if (e is FabricTransientException)
{
result = new ExceptionHandlingRetryResult
{
IsTransient = true,
RetryDelay = TimeSpan.FromMilliseconds(ThreadSafeRandom.Value.NextDouble() * WcfFactory.MaxRetryBackoffIntervalOnTransientErrors.TotalMilliseconds)
};
return true;
}
if (e is FabricNotPrimaryException)
{
result = new ExceptionHandlingRetryResult
{
IsTransient = false,
RetryDelay = TimeSpan.FromMilliseconds(ThreadSafeRandom.Value.NextDouble() * WcfFactory.MaxRetryBackoffIntervalOnNonTransientErrors.TotalMilliseconds)
};
return true;
}
result = new ExceptionHandlingThrowResult
{
ExceptionToThrow = new AggregateException(e)
};
return true;
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is RpcException rpcEx)
{
switch (rpcEx.Status.StatusCode)
{
case StatusCode.Unavailable when rpcEx.Status.Detail == "Endpoint read failed":
Log.LogInformation(exceptionInformation.Exception, "Throwing: {Exception}", exceptionInformation.Exception.Message);
result = new ExceptionHandlingThrowResult();
return(true);
case StatusCode.Unavailable:
case StatusCode.Unknown:
case StatusCode.Cancelled:
Log.LogInformation(exceptionInformation.Exception, "Not transient exception: {Exception}, Retry {@Retry}", exceptionInformation.Exception.Message, retrySettings);
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, int.MaxValue);
return(true);
default:
Log.LogInformation(exceptionInformation.Exception, "Unknown exception: {Exception}, Retry {@Retry}", exceptionInformation.Exception.Message, retrySettings);
result = new ExceptionHandlingThrowResult();
return(true);
}
}
else
{
Log.LogInformation(exceptionInformation.Exception, "Throwing: {Exception}", exceptionInformation.Exception.Message);
result = new ExceptionHandlingThrowResult();
return(true);
}
}
private bool CreateExceptionHandlingRetryResult(bool isTransient, Exception ex, out ExceptionHandlingResult result)
{
result = new ExceptionHandlingRetryResult(
exceptionId: ex.GetType().Name,
isTransient: isTransient,
retryDelay: TimeSpan.FromMilliseconds(_rand.NextDouble() * _options.MaxRetryBackoffInterval.TotalMilliseconds),
maxRetryCount: _options.MaxRetryCount);
return(true);
}
private bool CreateExceptionHandlingResult(bool isTransient, out ExceptionHandlingResult result)
{
result = new ExceptionHandlingRetryResult()
{
IsTransient = isTransient,
RetryDelay = TimeSpan.FromMilliseconds(MaxRetryBackoffIntervalOnNonTransientErrors.TotalMilliseconds),
};
return(true);
}
/// <summary>
/// Method that examines the exception and determines how that exception can be handled.
/// </summary>
/// <param name="exceptionInformation">Information about the exception</param>
/// <param name="retrySettings">The operation retry preferences.</param>
/// <param name="result">Result of the exception handling</param>
/// <returns>true if the exception is handled, false otherwise</returns>
bool IExceptionHandler.TryHandleException(
ExceptionInformation exceptionInformation,
OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is FabricNotPrimaryException)
{
if (exceptionInformation.TargetReplica == TargetReplicaSelector.PrimaryReplica)
{
result = new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
false,
retrySettings,
int.MaxValue);
return(true);
}
ServiceTrace.Source.WriteInfo(
TraceType,
"{0} Got exception {1} which does not match the replica target : {2}",
this.traceId,
exceptionInformation.Exception,
exceptionInformation.TargetReplica);
result = null;
return(false);
}
if (exceptionInformation.Exception is FabricNotReadableException)
{
result = new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
false,
retrySettings,
int.MaxValue);
return(true);
}
// Note: This code handles retries for FabricTransientException even from Actors, eg. ActorDeletedException.
if (exceptionInformation.Exception is FabricTransientException)
{
result = new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
true,
retrySettings,
int.MaxValue);
return(true);
}
result = null;
return(false);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is TimeoutException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
else if (exceptionInformation.Exception is ProtocolViolationException)
{
result = new ExceptionHandlingThrowResult();
return true;
}
else if (exceptionInformation.Exception is WebException)
{
WebException we = exceptionInformation.Exception as WebException;
HttpWebResponse errorResponse = we.Response as HttpWebResponse;
if (we.Status == WebExceptionStatus.ProtocolError)
{
if (errorResponse.StatusCode == HttpStatusCode.NotFound)
{
// This could either mean we requested an endpoint that does not exist in the service API (a user error)
// or the address that was resolved by fabric client is stale (transient runtime error) in which we should re-resolve.
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
if (errorResponse.StatusCode == HttpStatusCode.InternalServerError)
{
// The address is correct, but the server processing failed.
// Retry the operation without re-resolving the address.
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, true, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
}
if (we.Status == WebExceptionStatus.Timeout ||
we.Status == WebExceptionStatus.RequestCanceled ||
we.Status == WebExceptionStatus.ConnectionClosed ||
we.Status == WebExceptionStatus.ConnectFailure)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
}
result = null;
return false;
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is TimeoutException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
else if (exceptionInformation.Exception is ProtocolViolationException)
{
result = new ExceptionHandlingThrowResult();
return(true);
}
else if (exceptionInformation.Exception is WebException)
{
WebException we = exceptionInformation.Exception as WebException;
HttpWebResponse errorResponse = we.Response as HttpWebResponse;
if (we.Status == WebExceptionStatus.ProtocolError)
{
if (errorResponse.StatusCode == HttpStatusCode.NotFound)
{
// This could either mean we requested an endpoint that does not exist in the service API (a user error)
// or the address that was resolved by fabric client is stale (transient runtime error) in which we should re-resolve.
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
if (errorResponse.StatusCode == HttpStatusCode.InternalServerError)
{
// The address is correct, but the server processing failed.
// Retry the operation without re-resolving the address.
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, true, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
}
if (we.Status == WebExceptionStatus.Timeout ||
we.Status == WebExceptionStatus.RequestCanceled ||
we.Status == WebExceptionStatus.ConnectionClosed ||
we.Status == WebExceptionStatus.ConnectFailure)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
}
result = null;
return(false);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
// if exceptionInformation.Exception is known and is transient (can be retried without re-resolving)
//TODO: Add decision logic to determine whether exception is transient or not.
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, true, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
//// if exceptionInformation.Exception is known and is not transient (indicates a new service endpoint address must be resolved)
//result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
//return true;
//// if exceptionInformation.Exception is unknown (let the next IExceptionHandler attempt to handle it)
//result = null;
//return false;
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation == null)
{
throw new ArgumentNullException(nameof(exceptionInformation));
}
if (retrySettings == null)
{
throw new ArgumentNullException(nameof(retrySettings));
}
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation == null)
{
throw new ArgumentNullException(nameof(exceptionInformation));
}
if (retrySettings == null)
{
throw new ArgumentNullException(nameof(retrySettings));
}
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
protected override bool OnHandleException(Exception ex, out ExceptionHandlingResult result)
{
//
// TODO:
// Analyze the given exception and return a proper result.
result = new ExceptionHandlingRetryResult()
{
ExceptionId = ex.GetType().GUID.ToString(),
IsTransient = false,
MaxRetryCount = 5,
RetryDelay = TimeSpan.FromSeconds(1)
};
return(true);
}
public bool TryHandleException(
ExceptionInformation exceptionInformation,
OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is InvalidOperationException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception,
isTransient: true,
retryDelay: TimeSpan.FromSeconds(2),
maxRetryCount: 3);
return(true);
}
result = new ExceptionHandlingThrowResult();
return(false);
}
public bool TryHandleException(
ExceptionInformation exceptionInformation,
OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is TException)
{
result = new ExceptionHandlingRetryResult(
exception: exceptionInformation.Exception,
isTransient: _isTransient,
retryDelay: _retryDelay,
maxRetryCount: _maxRetryCount);
return(true);
}
result = new ExceptionHandlingThrowResult();
return(false);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is FabricCannotConnectException)
{
// <param name="isTransient">
// Indicates if this is a transient retriable exception.
// Transient retriable exceptions are those where the communication channel from client
// to service still exists.
// Non transient retriable exceptions are those where we need to re-resolve the service endpoint
// before we retry.
// </param>
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
result = null;
return(false);
}
bool IExceptionHandler.TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is TimeoutException ||
exceptionInformation.Exception is ConnectionFailedException ||
exceptionInformation.Exception is OperationCanceledException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, 1);
return(true);
}
else if (exceptionInformation.Exception is ProcessingException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, 0);
return(true);
}
result = null;
return(false);
}
/// <summary>
/// Method that examines the exception and determines how that exception can be handled.
/// </summary>
/// <param name="exceptionInformation">Information about the exception</param>
/// <param name="retrySettings">The operation retry preferences.</param>
/// <param name="result">Result of the exception handling</param>
/// <returns>true if the exception is handled, false otherwise</returns>
bool IExceptionHandler.TryHandleException(
ExceptionInformation exceptionInformation,
OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
var e = exceptionInformation.Exception;
if (e is ActorConcurrencyLockTimeoutException)
{
if (ActorLogicalCallContext.IsPresent())
{
result = new ExceptionHandlingThrowResult()
{
ExceptionToThrow = e
};
return(true);
}
result = new ExceptionHandlingRetryResult(
e,
true,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
// The messaging layer may deliver duplicate messages during the connection failures.
//E.g when client connection is disconnected but service is still processing the message. We retry on client connection failures.
//This results to service receiving duplicate message.
//And Actor Reentrancy throws DuplicateMessageException exception when it sees a duplicate Message (message with same callContext).
if (e is DuplicateMessageException)
{
result = new ExceptionHandlingRetryResult(
e,
true,
retrySettings,
int.MaxValue);
return(true);
}
result = null;
return(false);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
_context.WriteEvent($"HttpExceptionHandler::TryHandleException: Encountered Exception {exceptionInformation.Exception}");
if (exceptionInformation.Exception is TimeoutException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
else if (exceptionInformation.Exception is ProtocolViolationException)
{
result = new ExceptionHandlingThrowResult();
return(true);
}
else if (exceptionInformation.Exception is SocketException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
HttpRequestException httpException = exceptionInformation.Exception as HttpRequestException;
if (httpException != null)
{
_context.WriteEvent($"HttpExceptionHandler::TryHandleException: HttpRequestException {httpException}");
result = null;
return(false);
}
WebException we = exceptionInformation.Exception as WebException;
if (we == null)
{
we = exceptionInformation.Exception.InnerException as WebException;
}
if (we != null)
{
_context.WriteEvent($"HttpExceptionHandler::TryHandleException: WebException {we}");
HttpWebResponse errorResponse = we.Response as HttpWebResponse;
if (we.Status == WebExceptionStatus.ProtocolError)
{
_context.WriteEvent($"HttpExceptionHandler::TryHandleException: HttpWebResponse {errorResponse}");
_context.WriteEvent($"HttpExceptionHandler::TryHandleException: HttpWebResponse Status Code {errorResponse.StatusCode} Decription = {errorResponse.StatusDescription}");
result = null;
return(false);
}
if (we.Status == WebExceptionStatus.Timeout ||
we.Status == WebExceptionStatus.RequestCanceled ||
we.Status == WebExceptionStatus.ConnectionClosed ||
we.Status == WebExceptionStatus.ConnectFailure)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
}
result = null;
return(false);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is TimeoutException)
{
result =
new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
false,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
if (exceptionInformation.Exception is SocketException)
{
result =
new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
false,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
if (exceptionInformation.Exception is ProtocolViolationException)
{
result = new ExceptionHandlingThrowResult();
return(true);
}
var we = exceptionInformation.Exception.InnerException as WebException ??
exceptionInformation.Exception.InnerException as WebException;
if (we != null)
{
var errorResponse = we.Response as HttpWebResponse;
if (we.Status == WebExceptionStatus.ProtocolError)
{
if (errorResponse != null && errorResponse.StatusCode == HttpStatusCode.NotFound)
{
result =
new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
false,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
if (errorResponse != null && errorResponse.StatusCode == HttpStatusCode.InternalServerError)
{
result =
new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
true,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
}
if (we.Status == WebExceptionStatus.Timeout ||
we.Status == WebExceptionStatus.RequestCanceled ||
we.Status == WebExceptionStatus.ConnectionClosed ||
we.Status == WebExceptionStatus.ConnectFailure)
{
result =
new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
false,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
}
result = null;
return(false);
}
/// <summary>
/// Method that examines the exception and determines how that exception can be handled.
/// </summary>
/// <param name="exceptionInformation">Information about the exception</param>
/// <param name="retrySettings">The operation retry preferences.</param>
/// <param name="result">Result of the exception handling</param>
/// <returns>true if the exception is handled, false otherwise</returns>
bool IExceptionHandler.TryHandleException(
ExceptionInformation exceptionInformation,
OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
var e = exceptionInformation.Exception;
// retry with resolve - these exceptions indicate a possible fail over
if ((e is EndpointNotFoundException) ||
(e is CommunicationObjectAbortedException) ||
(e is CommunicationObjectFaultedException) ||
(e is ObjectDisposedException) ||
(e is ChannelTerminatedException))
{
result = new ExceptionHandlingRetryResult(
e,
false,
retrySettings,
int.MaxValue);
return(true);
}
// retry on timeout and service busy exceptions
if ((e is TimeoutException) ||
(e is ServerTooBusyException))
{
result = new ExceptionHandlingRetryResult(
e,
true,
retrySettings,
int.MaxValue);
return(true);
}
// Derived types of Communication Exception that are not retriable.
if ((e is ActionNotSupportedException) ||
(e is AddressAccessDeniedException))
{
result = new ExceptionHandlingThrowResult()
{
ExceptionToThrow = e
};
return(true);
}
// Security related derived types of Communication Exception that are not retriable
if (e is SecurityAccessDeniedException)
{
result = new ExceptionHandlingThrowResult()
{
ExceptionToThrow = e
};
return(true);
}
var faultException = e as FaultException;
if (faultException != null)
{
if (faultException.Code.Name == WcfRemoteExceptionInformation.FaultCodeName)
{
var actualException = WcfRemoteExceptionInformation.ToException(faultException.Reason.ToString());
if (faultException.Code.SubCode.Name == WcfRemoteExceptionInformation.FaultSubCodeRetryName)
{
result = new ExceptionHandlingRetryResult(
actualException,
false,
retrySettings,
int.MaxValue);
return(true);
}
}
}
// retry on all communication exceptions, including the protocol exceptions for default max retry
if ((faultException == null) && (e is CommunicationException))
{
result = new ExceptionHandlingRetryResult(
e,
false,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
result = null;
return(false);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
Exception e = exceptionInformation.Exception;
Logger.Debug("OnHandleException {0}", e);
if (e is TimeoutException)
{
result = new ExceptionHandlingRetryResult(e, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
if (e is ProtocolViolationException)
{
result = new ExceptionHandlingRetryResult(e, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
if (e is HttpRequestException)
{
HttpRequestException he = (HttpRequestException)e;
// this should not happen, but let's do a sanity check
if (null == he.InnerException)
{
result = new ExceptionHandlingRetryResult(e, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
e = he.InnerException;
}
if (e is WebException)
{
WebException we = (WebException)e;
HttpWebResponse errorResponse = we.Response as HttpWebResponse;
if (we.Status == WebExceptionStatus.ProtocolError)
{
if (errorResponse.StatusCode == HttpStatusCode.NotFound)
{
// This could either mean we requested an endpoint that does not exist in the service API (a user error)
// or the address that was resolved by fabric client is stale (transient runtime error) in which we should re-resolve.
result = new ExceptionHandlingRetryResult(e, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
if (errorResponse.StatusCode == HttpStatusCode.InternalServerError)
{
// The address is correct, but the server processing failed.
// This could be due to conflicts when writing the word to the dictionary.
// Retry the operation without re-resolving the address.
result = new ExceptionHandlingRetryResult(e, true, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
}
if (we.Status == WebExceptionStatus.Timeout ||
we.Status == WebExceptionStatus.RequestCanceled ||
we.Status == WebExceptionStatus.ConnectionClosed ||
we.Status == WebExceptionStatus.ConnectFailure)
{
result = new ExceptionHandlingRetryResult(e, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
}
result = new ExceptionHandlingRetryResult(e, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation == null)
{
throw new ArgumentNullException(nameof(exceptionInformation));
}
var ex = exceptionInformation.Exception;
// errors where we didn't get a response from the service.
if (ex is TaskCanceledException || ex is TimeoutException)
{
_logger.RetryingServiceCall(ex.GetType().Name);
return(CreateExceptionHandlingRetryResult(false, ex, out result));
}
if (ex is ProtocolViolationException)
{
_logger.RetryingServiceCall("ProtocolViolationException", null, ex);
return(CreateExceptionHandlingRetryResult(false, ex, out result));
}
var webEx = ex as WebException ?? ex.InnerException as WebException;
if (webEx != null)
{
if (webEx.Status == WebExceptionStatus.Timeout ||
webEx.Status == WebExceptionStatus.RequestCanceled ||
webEx.Status == WebExceptionStatus.ConnectionClosed ||
webEx.Status == WebExceptionStatus.ConnectFailure)
{
_logger.RetryingServiceCall("WebExceptionStatus " + webEx.Status, null, ex);
return(CreateExceptionHandlingRetryResult(false, webEx, out result));
}
}
// we got a response from the service - let's try to get the StatusCode to see if we should retry.
if (_options.RetryHttpStatusCodeErrors)
{
HttpStatusCode? httpStatusCode = null;
HttpWebResponse webResponse = null;
HttpResponseMessage responseMessage = null;
var httpEx = ex as HttpResponseException;
if (httpEx != null)
{
responseMessage = httpEx.Response;
httpStatusCode = httpEx.Response.StatusCode;
}
else if (webEx != null)
{
webResponse = webEx.Response as HttpWebResponse;
httpStatusCode = webResponse?.StatusCode;
}
if (httpStatusCode.HasValue)
{
if (httpStatusCode == HttpStatusCode.NotFound)
{
// This could either mean we requested an endpoint that does not exist in the service API (a user error)
// or the address that was resolved by fabric client is stale (transient runtime error) in which we should re-resolve.
_logger.RetryingServiceCall("HTTP 404");
result = new ExceptionHandlingRetryResult(
exceptionId: "HTTP 404",
isTransient: false,
retryDelay: TimeSpan.FromMilliseconds(100),
maxRetryCount: 2);
return(true);
}
if ((int)httpStatusCode >= 500 && (int)httpStatusCode < 600)
{
// The address is correct, but the server processing failed.
// Retry the operation without re-resolving the address.
// we want to log the response in case it contains useful information (e.g. in dev environments)
string errorResponse = null;
if (webResponse != null)
{
using (StreamReader streamReader = new StreamReader(webResponse.GetResponseStream()))
{
errorResponse = streamReader.ReadToEnd();
}
}
else if (responseMessage != null)
{
// not sure if just calling ReadAsStringAsync().Result can result in a deadlock.
// so better safe than sorry...
// http://stackoverflow.com/questions/22628087/calling-async-method-synchronously
// AsyncEx library would be good but I don't want to take a dependency on that just for this one case.
errorResponse = Task.Run(() => responseMessage.Content.ReadAsStringAsync()).Result;
}
_logger.RetryingServiceCall($"HTTP {(int) httpStatusCode}", errorResponse);
return(CreateExceptionHandlingRetryResult(true, ex, out result));
}
}
}
_logger.ServiceCallFailed(ex);
result = null;
return(false);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation == null)
{
throw new ArgumentNullException(nameof(exceptionInformation));
}
var ex = exceptionInformation.Exception;
// errors where we didn't get a response from the service.
if (ex is TaskCanceledException || ex is TimeoutException)
{
_logger.RetryingServiceCall(ex.GetType().Name);
return CreateExceptionHandlingRetryResult(false, ex, out result);
}
if (ex is ProtocolViolationException)
{
_logger.RetryingServiceCall("ProtocolViolationException", null, ex);
return CreateExceptionHandlingRetryResult(false, ex, out result);
}
var webEx = ex as WebException ?? ex.InnerException as WebException;
if (webEx != null)
{
if (webEx.Status == WebExceptionStatus.Timeout ||
webEx.Status == WebExceptionStatus.RequestCanceled ||
webEx.Status == WebExceptionStatus.ConnectionClosed ||
webEx.Status == WebExceptionStatus.ConnectFailure)
{
_logger.RetryingServiceCall("WebExceptionStatus " + webEx.Status, null, ex);
return CreateExceptionHandlingRetryResult(false, webEx, out result);
}
}
// we got a response from the service - let's try to get the StatusCode to see if we should retry.
if (_options.RetryHttpStatusCodeErrors)
{
HttpStatusCode? httpStatusCode = null;
HttpWebResponse webResponse = null;
HttpResponseMessage responseMessage = null;
var httpEx = ex as HttpResponseException;
if (httpEx != null)
{
responseMessage = httpEx.Response;
httpStatusCode = httpEx.Response.StatusCode;
}
else if (webEx != null)
{
webResponse = webEx.Response as HttpWebResponse;
httpStatusCode = webResponse?.StatusCode;
}
if (httpStatusCode.HasValue)
{
if (httpStatusCode == HttpStatusCode.NotFound)
{
// This could either mean we requested an endpoint that does not exist in the service API (a user error)
// or the address that was resolved by fabric client is stale (transient runtime error) in which we should re-resolve.
_logger.RetryingServiceCall("HTTP 404");
result = new ExceptionHandlingRetryResult(
exceptionId: "HTTP 404",
isTransient: false,
retryDelay: TimeSpan.FromMilliseconds(100),
maxRetryCount: 2);
return true;
}
if ((int)httpStatusCode >= 500 && (int)httpStatusCode < 600)
{
// The address is correct, but the server processing failed.
// Retry the operation without re-resolving the address.
// we want to log the response in case it contains useful information (e.g. in dev environments)
string errorResponse = null;
if (webResponse != null)
{
using (StreamReader streamReader = new StreamReader(webResponse.GetResponseStream()))
{
errorResponse = streamReader.ReadToEnd();
}
}
else if (responseMessage != null)
{
// not sure if just calling ReadAsStringAsync().Result can result in a deadlock.
// so better safe than sorry...
// http://stackoverflow.com/questions/22628087/calling-async-method-synchronously
// AsyncEx library would be good but I don't want to take a dependency on that just for this one case.
errorResponse = Task.Run(() => responseMessage.Content.ReadAsStringAsync()).Result;
}
_logger.RetryingServiceCall($"HTTP {(int) httpStatusCode}", errorResponse);
return CreateExceptionHandlingRetryResult(true, ex, out result);
}
}
}
_logger.ServiceCallFailed(ex);
result = null;
return false;
}
private bool CreateExceptionHandlingRetryResult(bool isTransient, Exception ex, out ExceptionHandlingResult result)
{
result = new ExceptionHandlingRetryResult(
exceptionId: ex.GetType().Name,
isTransient: isTransient,
retryDelay: TimeSpan.FromMilliseconds(_rand.NextDouble()*_options.MaxRetryBackoffInterval.TotalMilliseconds),
maxRetryCount: _options.MaxRetryCount);
return true;
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
Exception e = exceptionInformation.Exception;
Logger.Debug("OnHandleException {0}", e);
if (e is TimeoutException)
{
result = new ExceptionHandlingRetryResult(e, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
if (e is ProtocolViolationException)
{
result = new ExceptionHandlingRetryResult(e, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
if (e is HttpRequestException)
{
HttpRequestException he = (HttpRequestException) e;
// this should not happen, but let's do a sanity check
if (null == he.InnerException)
{
result = new ExceptionHandlingRetryResult(e, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
e = he.InnerException;
}
if (e is WebException)
{
WebException we = (WebException) e;
HttpWebResponse errorResponse = we.Response as HttpWebResponse;
if (we.Status == WebExceptionStatus.ProtocolError)
{
if (errorResponse.StatusCode == HttpStatusCode.NotFound)
{
// This could either mean we requested an endpoint that does not exist in the service API (a user error)
// or the address that was resolved by fabric client is stale (transient runtime error) in which we should re-resolve.
result = new ExceptionHandlingRetryResult(e, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
if (errorResponse.StatusCode == HttpStatusCode.InternalServerError)
{
// The address is correct, but the server processing failed.
// This could be due to conflicts when writing the word to the dictionary.
// Retry the operation without re-resolving the address.
result = new ExceptionHandlingRetryResult(e, true, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
}
if (we.Status == WebExceptionStatus.Timeout ||
we.Status == WebExceptionStatus.RequestCanceled ||
we.Status == WebExceptionStatus.ConnectionClosed ||
we.Status == WebExceptionStatus.ConnectFailure)
{
result = new ExceptionHandlingRetryResult(e, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
}
result = new ExceptionHandlingRetryResult(e, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
private bool RetryResult(ExceptionInformation exceptionInformation, out ExceptionHandlingResult result)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, TransientException, this.operationRetrySettings, this.operationRetrySettings.DefaultMaxRetryCount);
return(KnownException);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
//// Log Error Here.
return(true);
}
