public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is RpcException rpcEx)
{
switch (rpcEx.Status.StatusCode)
{
case StatusCode.Unavailable when rpcEx.Status.Detail == "Endpoint read failed":
Log.LogInformation(exceptionInformation.Exception, "Throwing: {Exception}", exceptionInformation.Exception.Message);
result = new ExceptionHandlingThrowResult();
return(true);
case StatusCode.Unavailable:
case StatusCode.Unknown:
case StatusCode.Cancelled:
Log.LogInformation(exceptionInformation.Exception, "Not transient exception: {Exception}, Retry {@Retry}", exceptionInformation.Exception.Message, retrySettings);
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, int.MaxValue);
return(true);
default:
Log.LogInformation(exceptionInformation.Exception, "Unknown exception: {Exception}, Retry {@Retry}", exceptionInformation.Exception.Message, retrySettings);
result = new ExceptionHandlingThrowResult();
return(true);
}
}
else
{
Log.LogInformation(exceptionInformation.Exception, "Throwing: {Exception}", exceptionInformation.Exception.Message);
result = new ExceptionHandlingThrowResult();
return(true);
}
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
ServiceEventSource.Current.ReportServiceHealth(_serviceContext, HealthState.Error, "Exception");
result = new ExceptionHandlingThrowResult();
return(true);
}
/// <summary>
/// Method that examines the exception and determines how that exception can be handled.
/// </summary>
/// <param name="exceptionInformation">Information about the exception</param>
/// <param name="retrySettings">The operation retry preferences.</param>
/// <param name="result">Result of the exception handling</param>
/// <returns>true if the exception is handled, false otherwise</returns>
bool IExceptionHandler.TryHandleException(
ExceptionInformation exceptionInformation,
OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
var e = exceptionInformation.Exception;
if (e is ActorConcurrencyLockTimeoutException)
{
if (ActorLogicalCallContext.IsPresent())
{
result = new ExceptionHandlingThrowResult()
{
ExceptionToThrow = e
};
return(true);
}
result = new ExceptionHandlingRetryResult(
e,
true,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
result = null;
return(false);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is TimeoutException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
else if (exceptionInformation.Exception is ProtocolViolationException)
{
result = new ExceptionHandlingThrowResult();
return(true);
}
else if (exceptionInformation.Exception is WebException)
{
WebException we = exceptionInformation.Exception as WebException;
HttpWebResponse errorResponse = we.Response as HttpWebResponse;
if (we.Status == WebExceptionStatus.ProtocolError)
{
if (errorResponse.StatusCode == HttpStatusCode.NotFound)
{
// This could either mean we requested an endpoint that does not exist in the service API (a user error)
// or the address that was resolved by fabric client is stale (transient runtime error) in which we should re-resolve.
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
if (errorResponse.StatusCode == HttpStatusCode.InternalServerError)
{
// The address is correct, but the server processing failed.
// Retry the operation without re-resolving the address.
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, true, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
}
if (we.Status == WebExceptionStatus.Timeout ||
we.Status == WebExceptionStatus.RequestCanceled ||
we.Status == WebExceptionStatus.ConnectionClosed ||
we.Status == WebExceptionStatus.ConnectFailure)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
}
result = null;
return(false);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is TimeoutException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
else if (exceptionInformation.Exception is ProtocolViolationException)
{
result = new ExceptionHandlingThrowResult();
return true;
}
else if (exceptionInformation.Exception is WebException)
{
WebException we = exceptionInformation.Exception as WebException;
HttpWebResponse errorResponse = we.Response as HttpWebResponse;
if (we.Status == WebExceptionStatus.ProtocolError)
{
if (errorResponse.StatusCode == HttpStatusCode.NotFound)
{
// This could either mean we requested an endpoint that does not exist in the service API (a user error)
// or the address that was resolved by fabric client is stale (transient runtime error) in which we should re-resolve.
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
if (errorResponse.StatusCode == HttpStatusCode.InternalServerError)
{
// The address is correct, but the server processing failed.
// Retry the operation without re-resolving the address.
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, true, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
}
if (we.Status == WebExceptionStatus.Timeout ||
we.Status == WebExceptionStatus.RequestCanceled ||
we.Status == WebExceptionStatus.ConnectionClosed ||
we.Status == WebExceptionStatus.ConnectFailure)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return true;
}
}
result = null;
return false;
}
public bool TryHandleException(
ExceptionInformation exceptionInformation,
OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is InvalidOperationException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception,
isTransient: true,
retryDelay: TimeSpan.FromSeconds(2),
maxRetryCount: 3);
return(true);
}
result = new ExceptionHandlingThrowResult();
return(false);
}
public bool TryHandleException(
ExceptionInformation exceptionInformation,
OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is TException)
{
result = new ExceptionHandlingRetryResult(
exception: exceptionInformation.Exception,
isTransient: _isTransient,
retryDelay: _retryDelay,
maxRetryCount: _maxRetryCount);
return(true);
}
result = new ExceptionHandlingThrowResult();
return(false);
}
/// <summary>
/// Method that examines the exception and determines how that exception can be handled.
/// </summary>
/// <param name="exceptionInformation">Information about the exception</param>
/// <param name="retrySettings">The operation retry preferences.</param>
/// <param name="result">Result of the exception handling</param>
/// <returns>true if the exception is handled, false otherwise</returns>
bool IExceptionHandler.TryHandleException(
ExceptionInformation exceptionInformation,
OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
var e = exceptionInformation.Exception;
if (e is ActorConcurrencyLockTimeoutException)
{
if (ActorLogicalCallContext.IsPresent())
{
result = new ExceptionHandlingThrowResult()
{
ExceptionToThrow = e
};
return(true);
}
result = new ExceptionHandlingRetryResult(
e,
true,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
// The messaging layer may deliver duplicate messages during the connection failures.
//E.g when client connection is disconnected but service is still processing the message. We retry on client connection failures.
//This results to service receiving duplicate message.
//And Actor Reentrancy throws DuplicateMessageException exception when it sees a duplicate Message (message with same callContext).
if (e is DuplicateMessageException)
{
result = new ExceptionHandlingRetryResult(
e,
true,
retrySettings,
int.MaxValue);
return(true);
}
result = null;
return(false);
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
if (exceptionInformation.Exception is TimeoutException)
{
result =
new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
false,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
if (exceptionInformation.Exception is SocketException)
{
result =
new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
false,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
if (exceptionInformation.Exception is ProtocolViolationException)
{
result = new ExceptionHandlingThrowResult();
return(true);
}
var we = exceptionInformation.Exception.InnerException as WebException ??
exceptionInformation.Exception.InnerException as WebException;
if (we != null)
{
var errorResponse = we.Response as HttpWebResponse;
if (we.Status == WebExceptionStatus.ProtocolError)
{
if (errorResponse != null && errorResponse.StatusCode == HttpStatusCode.NotFound)
{
result =
new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
false,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
if (errorResponse != null && errorResponse.StatusCode == HttpStatusCode.InternalServerError)
{
result =
new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
true,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
}
if (we.Status == WebExceptionStatus.Timeout ||
we.Status == WebExceptionStatus.RequestCanceled ||
we.Status == WebExceptionStatus.ConnectionClosed ||
we.Status == WebExceptionStatus.ConnectFailure)
{
result =
new ExceptionHandlingRetryResult(
exceptionInformation.Exception,
false,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
}
result = null;
return(false);
}
/// <summary>
/// Method that examines the exception and determines how that exception can be handled.
/// </summary>
/// <param name="exceptionInformation">Information about the exception</param>
/// <param name="retrySettings">The operation retry preferences.</param>
/// <param name="result">Result of the exception handling</param>
/// <returns>true if the exception is handled, false otherwise</returns>
bool IExceptionHandler.TryHandleException(
ExceptionInformation exceptionInformation,
OperationRetrySettings retrySettings,
out ExceptionHandlingResult result)
{
var e = exceptionInformation.Exception;
// retry with resolve - these exceptions indicate a possible fail over
if ((e is EndpointNotFoundException) ||
(e is CommunicationObjectAbortedException) ||
(e is CommunicationObjectFaultedException) ||
(e is ObjectDisposedException) ||
(e is ChannelTerminatedException))
{
result = new ExceptionHandlingRetryResult(
e,
false,
retrySettings,
int.MaxValue);
return(true);
}
// retry on timeout and service busy exceptions
if ((e is TimeoutException) ||
(e is ServerTooBusyException))
{
result = new ExceptionHandlingRetryResult(
e,
true,
retrySettings,
int.MaxValue);
return(true);
}
// Derived types of Communication Exception that are not retriable.
if ((e is ActionNotSupportedException) ||
(e is AddressAccessDeniedException))
{
result = new ExceptionHandlingThrowResult()
{
ExceptionToThrow = e
};
return(true);
}
// Security related derived types of Communication Exception that are not retriable
if (e is SecurityAccessDeniedException)
{
result = new ExceptionHandlingThrowResult()
{
ExceptionToThrow = e
};
return(true);
}
var faultException = e as FaultException;
if (faultException != null)
{
if (faultException.Code.Name == WcfRemoteExceptionInformation.FaultCodeName)
{
var actualException = WcfRemoteExceptionInformation.ToException(faultException.Reason.ToString());
if (faultException.Code.SubCode.Name == WcfRemoteExceptionInformation.FaultSubCodeRetryName)
{
result = new ExceptionHandlingRetryResult(
actualException,
false,
retrySettings,
int.MaxValue);
return(true);
}
}
}
// retry on all communication exceptions, including the protocol exceptions for default max retry
if ((faultException == null) && (e is CommunicationException))
{
result = new ExceptionHandlingRetryResult(
e,
false,
retrySettings,
retrySettings.DefaultMaxRetryCount);
return(true);
}
result = null;
return(false);
}
protected bool HandleRemoteException(Exception e, out ExceptionHandlingResult result)
{
if (e is FabricTransientException)
{
result = new ExceptionHandlingRetryResult
{
IsTransient = true,
RetryDelay = TimeSpan.FromMilliseconds(ThreadSafeRandom.Value.NextDouble() * WcfFactory.MaxRetryBackoffIntervalOnTransientErrors.TotalMilliseconds)
};
return true;
}
if (e is FabricNotPrimaryException)
{
result = new ExceptionHandlingRetryResult
{
IsTransient = false,
RetryDelay = TimeSpan.FromMilliseconds(ThreadSafeRandom.Value.NextDouble() * WcfFactory.MaxRetryBackoffIntervalOnNonTransientErrors.TotalMilliseconds)
};
return true;
}
result = new ExceptionHandlingThrowResult
{
ExceptionToThrow = new AggregateException(e)
};
return true;
}
public bool TryHandleException(ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, out ExceptionHandlingResult result)
{
_context.WriteEvent($"HttpExceptionHandler::TryHandleException: Encountered Exception {exceptionInformation.Exception}");
if (exceptionInformation.Exception is TimeoutException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
else if (exceptionInformation.Exception is ProtocolViolationException)
{
result = new ExceptionHandlingThrowResult();
return(true);
}
else if (exceptionInformation.Exception is SocketException)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
HttpRequestException httpException = exceptionInformation.Exception as HttpRequestException;
if (httpException != null)
{
_context.WriteEvent($"HttpExceptionHandler::TryHandleException: HttpRequestException {httpException}");
result = null;
return(false);
}
WebException we = exceptionInformation.Exception as WebException;
if (we == null)
{
we = exceptionInformation.Exception.InnerException as WebException;
}
if (we != null)
{
_context.WriteEvent($"HttpExceptionHandler::TryHandleException: WebException {we}");
HttpWebResponse errorResponse = we.Response as HttpWebResponse;
if (we.Status == WebExceptionStatus.ProtocolError)
{
_context.WriteEvent($"HttpExceptionHandler::TryHandleException: HttpWebResponse {errorResponse}");
_context.WriteEvent($"HttpExceptionHandler::TryHandleException: HttpWebResponse Status Code {errorResponse.StatusCode} Decription = {errorResponse.StatusDescription}");
result = null;
return(false);
}
if (we.Status == WebExceptionStatus.Timeout ||
we.Status == WebExceptionStatus.RequestCanceled ||
we.Status == WebExceptionStatus.ConnectionClosed ||
we.Status == WebExceptionStatus.ConnectFailure)
{
result = new ExceptionHandlingRetryResult(exceptionInformation.Exception, false, retrySettings, retrySettings.DefaultMaxRetryCount);
return(true);
}
}
result = null;
return(false);
}
