/// <summary>
/// Handles the exceptions that occur in the CommunicationClient when sending a message to the Service
/// </summary>
/// <param name="client">Communication client</param>
/// <param name="exceptionInformation">Information about the exception that occurred when communicating with the service.</param>
/// <param name="retrySettings">Specifies the retry policy that should be used for handling the reported exception.</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>
/// A <see cref="System.Threading.Tasks.Task">Task</see> that represents outstanding operation. The result of the Task is
/// a <see cref="OperationRetryControl" /> object that determines
/// how the retry policy for this exception.
/// </returns>
public async Task <OperationRetryControl> ReportOperationExceptionAsync(
TCommunicationClient client,
ExceptionInformation exceptionInformation,
OperationRetrySettings retrySettings,
CancellationToken cancellationToken)
{
var partitionId = client.ResolvedServicePartition.Info.Id;
var entry = this.cache.GetOrAddClientCacheEntry(
partitionId,
client.Endpoint,
client.ListenerName,
client.ResolvedServicePartition);
var faultedClient = default(TCommunicationClient);
OperationRetryControl retval;
await entry.Semaphore.WaitAsync(cancellationToken);
try
{
ExceptionHandlingResult exceptionHandlingResult;
var handled = this.HandleReportedException(
exceptionInformation,
retrySettings,
out exceptionHandlingResult);
if (handled && (exceptionHandlingResult is ExceptionHandlingRetryResult))
{
var retryResult = (ExceptionHandlingRetryResult)exceptionHandlingResult;
if (!retryResult.IsTransient && (ReferenceEquals(client, entry.Client)))
{
// The endpoint isn't valid if it is a re-triable error and not transient.
this.AbortClient(entry.Client);
faultedClient = entry.Client;
entry.Client = default(TCommunicationClient);
entry.Rsp = null;
}
retval = new OperationRetryControl()
{
ShouldRetry = true,
IsTransient = retryResult.IsTransient,
RetryDelay = retryResult.RetryDelay,
Exception = null,
ExceptionId = retryResult.ExceptionId,
MaxRetryCount = retryResult.MaxRetryCount
};
}
else
{
retval = new OperationRetryControl()
{
ShouldRetry = false,
RetryDelay = Timeout.InfiniteTimeSpan,
Exception = exceptionInformation.Exception
};
var throwResult = exceptionHandlingResult as ExceptionHandlingThrowResult;
if ((throwResult != null) && (throwResult.ExceptionToThrow != null))
{
retval.Exception = throwResult.ExceptionToThrow;
}
}
}
finally
{
entry.Semaphore.Release();
}
if (faultedClient != null)
{
this.OnClientDisconnected(faultedClient);
}
return(retval);
}
private async Task <TCommunicationClient> CreateClientWithRetriesAsync(
ResolvedServicePartition previousRsp,
TargetReplicaSelector targetReplicaSelector,
string listenerName,
OperationRetrySettings retrySettings,
bool doInitialResolve,
CancellationToken cancellationToken)
{
var doResolve = doInitialResolve;
var currentRetryCount = 0;
string currentExceptionId = null;
while (true)
{
ExceptionHandlingResult result;
Exception actualException;
bool newClient = false;
try
{
if (doResolve)
{
var rsp = await this.ServiceResolver.ResolveAsync(
previousRsp,
ServicePartitionResolver.DefaultResolveTimeout,
retrySettings.MaxRetryBackoffIntervalOnTransientErrors,
cancellationToken);
previousRsp = rsp;
}
var endpoint = this.GetEndpoint(previousRsp, targetReplicaSelector);
var cacheEntry =
await
this.GetAndLockClientCacheEntryAsync(previousRsp.Info.Id, endpoint, listenerName, previousRsp,
cancellationToken);
TCommunicationClient client;
try
{
// The communication client in the cache is invalid.
// This could happen for these 2 cases,
// 1. The endpoint and RSP information is valid, but there are no active users for the
// communication client so the last reference to the client was GC'd.
//2.There was an exception during communication to the endpoint, and the ReportOperationException
//code path and the communication client was invalidated.
if (cacheEntry.Client == null)
{
ServiceTrace.Source.WriteInfo(
TraceType,
"{0} Creating Client for connecting to ListenerName : {1} Address : {2} Role : {3}",
this.traceId,
listenerName,
cacheEntry.GetEndpoint(),
cacheEntry.Endpoint.Role);
cacheEntry.Rsp = previousRsp;
client = await this.CreateClientAsync(cacheEntry.GetEndpoint(), cancellationToken);
cacheEntry.Client = client;
client.ResolvedServicePartition = cacheEntry.Rsp;
client.ListenerName = cacheEntry.ListenerName;
client.Endpoint = cacheEntry.Endpoint;
newClient = true;
}
else
{
var clientValid = this.ValidateLockedClientCacheEntry(
cacheEntry,
previousRsp,
out client);
if (!clientValid)
{
ServiceTrace.Source.WriteInfo(
TraceType,
"{0} Invalid Client found in Cache for ListenerName : {1} Address : {2} Role : {3}",
this.traceId,
listenerName,
cacheEntry.GetEndpoint(),
cacheEntry.Endpoint.Role);
doResolve = true;
continue;
}
else
{
ServiceTrace.Source.WriteInfo(
TraceType,
"{0} Found valid client for ListenerName : {1} Address : {2} Role : {3}",
this.traceId,
listenerName,
endpoint.Address,
endpoint.Role);
}
}
}
finally
{
cacheEntry.Semaphore.Release();
}
if (client != null && newClient)
{
this.OnClientConnected(client);
}
return(client);
}
catch (Exception e)
{
ServiceTrace.Source.WriteInfo(
TraceType,
"{0} Exception While CreatingClient {1}",
this.traceId,
e);
if (!this.HandleReportedException(
new ExceptionInformation(e, targetReplicaSelector),
retrySettings,
out result))
{
throw;
}
var throwResult = result as ExceptionHandlingThrowResult;
if (throwResult != null)
{
if (ReferenceEquals(e, throwResult.ExceptionToThrow))
{
throw;
}
throw throwResult.ExceptionToThrow;
}
// capture the exception so that we can throw based on the retry policy
actualException = e;
}
var retryResult = (ExceptionHandlingRetryResult)result;
if (!Utility.ShouldRetryOperation(
retryResult.ExceptionId,
retryResult.MaxRetryCount,
ref currentExceptionId,
ref currentRetryCount))
{
ServiceTrace.Source.WriteInfo(
TraceType,
"{0} Retry count for exception id {1} exceeded the retry limit : {2}, throwing exception - {3}",
this.traceId,
retryResult.ExceptionId,
retryResult.MaxRetryCount,
actualException);
throw new AggregateException(actualException);
}
doResolve = !retryResult.IsTransient;
await Task.Delay(retryResult.RetryDelay, cancellationToken);
}
}
/// <summary>
/// Gets or Creates the CommunicationClient for the specified listener name by resolving based on the given previousRsp.
/// </summary>
/// <param name="previousRsp">Previous ResolvedServicePartition value</param>
/// <param name="targetReplica">Specifies which replica in the partition identified by the partition key, the client should connect to</param>
/// <param name="listenerName">Specifies which listener in the endpoint of the chosen replica, to which the client should connect to</param>
/// <param name="retrySettings">Specifies the retry policy that should be used for exceptions that occur when creating the client.</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>
/// A <see cref="System.Threading.Tasks.Task">Task</see> that represents outstanding operation. The result of the Task is
/// the CommunicationClient(<see cref="ICommunicationClient" />) object.
/// </returns>
public async Task <TCommunicationClient> GetClientAsync(
ResolvedServicePartition previousRsp,
TargetReplicaSelector targetReplica,
string listenerName,
OperationRetrySettings retrySettings,
CancellationToken cancellationToken)
{
bool doResolve = false;
var endpoint = this.GetEndpoint(previousRsp, targetReplica);
CommunicationClientCacheEntry <TCommunicationClient> cacheEntry;
if (this.cache.TryGetClientCacheEntry(
previousRsp.Info.Id,
endpoint,
listenerName,
out cacheEntry))
{
await cacheEntry.Semaphore.WaitAsync(cancellationToken);
try
{
TCommunicationClient validClient;
var clientValid = this.ValidateLockedClientCacheEntry(
cacheEntry,
previousRsp,
out validClient);
if (clientValid)
{
return(validClient);
}
else
{
ServiceTrace.Source.WriteInfo(
TraceType,
"{0} Client not valid in Cached entry for ListenerName : {1} Address : {2} Role : {3}",
this.traceId,
listenerName,
endpoint.Address,
endpoint.Role);
}
}
finally
{
cacheEntry.Semaphore.Release();
}
//
// There was a cache hit, but the communication client in the cache is invalid.
// This could happen for these 2 cases,
// 1. The endpoint and RSP information is valid, but there are no active users for the
// communication client so the last reference to the client was GC'd.
// 2. There was an exception during communication to the endpoint, and the ReportOperationException
// code path and the communication client was invalidated.
//
doResolve = true;
}
//
// We did not find a cache entry or a valid client in our cache, so attempt to create a new client.
//
var newClient = await this.CreateClientWithRetriesAsync(
previousRsp,
targetReplica,
listenerName,
retrySettings,
doResolve,
cancellationToken);
if (newClient != null)
{
this.OnClientConnected(newClient);
}
return(newClient);
}