/// <summary> /// Handles the exceptions that occur in the CommunicationClient when sending a message to the Service /// </summary> /// <param name="client">Communication client</param> /// <param name="exceptionInformation">Information about the exception that occurred when communicating with the service.</param> /// <param name="retrySettings">Specifies the retry policy that should be used for handling the reported exception.</param> /// <param name="cancellationToken">Cancellation token</param> /// <returns> /// A <see cref="System.Threading.Tasks.Task">Task</see> that represents outstanding operation. The result of the Task is /// a <see cref="OperationRetryControl" /> object that determines /// how the retry policy for this exception. /// </returns> public async Task <OperationRetryControl> ReportOperationExceptionAsync( TCommunicationClient client, ExceptionInformation exceptionInformation, OperationRetrySettings retrySettings, CancellationToken cancellationToken) { var partitionId = client.ResolvedServicePartition.Info.Id; var entry = this.cache.GetOrAddClientCacheEntry( partitionId, client.Endpoint, client.ListenerName, client.ResolvedServicePartition); var faultedClient = default(TCommunicationClient); OperationRetryControl retval; await entry.Semaphore.WaitAsync(cancellationToken); try { ExceptionHandlingResult exceptionHandlingResult; var handled = this.HandleReportedException( exceptionInformation, retrySettings, out exceptionHandlingResult); if (handled && (exceptionHandlingResult is ExceptionHandlingRetryResult)) { var retryResult = (ExceptionHandlingRetryResult)exceptionHandlingResult; if (!retryResult.IsTransient && (ReferenceEquals(client, entry.Client))) { // The endpoint isn't valid if it is a re-triable error and not transient. this.AbortClient(entry.Client); faultedClient = entry.Client; entry.Client = default(TCommunicationClient); entry.Rsp = null; } retval = new OperationRetryControl() { ShouldRetry = true, IsTransient = retryResult.IsTransient, RetryDelay = retryResult.RetryDelay, Exception = null, ExceptionId = retryResult.ExceptionId, MaxRetryCount = retryResult.MaxRetryCount }; } else { retval = new OperationRetryControl() { ShouldRetry = false, RetryDelay = Timeout.InfiniteTimeSpan, Exception = exceptionInformation.Exception }; var throwResult = exceptionHandlingResult as ExceptionHandlingThrowResult; if ((throwResult != null) && (throwResult.ExceptionToThrow != null)) { retval.Exception = throwResult.ExceptionToThrow; } } } finally { entry.Semaphore.Release(); } if (faultedClient != null) { this.OnClientDisconnected(faultedClient); } return(retval); }
private async Task <TCommunicationClient> CreateClientWithRetriesAsync( ResolvedServicePartition previousRsp, TargetReplicaSelector targetReplicaSelector, string listenerName, OperationRetrySettings retrySettings, bool doInitialResolve, CancellationToken cancellationToken) { var doResolve = doInitialResolve; var currentRetryCount = 0; string currentExceptionId = null; while (true) { ExceptionHandlingResult result; Exception actualException; bool newClient = false; try { if (doResolve) { var rsp = await this.ServiceResolver.ResolveAsync( previousRsp, ServicePartitionResolver.DefaultResolveTimeout, retrySettings.MaxRetryBackoffIntervalOnTransientErrors, cancellationToken); previousRsp = rsp; } var endpoint = this.GetEndpoint(previousRsp, targetReplicaSelector); var cacheEntry = await this.GetAndLockClientCacheEntryAsync(previousRsp.Info.Id, endpoint, listenerName, previousRsp, cancellationToken); TCommunicationClient client; try { // The communication client in the cache is invalid. // This could happen for these 2 cases, // 1. The endpoint and RSP information is valid, but there are no active users for the // communication client so the last reference to the client was GC'd. //2.There was an exception during communication to the endpoint, and the ReportOperationException //code path and the communication client was invalidated. if (cacheEntry.Client == null) { ServiceTrace.Source.WriteInfo( TraceType, "{0} Creating Client for connecting to ListenerName : {1} Address : {2} Role : {3}", this.traceId, listenerName, cacheEntry.GetEndpoint(), cacheEntry.Endpoint.Role); cacheEntry.Rsp = previousRsp; client = await this.CreateClientAsync(cacheEntry.GetEndpoint(), cancellationToken); cacheEntry.Client = client; client.ResolvedServicePartition = cacheEntry.Rsp; client.ListenerName = cacheEntry.ListenerName; client.Endpoint = cacheEntry.Endpoint; newClient = true; } else { var clientValid = this.ValidateLockedClientCacheEntry( cacheEntry, previousRsp, out client); if (!clientValid) { ServiceTrace.Source.WriteInfo( TraceType, "{0} Invalid Client found in Cache for ListenerName : {1} Address : {2} Role : {3}", this.traceId, listenerName, cacheEntry.GetEndpoint(), cacheEntry.Endpoint.Role); doResolve = true; continue; } else { ServiceTrace.Source.WriteInfo( TraceType, "{0} Found valid client for ListenerName : {1} Address : {2} Role : {3}", this.traceId, listenerName, endpoint.Address, endpoint.Role); } } } finally { cacheEntry.Semaphore.Release(); } if (client != null && newClient) { this.OnClientConnected(client); } return(client); } catch (Exception e) { ServiceTrace.Source.WriteInfo( TraceType, "{0} Exception While CreatingClient {1}", this.traceId, e); if (!this.HandleReportedException( new ExceptionInformation(e, targetReplicaSelector), retrySettings, out result)) { throw; } var throwResult = result as ExceptionHandlingThrowResult; if (throwResult != null) { if (ReferenceEquals(e, throwResult.ExceptionToThrow)) { throw; } throw throwResult.ExceptionToThrow; } // capture the exception so that we can throw based on the retry policy actualException = e; } var retryResult = (ExceptionHandlingRetryResult)result; if (!Utility.ShouldRetryOperation( retryResult.ExceptionId, retryResult.MaxRetryCount, ref currentExceptionId, ref currentRetryCount)) { ServiceTrace.Source.WriteInfo( TraceType, "{0} Retry count for exception id {1} exceeded the retry limit : {2}, throwing exception - {3}", this.traceId, retryResult.ExceptionId, retryResult.MaxRetryCount, actualException); throw new AggregateException(actualException); } doResolve = !retryResult.IsTransient; await Task.Delay(retryResult.RetryDelay, cancellationToken); } }
/// <summary> /// Gets or Creates the CommunicationClient for the specified listener name by resolving based on the given previousRsp. /// </summary> /// <param name="previousRsp">Previous ResolvedServicePartition value</param> /// <param name="targetReplica">Specifies which replica in the partition identified by the partition key, the client should connect to</param> /// <param name="listenerName">Specifies which listener in the endpoint of the chosen replica, to which the client should connect to</param> /// <param name="retrySettings">Specifies the retry policy that should be used for exceptions that occur when creating the client.</param> /// <param name="cancellationToken">Cancellation token</param> /// <returns> /// A <see cref="System.Threading.Tasks.Task">Task</see> that represents outstanding operation. The result of the Task is /// the CommunicationClient(<see cref="ICommunicationClient" />) object. /// </returns> public async Task <TCommunicationClient> GetClientAsync( ResolvedServicePartition previousRsp, TargetReplicaSelector targetReplica, string listenerName, OperationRetrySettings retrySettings, CancellationToken cancellationToken) { bool doResolve = false; var endpoint = this.GetEndpoint(previousRsp, targetReplica); CommunicationClientCacheEntry <TCommunicationClient> cacheEntry; if (this.cache.TryGetClientCacheEntry( previousRsp.Info.Id, endpoint, listenerName, out cacheEntry)) { await cacheEntry.Semaphore.WaitAsync(cancellationToken); try { TCommunicationClient validClient; var clientValid = this.ValidateLockedClientCacheEntry( cacheEntry, previousRsp, out validClient); if (clientValid) { return(validClient); } else { ServiceTrace.Source.WriteInfo( TraceType, "{0} Client not valid in Cached entry for ListenerName : {1} Address : {2} Role : {3}", this.traceId, listenerName, endpoint.Address, endpoint.Role); } } finally { cacheEntry.Semaphore.Release(); } // // There was a cache hit, but the communication client in the cache is invalid. // This could happen for these 2 cases, // 1. The endpoint and RSP information is valid, but there are no active users for the // communication client so the last reference to the client was GC'd. // 2. There was an exception during communication to the endpoint, and the ReportOperationException // code path and the communication client was invalidated. // doResolve = true; } // // We did not find a cache entry or a valid client in our cache, so attempt to create a new client. // var newClient = await this.CreateClientWithRetriesAsync( previousRsp, targetReplica, listenerName, retrySettings, doResolve, cancellationToken); if (newClient != null) { this.OnClientConnected(newClient); } return(newClient); }