/// <summary> /// Get a service client from pool and make sure it is workable. /// </summary> /// <remarks> /// Notice: Call ReleaseProxyClient when you're done with ServiceClient. /// </remarks> /// <returns>service client</returns> public AzureServiceClient GetProxyClient() { while (true) { AzureServiceClient client = this.InternalGetProxyClient(); if (client.ServiceClient.State == CommunicationState.Closed || client.ServiceClient.State == CommunicationState.Closing || client.ServiceClient.State == CommunicationState.Faulted) { BrokerTracing.TraceVerbose( "[ProxyClientPool].GetProxyClient: Client is not ready for use, remove it, {0}, {1}", client, client.ServiceClient.Endpoint.Address); this.RemoveProxyClient(client); if (client.ServiceClient.State == CommunicationState.Faulted) { client.AsyncClose(); } } else { BrokerTracing.TraceVerbose( "[ProxyClientPool].GetProxyClient: Get a client ready for use, {0}, {1}", client, client.ServiceClient.Endpoint.Address); return(client); } } }
/// <summary> /// Returns one ServieClient in the pool. /// </summary> private AzureServiceClient InternalGetProxyClient() { AzureServiceClient client = null; if (!this.IsFull) { client = new AzureServiceClient(ProxyBinding.BrokerProxyBinding, this.proxyEpr); BrokerTracing.TraceInfo( "[ProxyClientPool].InternalGetProxyClient: Create a new client, {0}, {1}", client, client.ServiceClient.Endpoint.Address); this.clientPool.Add(client); BrokerTracing.TraceInfo( "[ProxyClientPool].InternalGetProxyClient: Pool size is {0}", this.clientPool.Count); // add "1" to the RefCounts list since we will return this new client this.clientRefCounts.Add(1); return(client); } else { int poolSize = this.clientPool.Count; if (poolSize > 0) { if (this.currentIndex <= 0 || this.currentIndex >= poolSize) { this.currentIndex = 0; } client = this.clientPool[this.currentIndex]; this.clientRefCounts[this.currentIndex]++; this.currentIndex++; } BrokerTracing.TraceInfo( "[ProxyClientPool].InternalGetProxyClient: Get a client from pool, {0}, {1}", client, client.ServiceClient.Endpoint.Address); return(client); } }
/// <summary> /// Remove the specified client from pool. /// </summary> /// <param name="client">targeted client</param> /// <returns>removed from pool or not</returns> public bool RemoveProxyClient(AzureServiceClient client) { for (int i = 0; i < this.clientPool.Count; i++) { if (ReferenceEquals(this.clientPool[i], client)) { this.clientPool.RemoveAt(i); this.clientRefCounts.RemoveAt(i); BrokerTracing.TraceInfo("ProxyClientPool. RemoveProxyClient: Close client {0}.", client.ServiceClient.Endpoint.Address); return(true); } } return(false); }
/// <summary> /// Handle client failure /// </summary> /// <param name="client">targeted client</param> private static void HandleClientFailure(AzureServiceClient client) { try { BrokerTracing.TraceWarning( "[AzureDispatcher] HandleClientFailure: Handle invalid client, {0}, {1}", client, client.ServiceClient.Endpoint.Address); Dictionary <EndpointAddress, ProxyClientPool> .ValueCollection pools; lock (LockProxyClientPoolDic) { pools = ProxyClientPoolDic.Values; } foreach (var pool in pools) { bool existInPool = false; lock (pool) { BrokerTracing.TraceVerbose( "[AzureDispatcher] HandleClientFailure: Remove client {0} from pool.", client); existInPool = pool.RemoveProxyClient(client); } if (existInPool) { BrokerTracing.TraceVerbose( "[AzureDispatcher] HandleClientFailure: Close client {0}", client); // Close the proxy client if any exception is encountered. // As a result, async pending callback on clients will be // invoked (with exception). client.AsyncClose(); return; } } } catch (Exception e) { BrokerTracing.TraceError( "[AzureDispatcher] HandleClientFailure: Error occurs, {0}", e); } }
/// <summary> /// If parameter serviceClient is in the pool, clear the pool and return the list items. /// The invoker of this method will async close returned clients outside a lock scope. /// </summary> /// <returns>the list items</returns> public AzureServiceClient[] Clear(AzureServiceClient serviceClient) { if (serviceClient != null && this.clientPool.Contains(serviceClient)) { AzureServiceClient[] clients = this.clientPool.ToArray(); this.clientPool.Clear(); this.clientRefCounts.Clear(); BrokerTracing.TraceWarning("[ProxyClientPool]. Clear: Client {0} is in the pool, so clear the pool {1}.", serviceClient, this.proxyEpr.ToString()); return(clients); } else { BrokerTracing.TraceWarning("[ProxyClientPool]. Clear: Client {0} is not in the pool {1}.", serviceClient, this.proxyEpr.ToString()); return(null); } }
/// <summary> /// Remove the specified client from pool and update the reference count. /// </summary> /// <param name="client">targeted client</param> /// <returns>removed from pool or not</returns> public bool ReleaseProxyClient(AzureServiceClient client) { for (int i = 0; i < this.clientPool.Count; i++) { if (ReferenceEquals(this.clientPool[i], client)) { this.clientRefCounts[i]--; if (this.clientRefCounts[i] == 0) { this.clientPool.RemoveAt(i); this.clientRefCounts.RemoveAt(i); BrokerTracing.TraceInfo("[ProxyClientPool] .ReleaseProxyClient: close client {0} as its reference count reaches 0", client.ServiceClient.Endpoint.Address); return(true); } } } return(false); }
/// <summary> /// Clear the client pool and close all the connections in it. /// </summary> /// <remarks> /// This method has high cost. We need to be pretty sure that the exception /// happens on the connection between broker and proxy, then call this method. /// </remarks> /// <returns> /// is the specified client in the pool /// </returns> public override bool CleanupClient(IService client) { AzureServiceClient serviceClient = client as AzureServiceClient; Debug.Assert(serviceClient != null); BrokerTracing.TraceWarning( BrokerTracing.GenerateTraceString( "AzureDispatcher", "CleanupClient", this.TaskId, -1, serviceClient.ToString(), string.Empty, "Cleanup client.")); AzureServiceClient[] clients = null; lock (this.proxyClientPool) { clients = this.proxyClientPool.Clear(serviceClient); } if (clients != null) { BrokerTracing.TraceWarning( BrokerTracing.GenerateTraceString( "AzureDispatcher", "CleanupClient", this.TaskId, -1, serviceClient.ToString(), string.Empty, string.Format("Close clients in the pool. count = {0}", clients.Length))); // don't close client in above lock (this.proxyClientPool) scope to avoid deadlock foreach (AzureServiceClient asc in clients) { asc.AsyncClose(); } } return(clients != null); }
/// <summary> /// Callback function to check ping results for each service client /// </summary> /// <param name="ar">async result object</param> private static void EndPingProxyClient(IAsyncResult ar) { AzureServiceClient client = ar.AsyncState as AzureServiceClient; try { client.EndProcessMessage(ar); BrokerTracing.TraceVerbose("[AzureDispatcher] EndPingProxyClients: client {0} is ok", client.ServiceClient.Endpoint.Address); } catch (TimeoutException) { // TimeoutException may happen when the channel is sending a big request message. // It doesn't mean the channel is broken. BrokerTracing.TraceVerbose("[AzureDispatcher] EndPingProxyClients: client {0} timeout", client.ServiceClient.Endpoint.Address); } catch (Exception e) { BrokerTracing.TraceError("[AzureDispatcher] EndPingProxyClients: client {0} receives exception: {1}", client.ServiceClient.Endpoint.Address, e); HandleClientFailure(client); } }
/// <summary> /// Check connection status between the specified client and broker proxy by sending a ping message to broker proxy, /// and close the client if the connnection is broken. /// </summary> /// <param name="client">the service client instance to be </param> /// <returns>async result object</returns> private static IAsyncResult BeginPingProxyClient(AzureServiceClient client) { Debug.Assert(client != null); try { // no need to ping proxy if it is not ready if (!client.IsConnectionReady()) { return(null); } } catch (Exception ex) { BrokerTracing.TraceWarning("[AzureDispatcher].BeginPingProxyClient: Exception {0}", ex); // the client may be disposed, so its WaitHandleForOpen is already closed adn set null return(null); } Message pingMessage = Message.CreateMessage(MessageVersion.Default, Constant.PingBrokerProxyAction); BrokerTracing.TraceVerbose("[AzureDispatcher] BeginPingProxyClient: ping client {0}", client.ServiceClient.Endpoint.Address); try { return(client.BeginProcessMessage( pingMessage, new ThreadHelper <IAsyncResult>(new AsyncCallback(EndPingProxyClient)).CallbackRoot, client)); } catch (Exception e) { BrokerTracing.TraceError("[AzureDispatcher] BeginPingProxyClients: sending ping message via client {0} encounters excpetion: {1}", client.ServiceClient.Endpoint.Address, e); HandleClientFailure(client); return(null); } }
/// <summary> /// Opent the client and trigger GetNextRequest method. /// </summary> private static void AsyncStartWorker(object state) { BrokerTracing.TraceVerbose("[AzureServiceClient]. AsyncStartWorker is called."); object[] objs = state as object[]; Debug.Assert(objs != null && objs.Length == 2); AzureServiceClient client = objs[0] as AzureServiceClient; int serviceOperationTimeout = (int)objs[1]; bool validConnection = false; string clientInfo = string.Empty; try { clientInfo = client.ToString(); // open the channel explicitly when it is shared by multi threads client.ServiceClient.Open(); BrokerTracing.TraceVerbose("[AzureServiceClient]. AsyncStartWorker: Open the client succeeded, {0}", clientInfo); // setting client.InnerChannel.OperationTimeout causes the channel factory to open, and then causes client.Open to fail, // becasue we can't call Open when channel factory is already opened. // it is fine to set OperationTimeout after client.Open if (serviceOperationTimeout > 0) { BrokerTracing.TraceVerbose("[AzureServiceClient]. AsyncStartWorker: Set OperationTimeout {0} to the client {1}", serviceOperationTimeout, clientInfo); client.ServiceClient.InnerChannel.OperationTimeout = TimeSpan.FromMilliseconds(serviceOperationTimeout); } validConnection = true; } catch (Exception e) { BrokerTracing.TraceError("[AzureServiceClient]. AsyncStartWorker: Open the client failed, {0}, {1}", clientInfo, e); } try { if (validConnection) { try { if (client.waitHandleForOpen != null) { BrokerTracing.TraceVerbose("[AzureServiceClient]. AsyncStartWorker: Release the wait handle, {0}", clientInfo); client.waitHandleForOpen.Set(); } } catch (Exception ex) { BrokerTracing.TraceWarning("[AzureServiceClient].AsyncStartWorker: Exception {0}", ex); // Exception may happen if the client is already closed. Ignore this // exception because the client released that WaitHandle when closed. } BrokerTracing.TraceVerbose("[AzureServiceClient]. AsyncStartWorker: Call TriggerGetNextRequest, {0}", clientInfo); // the client.WaitHandleForOpen.Set() is already called above before this. client.TriggerGetNextRequest(); } else { client.RecreateClient(); } } catch (Exception e) { BrokerTracing.TraceError("[AzureServiceClient]. AsyncStartWorker: Execption happens, {0}, {1}", clientInfo, e); } }