/// <summary>
/// Get a service client from pool and make sure it is workable.
/// </summary>
/// <remarks>
/// Notice: Call ReleaseProxyClient when you're done with ServiceClient.
/// </remarks>
/// <returns>service client</returns>
public AzureServiceClient GetProxyClient()
{
while (true)
{
AzureServiceClient client = this.InternalGetProxyClient();
if (client.ServiceClient.State == CommunicationState.Closed ||
client.ServiceClient.State == CommunicationState.Closing ||
client.ServiceClient.State == CommunicationState.Faulted)
{
BrokerTracing.TraceVerbose(
"[ProxyClientPool].GetProxyClient: Client is not ready for use, remove it, {0}, {1}",
client,
client.ServiceClient.Endpoint.Address);
this.RemoveProxyClient(client);
if (client.ServiceClient.State == CommunicationState.Faulted)
{
client.AsyncClose();
}
}
else
{
BrokerTracing.TraceVerbose(
"[ProxyClientPool].GetProxyClient: Get a client ready for use, {0}, {1}",
client,
client.ServiceClient.Endpoint.Address);
return(client);
}
}
}
/// <summary>
/// Returns one ServieClient in the pool.
/// </summary>
private AzureServiceClient InternalGetProxyClient()
{
AzureServiceClient client = null;
if (!this.IsFull)
{
client = new AzureServiceClient(ProxyBinding.BrokerProxyBinding, this.proxyEpr);
BrokerTracing.TraceInfo(
"[ProxyClientPool].InternalGetProxyClient: Create a new client, {0}, {1}",
client,
client.ServiceClient.Endpoint.Address);
this.clientPool.Add(client);
BrokerTracing.TraceInfo(
"[ProxyClientPool].InternalGetProxyClient: Pool size is {0}",
this.clientPool.Count);
// add "1" to the RefCounts list since we will return this new client
this.clientRefCounts.Add(1);
return(client);
}
else
{
int poolSize = this.clientPool.Count;
if (poolSize > 0)
{
if (this.currentIndex <= 0 || this.currentIndex >= poolSize)
{
this.currentIndex = 0;
}
client = this.clientPool[this.currentIndex];
this.clientRefCounts[this.currentIndex]++;
this.currentIndex++;
}
BrokerTracing.TraceInfo(
"[ProxyClientPool].InternalGetProxyClient: Get a client from pool, {0}, {1}",
client,
client.ServiceClient.Endpoint.Address);
return(client);
}
}
/// <summary>
/// Remove the specified client from pool.
/// </summary>
/// <param name="client">targeted client</param>
/// <returns>removed from pool or not</returns>
public bool RemoveProxyClient(AzureServiceClient client)
{
for (int i = 0; i < this.clientPool.Count; i++)
{
if (ReferenceEquals(this.clientPool[i], client))
{
this.clientPool.RemoveAt(i);
this.clientRefCounts.RemoveAt(i);
BrokerTracing.TraceInfo("ProxyClientPool. RemoveProxyClient: Close client {0}.", client.ServiceClient.Endpoint.Address);
return(true);
}
}
return(false);
}
/// <summary>
/// Handle client failure
/// </summary>
/// <param name="client">targeted client</param>
private static void HandleClientFailure(AzureServiceClient client)
{
try
{
BrokerTracing.TraceWarning(
"[AzureDispatcher] HandleClientFailure: Handle invalid client, {0}, {1}",
client,
client.ServiceClient.Endpoint.Address);
Dictionary <EndpointAddress, ProxyClientPool> .ValueCollection pools;
lock (LockProxyClientPoolDic)
{
pools = ProxyClientPoolDic.Values;
}
foreach (var pool in pools)
{
bool existInPool = false;
lock (pool)
{
BrokerTracing.TraceVerbose(
"[AzureDispatcher] HandleClientFailure: Remove client {0} from pool.", client);
existInPool = pool.RemoveProxyClient(client);
}
if (existInPool)
{
BrokerTracing.TraceVerbose(
"[AzureDispatcher] HandleClientFailure: Close client {0}", client);
// Close the proxy client if any exception is encountered.
// As a result, async pending callback on clients will be
// invoked (with exception).
client.AsyncClose();
return;
}
}
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureDispatcher] HandleClientFailure: Error occurs, {0}", e);
}
}
/// <summary>
/// If parameter serviceClient is in the pool, clear the pool and return the list items.
/// The invoker of this method will async close returned clients outside a lock scope.
/// </summary>
/// <returns>the list items</returns>
public AzureServiceClient[] Clear(AzureServiceClient serviceClient)
{
if (serviceClient != null && this.clientPool.Contains(serviceClient))
{
AzureServiceClient[] clients = this.clientPool.ToArray();
this.clientPool.Clear();
this.clientRefCounts.Clear();
BrokerTracing.TraceWarning("[ProxyClientPool]. Clear: Client {0} is in the pool, so clear the pool {1}.", serviceClient, this.proxyEpr.ToString());
return(clients);
}
else
{
BrokerTracing.TraceWarning("[ProxyClientPool]. Clear: Client {0} is not in the pool {1}.", serviceClient, this.proxyEpr.ToString());
return(null);
}
}
/// <summary>
/// Remove the specified client from pool and update the reference count.
/// </summary>
/// <param name="client">targeted client</param>
/// <returns>removed from pool or not</returns>
public bool ReleaseProxyClient(AzureServiceClient client)
{
for (int i = 0; i < this.clientPool.Count; i++)
{
if (ReferenceEquals(this.clientPool[i], client))
{
this.clientRefCounts[i]--;
if (this.clientRefCounts[i] == 0)
{
this.clientPool.RemoveAt(i);
this.clientRefCounts.RemoveAt(i);
BrokerTracing.TraceInfo("[ProxyClientPool] .ReleaseProxyClient: close client {0} as its reference count reaches 0", client.ServiceClient.Endpoint.Address);
return(true);
}
}
}
return(false);
}
/// <summary>
/// Clear the client pool and close all the connections in it.
/// </summary>
/// <remarks>
/// This method has high cost. We need to be pretty sure that the exception
/// happens on the connection between broker and proxy, then call this method.
/// </remarks>
/// <returns>
/// is the specified client in the pool
/// </returns>
public override bool CleanupClient(IService client)
{
AzureServiceClient serviceClient = client as AzureServiceClient;
Debug.Assert(serviceClient != null);
BrokerTracing.TraceWarning(
BrokerTracing.GenerateTraceString(
"AzureDispatcher",
"CleanupClient",
this.TaskId,
-1,
serviceClient.ToString(),
string.Empty,
"Cleanup client."));
AzureServiceClient[] clients = null;
lock (this.proxyClientPool)
{
clients = this.proxyClientPool.Clear(serviceClient);
}
if (clients != null)
{
BrokerTracing.TraceWarning(
BrokerTracing.GenerateTraceString(
"AzureDispatcher",
"CleanupClient",
this.TaskId,
-1,
serviceClient.ToString(),
string.Empty,
string.Format("Close clients in the pool. count = {0}", clients.Length)));
// don't close client in above lock (this.proxyClientPool) scope to avoid deadlock
foreach (AzureServiceClient asc in clients)
{
asc.AsyncClose();
}
}
return(clients != null);
}
/// <summary>
/// Callback function to check ping results for each service client
/// </summary>
/// <param name="ar">async result object</param>
private static void EndPingProxyClient(IAsyncResult ar)
{
AzureServiceClient client = ar.AsyncState as AzureServiceClient;
try
{
client.EndProcessMessage(ar);
BrokerTracing.TraceVerbose("[AzureDispatcher] EndPingProxyClients: client {0} is ok", client.ServiceClient.Endpoint.Address);
}
catch (TimeoutException)
{
// TimeoutException may happen when the channel is sending a big request message.
// It doesn't mean the channel is broken.
BrokerTracing.TraceVerbose("[AzureDispatcher] EndPingProxyClients: client {0} timeout", client.ServiceClient.Endpoint.Address);
}
catch (Exception e)
{
BrokerTracing.TraceError("[AzureDispatcher] EndPingProxyClients: client {0} receives exception: {1}", client.ServiceClient.Endpoint.Address, e);
HandleClientFailure(client);
}
}
/// <summary>
/// Check connection status between the specified client and broker proxy by sending a ping message to broker proxy,
/// and close the client if the connnection is broken.
/// </summary>
/// <param name="client">the service client instance to be </param>
/// <returns>async result object</returns>
private static IAsyncResult BeginPingProxyClient(AzureServiceClient client)
{
Debug.Assert(client != null);
try
{
// no need to ping proxy if it is not ready
if (!client.IsConnectionReady())
{
return(null);
}
}
catch (Exception ex)
{
BrokerTracing.TraceWarning("[AzureDispatcher].BeginPingProxyClient: Exception {0}", ex);
// the client may be disposed, so its WaitHandleForOpen is already closed adn set null
return(null);
}
Message pingMessage = Message.CreateMessage(MessageVersion.Default, Constant.PingBrokerProxyAction);
BrokerTracing.TraceVerbose("[AzureDispatcher] BeginPingProxyClient: ping client {0}", client.ServiceClient.Endpoint.Address);
try
{
return(client.BeginProcessMessage(
pingMessage,
new ThreadHelper <IAsyncResult>(new AsyncCallback(EndPingProxyClient)).CallbackRoot,
client));
}
catch (Exception e)
{
BrokerTracing.TraceError("[AzureDispatcher] BeginPingProxyClients: sending ping message via client {0} encounters excpetion: {1}", client.ServiceClient.Endpoint.Address, e);
HandleClientFailure(client);
return(null);
}
}
/// <summary>
/// Opent the client and trigger GetNextRequest method.
/// </summary>
private static void AsyncStartWorker(object state)
{
BrokerTracing.TraceVerbose("[AzureServiceClient]. AsyncStartWorker is called.");
object[] objs = state as object[];
Debug.Assert(objs != null && objs.Length == 2);
AzureServiceClient client = objs[0] as AzureServiceClient;
int serviceOperationTimeout = (int)objs[1];
bool validConnection = false;
string clientInfo = string.Empty;
try
{
clientInfo = client.ToString();
// open the channel explicitly when it is shared by multi threads
client.ServiceClient.Open();
BrokerTracing.TraceVerbose("[AzureServiceClient]. AsyncStartWorker: Open the client succeeded, {0}", clientInfo);
// setting client.InnerChannel.OperationTimeout causes the channel factory to open, and then causes client.Open to fail,
// becasue we can't call Open when channel factory is already opened.
// it is fine to set OperationTimeout after client.Open
if (serviceOperationTimeout > 0)
{
BrokerTracing.TraceVerbose("[AzureServiceClient]. AsyncStartWorker: Set OperationTimeout {0} to the client {1}", serviceOperationTimeout, clientInfo);
client.ServiceClient.InnerChannel.OperationTimeout = TimeSpan.FromMilliseconds(serviceOperationTimeout);
}
validConnection = true;
}
catch (Exception e)
{
BrokerTracing.TraceError("[AzureServiceClient]. AsyncStartWorker: Open the client failed, {0}, {1}", clientInfo, e);
}
try
{
if (validConnection)
{
try
{
if (client.waitHandleForOpen != null)
{
BrokerTracing.TraceVerbose("[AzureServiceClient]. AsyncStartWorker: Release the wait handle, {0}", clientInfo);
client.waitHandleForOpen.Set();
}
}
catch (Exception ex)
{
BrokerTracing.TraceWarning("[AzureServiceClient].AsyncStartWorker: Exception {0}", ex);
// Exception may happen if the client is already closed. Ignore this
// exception because the client released that WaitHandle when closed.
}
BrokerTracing.TraceVerbose("[AzureServiceClient]. AsyncStartWorker: Call TriggerGetNextRequest, {0}", clientInfo);
// the client.WaitHandleForOpen.Set() is already called above before this.
client.TriggerGetNextRequest();
}
else
{
client.RecreateClient();
}
}
catch (Exception e)
{
BrokerTracing.TraceError("[AzureServiceClient]. AsyncStartWorker: Execption happens, {0}, {1}", clientInfo, e);
}
}