/// <summary>
/// Refresh the client.
/// For burst, cleanup the connection pool if the exception occurs
/// between broker and proxy.
/// </summary>
/// <param name="clientIndex">client index</param>
/// <param name="exceptionIndirect">
/// is the exception from the connection between broker and proxy,
/// or between proxy and host
/// </param>
/// <param name="messageId">message Id</param>
/// <returns>should increase the retry count or not</returns>
public override async Task <bool> RefreshClientAsync(int clientIndex, bool exceptionIndirect, Guid messageId)
{
BrokerTracing.TraceWarning(
BrokerTracing.GenerateTraceString(
"AzureNettcpRequestSender",
"RefreshClientAsync",
this.Dispatcher.TaskId,
clientIndex,
this.Client.ToString(),
messageId,
string.Format("exceptionIndirect = {0}", exceptionIndirect)));
bool increaseRetryCount = false;
if (exceptionIndirect)
{
this.Dispatcher.PassBindingFlags[clientIndex] = true;
increaseRetryCount = true;
}
else
{
increaseRetryCount = this.Dispatcher.CleanupClient(this.Client);
await this.CreateClientAsync(false, clientIndex).ConfigureAwait(false);
}
return(increaseRetryCount);
}
/// <summary>
/// Create a IService instance with specified index.
/// </summary>
/// <param name="getNextRequest">
/// trigger the client to retrieve next request
/// </param>
/// <param name="clientIndex">
/// index of the client
/// </param>
public virtual async Task CreateClientAsync(bool getNextRequest, int clientIndex)
{
if (this.IServiceClient != null)
{
BrokerTracing.TraceWarning(
BrokerTracing.GenerateTraceString(
"RequestSender",
"CreateClientAsync",
this.TaskId,
clientIndex,
this.IServiceClient.ToString(),
string.Empty,
"Closed former client proxy."));
this.CloseClient();
}
try
{
await this.CreateClientAsync().ConfigureAwait(false);
BrokerTracing.TraceVerbose(
BrokerTracing.GenerateTraceString(
"RequestSender",
"CreateClientAsync",
this.TaskId,
clientIndex,
this.IServiceClient.ToString(),
string.Empty,
"Created a new client."));
this.Dispatcher.PassBindingFlags[clientIndex] = true;
GetNextRequestState state = null;
if (getNextRequest)
{
state = new GetNextRequestState(this.Dispatcher.GetNextRequest, clientIndex);
}
this.StartClient(state);
}
catch (Exception e)
{
BrokerTracing.TraceEvent(
TraceEventType.Error,
0,
"[RequestSender] .CreateClientAsync: ID = {0}, init client failed: {1}",
this.TaskId,
e);
if (this.IServiceClient != null)
{
this.CloseClient();
}
this.Dispatcher.CloseThis();
}
}
/// <summary>
/// Send out request to the target proxy or host.
/// </summary>
/// <param name="data">dispatch data</param>
/// <param name="dispatchId">dispatch Id</param>
/// <param name="clientIndex">client index</param>
public virtual void SendRequest(DispatchData data, Guid dispatchId, int clientIndex)
{
Message requestMessage = data.BrokerQueueItem.Message;
Guid messageId = Utility.GetMessageIdFromMessage(requestMessage);
// Bug 12045: Convert message before preparing it as some headers
// might be added during preparation.
// Check version
if (requestMessage.Headers.MessageVersion != this.BackendBinding.MessageVersion)
{
BrokerTracing.TraceVerbose("[RequestSender].SendRequest: Convert message version from {0} to {1}", requestMessage.Headers.MessageVersion, this.BackendBinding.MessageVersion);
requestMessage = Utility.ConvertMessage(requestMessage, this.BackendBinding.MessageVersion);
}
this.PrepareMessage(requestMessage, dispatchId, this.Dispatcher.PassBindingFlags[clientIndex]);
data.Client = this.IServiceClient;
// Bug #16197: reserve request message action in case it gets dropped when wcf processes the message
data.RequestAction = requestMessage.Headers.Action;
data.DispatchTime = DateTime.Now;
this.IServiceClient.BeginProcessMessage(
requestMessage,
this.ProcessMessageCallback,
data);
this.lastSentTime = DateTime.UtcNow;
// Notice: the request may complete fast, and it is disposed when response comes back before following code executes.
// So don't access item.Message in following code.
// Bug #13430:
// (1) Define "try" as request sent by broker, so any request sent by broker successfully should
// have "current try count + 1" and not retried any more if retrylimit=0.
// (2) Don't increase the try count if EndpointNotFoundException occurs.
// (3) In "Burst to Azure" mode, connections are shared by dispatchers. The connection is possible to
// be killed because of failure in one dispatcher. Don't increase the try count in such case.
// Increase the try count when BeginProcessMessage succeeds.
data.BrokerQueueItem.TryCount++;
BrokerTracing.TraceVerbose(
BrokerTracing.GenerateTraceString(
"RequestSender",
"SendRequest",
this.TaskId,
clientIndex,
this.IServiceClient.ToString(),
messageId,
string.Format("Sent out message and increase the try count to {0}", data.BrokerQueueItem.TryCount)));
}
/// <summary>
/// Refresh the client.
/// For on-premise cluster, just create a new client.
/// </summary>
/// <param name="clientIndex">client index</param>
/// <param name="exceptionIndirect">
/// it is false for the on-premise cluster
/// </param>
/// <param name="messageId">message Id</param>
/// <returns>should increase the retry count or not</returns>
public override async Task <bool> RefreshClientAsync(int clientIndex, bool exceptionIndirect, Guid messageId)
{
BrokerTracing.TraceError(
BrokerTracing.GenerateTraceString(
"OnPremiseRequestSender",
"RefreshClientAsync",
this.TaskId,
clientIndex,
this.Client.ToString(),
messageId,
string.Format("exceptionIndirect = {0}", exceptionIndirect)));
await this.CreateClientAsync(false, clientIndex).ConfigureAwait(false);
return(true);
}
/// <summary>
/// Clear the client pool and close all the connections in it.
/// </summary>
/// <remarks>
/// This method has high cost. We need to be pretty sure that the exception
/// happens on the connection between broker and proxy, then call this method.
/// </remarks>
/// <returns>
/// is the specified client in the pool
/// </returns>
public override bool CleanupClient(IService client)
{
AzureServiceClient serviceClient = client as AzureServiceClient;
Debug.Assert(serviceClient != null);
BrokerTracing.TraceWarning(
BrokerTracing.GenerateTraceString(
"AzureDispatcher",
"CleanupClient",
this.TaskId,
-1,
serviceClient.ToString(),
string.Empty,
"Cleanup client."));
AzureServiceClient[] clients = null;
lock (this.proxyClientPool)
{
clients = this.proxyClientPool.Clear(serviceClient);
}
if (clients != null)
{
BrokerTracing.TraceWarning(
BrokerTracing.GenerateTraceString(
"AzureDispatcher",
"CleanupClient",
this.TaskId,
-1,
serviceClient.ToString(),
string.Empty,
string.Format("Close clients in the pool. count = {0}", clients.Length)));
// don't close client in above lock (this.proxyClientPool) scope to avoid deadlock
foreach (AzureServiceClient asc in clients)
{
asc.AsyncClose();
}
}
return(clients != null);
}
private void ProcessMessageCallback(IAsyncResult ar)
{
var data = (DispatchData)ar.AsyncState;
double seconds = (DateTime.UtcNow - this.lastSentTime).TotalSeconds;
this.totalSeconds += seconds;
this.totalCount++;
BrokerTracing.TraceVerbose(
BrokerTracing.GenerateTraceString(
"RequestSender",
"SendRequest",
this.TaskId,
data.ClientIndex,
this.IServiceClient.ToString(),
data.MessageId,
$"Current process time {seconds}, total seconds {this.totalSeconds.ToString()}, total count {this.totalCount.ToString()}, average {(this.totalSeconds / this.totalCount).ToString()}"));
this.Dispatcher.ProcessMessageCallback(ar); // Fire and forget here
}
/// <summary>
/// Receive response message from host.
/// </summary>
/// <param name="data">DispatchData instance</param>
public void ReceiveResponse(DispatchData data)
{
Contract.Requires(data.AsyncResult != null, string.Format(ContractMessageFormat, "DispatchData.AsyncResult"));
Contract.Requires(data.Client != null, string.Format(ContractMessageFormat, "DispatchData.Client"));
Contract.Requires(data.BrokerQueueItem != null, string.Format(ContractMessageFormat, "DispatchData.BrokerQueueItem"));
Contract.Ensures(
data.ReplyMessage != null || data.Exception != null,
"DispatchData should have either a reply message either an exception.");
Message reply = null;
string taskId = data.TaskId;
DateTime dispatchTime = data.DispatchTime;
int clientIndex = data.ClientIndex;
IService client = data.Client;
BrokerQueueItem item = data.BrokerQueueItem;
// Bug #16197: if request action field is dropped by wcf, restore it back
if (string.IsNullOrEmpty(item.Message.Headers.Action))
{
item.Message.Headers.Action = data.RequestAction;
}
this.dispatcher.items[clientIndex] = null;
this.dispatcher.DecreaseProcessingCount();
Guid messageId = Utility.GetMessageIdFromMessage(item.Message);
try
{
// Get the response message
reply = client.EndProcessMessage(data.AsyncResult);
data.ReplyMessage = reply;
// following method needs to be called before setting bServiceInitalizationCompleted flag
this.PostProcessMessage(reply);
// mark bServiceInitalizationCompleted flag to true;
if (!this.dispatcher.ServiceInitializationCompleted)
{
this.dispatcher.ServiceInitializationCompleted = true;
// nofity that it is connected to service host
this.dispatcher.OnServiceInstanceConnected(null);
}
BrokerTracing.EtwTrace.LogBackendResponseReceived(data.SessionId, taskId, messageId, reply.IsFault);
}
catch (EndpointNotFoundException e)
{
data.Exception = e;
// TODO: need ExceptionHandler
this.dispatcher.HandleEndpointNotFoundException(clientIndex, client, item, messageId, e);
return;
}
catch (StorageException e)
{
data.Exception = e;
// TODO: need ExceptionHandler
this.dispatcher.HandleStorageException(dispatchTime, clientIndex, client, item, messageId, e);
return;
}
catch (Exception e)
{
data.Exception = e;
// TODO: need ExceptionHandler
this.dispatcher.HandleException(dispatchTime, clientIndex, client, item, messageId, e);
return;
}
if (reply.IsFault)
{
// If any fault message is received, consider passing binding data again.
// TODO: pass binding data only on special error code
this.dispatcher.PassBindingFlags[clientIndex] = true;
}
bool servicePreempted = false;
if (reply.IsFault)
{
BrokerTracing.TraceVerbose(
BrokerTracing.GenerateTraceString(
"ResponseReceiver",
"ReceiveResponse",
taskId,
clientIndex,
client.ToString(),
messageId,
"The response is a fault message."));
// TODO: need ExceptionHandler
data.ExceptionHandled = this.dispatcher.HandleFaultExceptionRetry(clientIndex, item, reply, dispatchTime, out servicePreempted);
}
else
{
// if it is a normal response, check if the CancelEvent happens when request is being processed.
int index = reply.Headers.FindHeader(Constant.MessageHeaderPreemption, Constant.HpcHeaderNS);
if (index >= 0 && index < reply.Headers.Count)
{
servicePreempted = true;
int messageCount = reply.Headers.GetHeader <int>(index);
BrokerTracing.TraceVerbose(
BrokerTracing.GenerateTraceString(
"ResponseReceiver",
"ReceiveResponse",
taskId,
clientIndex,
client.ToString(),
messageId,
string.Format("The count of processing message in the counterpart host is {0}.", messageCount)));
// will remove the dispatcher if no processing message left on the host
if (messageCount == 0)
{
// Simulate a SessionFault and reuse the method HandleServiceInstanceFailure,
// and the message will be processed later as normal.
BrokerTracing.TraceVerbose(
BrokerTracing.GenerateTraceString(
"ResponseReceiver",
"ReceiveResponse",
taskId,
clientIndex,
client.ToString(),
messageId,
"(Preemption) Call HandleServiceInstanceFailure."));
// TODO: need ServiceInstanceFailureHandler
this.dispatcher.HandleServiceInstanceFailure(new SessionFault(SOAFaultCode.Service_Preempted, string.Empty));
}
}
}
data.ServicePreempted = servicePreempted;
// Debug Failure Test
SimulateFailure.FailOperation(1);
}