/// <summary> /// Send out request to the target proxy or host. /// </summary> /// <param name="data">dispatch data</param> /// <param name="dispatchId">dispatch Id</param> /// <param name="clientIndex">client index</param> public virtual void SendRequest(DispatchData data, Guid dispatchId, int clientIndex) { Message requestMessage = data.BrokerQueueItem.Message; Guid messageId = Utility.GetMessageIdFromMessage(requestMessage); // Bug 12045: Convert message before preparing it as some headers // might be added during preparation. // Check version if (requestMessage.Headers.MessageVersion != this.BackendBinding.MessageVersion) { BrokerTracing.TraceVerbose("[RequestSender].SendRequest: Convert message version from {0} to {1}", requestMessage.Headers.MessageVersion, this.BackendBinding.MessageVersion); requestMessage = Utility.ConvertMessage(requestMessage, this.BackendBinding.MessageVersion); } this.PrepareMessage(requestMessage, dispatchId, this.Dispatcher.PassBindingFlags[clientIndex]); data.Client = this.IServiceClient; // Bug #16197: reserve request message action in case it gets dropped when wcf processes the message data.RequestAction = requestMessage.Headers.Action; data.DispatchTime = DateTime.Now; this.IServiceClient.BeginProcessMessage( requestMessage, this.ProcessMessageCallback, data); this.lastSentTime = DateTime.UtcNow; // Notice: the request may complete fast, and it is disposed when response comes back before following code executes. // So don't access item.Message in following code. // Bug #13430: // (1) Define "try" as request sent by broker, so any request sent by broker successfully should // have "current try count + 1" and not retried any more if retrylimit=0. // (2) Don't increase the try count if EndpointNotFoundException occurs. // (3) In "Burst to Azure" mode, connections are shared by dispatchers. The connection is possible to // be killed because of failure in one dispatcher. Don't increase the try count in such case. // Increase the try count when BeginProcessMessage succeeds. data.BrokerQueueItem.TryCount++; BrokerTracing.TraceVerbose( BrokerTracing.GenerateTraceString( "RequestSender", "SendRequest", this.TaskId, clientIndex, this.IServiceClient.ToString(), messageId, string.Format("Sent out message and increase the try count to {0}", data.BrokerQueueItem.TryCount))); }
/// <summary> /// Put request back into broker queue /// </summary> /// <param name="data">indicating the instance of dispatch data</param> public void PutRequestBack(DispatchData data) { Contract.Requires(data.BrokerQueueItem != null); Contract.Ensures(data.BrokerQueueItem == null); BrokerTracing.EtwTrace.LogBackendRequestPutBack(data.SessionId, data.TaskId, data.MessageId); this.observer.RequestProcessingCompleted(); this.queueFactory.PutResponseAsync(null, data.BrokerQueueItem); // Set to null since we returned it back to queue data.BrokerQueueItem = null; }
/// <summary> /// Handle retry limit exceeded /// </summary> /// <param name="data"></param> public void HandleRetryLimitExceeded(DispatchData data) { // handle retry limit only when we get exception or fault message Contract.Requires(data.Exception != null || (data.ReplyMessage != null && data.ReplyMessage.IsFault == true), "Retry Limit Exceeded when there is exception or fault message"); int retryLimit = this.sharedData.Config.LoadBalancing.MessageResendLimit; BrokerTracing.EtwTrace.LogBackendResponseReceivedRetryLimitExceed(data.SessionId, data.TaskId, data.MessageId, retryLimit); // Initialize retry error and fault reason RetryOperationError retryError; FaultReason faultReason; if (data.Exception != null) { // generate fault exception from original exception retryError = new RetryOperationError(data.Exception.Message); string exceptionFaultReason = String.Format(SR.RetryLimitExceeded, retryLimit + 1, data.Exception.Message); faultReason = new FaultReason(exceptionFaultReason); } else { #region Debug Failure Test SimulateFailure.FailOperation(1); #endregion // generate fault exception from original reply MessageFault messageFault = MessageFault.CreateFault(data.ReplyMessage, Constant.MaxBufferSize); retryError = messageFault.GetDetail <RetryOperationError>(); faultReason = messageFault.Reason; // close original reply message data.ReplyMessage.Close(); } retryError.RetryCount = retryLimit + 1; retryError.LastFailedServiceId = data.TaskId; // Construct the new exception FaultException <RetryOperationError> faultException = new FaultException <RetryOperationError>(retryError, faultReason, Constant.RetryLimitExceedFaultCode, RetryOperationError.Action); data.Exception = faultException; this.responseQueueAdapter.PutResponseBack(data); }
/// <summary> /// Put response back into broker queue /// </summary> /// <param name="data">indicating the instance of dispatch data</param> public void PutResponseBack(DispatchData data) { Contract.Requires(data.BrokerQueueItem != null); // Either the exception is null or it is a FaultException<RetryOperationError>; var faultException = data.Exception as FaultException <RetryOperationError>; Contract.Requires(data.Exception == null || faultException != null); // Either we will get a reply message or an exception. Contract.Requires(data.ReplyMessage != null || data.Exception != null); Contract.Requires(data.DispatchTime != null); Contract.Ensures(data.ReplyMessage == null); Contract.Ensures(data.BrokerQueueItem == null); Contract.Ensures(data.Exception == null); // Indicate that a call has completed try { long callDuration = DateTime.Now.Subtract(data.DispatchTime).Ticks / 10000 / (this.serviceRequestPrefetchCount + 1); this.observer.CallComplete(callDuration); } catch (Exception e) { // There may be some null reference exception while cleaning up // Ignore these exceptions BrokerTracing.TraceEvent(TraceEventType.Warning, 0, "[ResponseQueueAdapter] ID = {0} Exception throwed while updating the broker observer: {1}", data.TaskId, e); } var item = data.BrokerQueueItem; if (item.Context is DummyRequestContext) { Message reply = null; if (data.Exception != null) { if (faultException != null) { reply = Message.CreateMessage(MessageVersion.Default, faultException.CreateMessageFault(), faultException.Action); // Only add relatesTo header to WSAddressing messages if (item.Message.Headers.MessageId != null && MessageVersion.Default.Addressing == AddressingVersion.WSAddressing10) { reply.Headers.RelatesTo = item.Message.Headers.MessageId; } BrokerTracing.EtwTrace.LogBackendGeneratedFaultReply(data.SessionId, data.TaskId, data.MessageId, reply.ToString()); } else { Debug.Fail("The exception in DispatchData should be an instance of FaultException<RetryOperationError>"); } } else if (data.ReplyMessage != null) { reply = data.ReplyMessage; // Put the response to the broker queue BrokerTracing.EtwTrace.LogBackendResponseStored(data.SessionId, data.TaskId, data.MessageId, reply.IsFault); } else { Debug.Fail("Both ReplyMessage and Exception are null. This shouldn't happen."); } this.queueFactory.PutResponseAsync(reply, item); } else { Message reply = null; if (data.Exception != null) { if (faultException != null) { reply = FrontEndFaultMessage.GenerateFaultMessage(item.Message, item.Context.MessageVersion, faultException); BrokerTracing.EtwTrace.LogBackendGeneratedFaultReply(data.SessionId, data.TaskId, data.MessageId, reply.ToString()); } else { Debug.Fail("The exception in DispatchData should be an instance of FaultException<RetryOperationError>"); } } else if (data.ReplyMessage != null) { reply = data.ReplyMessage; // Reply the message // Convert the message version if (reply.Version != item.Context.MessageVersion) { if (reply.IsFault) { MessageFault fault = MessageFault.CreateFault(reply, int.MaxValue); reply = Message.CreateMessage(item.Context.MessageVersion, fault, reply.Headers.Action); } else { reply = Message.CreateMessage(item.Context.MessageVersion, reply.Headers.Action, reply.GetReaderAtBodyContents()); } } } else { Debug.Fail("Both ReplyMessage and Exception are null. This shouldn't happen."); } this.TryReplyMessage(data.SessionId, item.Context, reply, data.MessageId); // dispose request item item.Dispose(); } // Set to null since we returned it back to queue data.BrokerQueueItem = null; data.ReplyMessage = null; data.Exception = null; }
/// <summary> /// Receive response message from host. /// </summary> /// <param name="data">DispatchData instance</param> public void ReceiveResponse(DispatchData data) { Contract.Requires(data.AsyncResult != null, string.Format(ContractMessageFormat, "DispatchData.AsyncResult")); Contract.Requires(data.Client != null, string.Format(ContractMessageFormat, "DispatchData.Client")); Contract.Requires(data.BrokerQueueItem != null, string.Format(ContractMessageFormat, "DispatchData.BrokerQueueItem")); Contract.Ensures( data.ReplyMessage != null || data.Exception != null, "DispatchData should have either a reply message either an exception."); Message reply = null; string taskId = data.TaskId; DateTime dispatchTime = data.DispatchTime; int clientIndex = data.ClientIndex; IService client = data.Client; BrokerQueueItem item = data.BrokerQueueItem; // Bug #16197: if request action field is dropped by wcf, restore it back if (string.IsNullOrEmpty(item.Message.Headers.Action)) { item.Message.Headers.Action = data.RequestAction; } this.dispatcher.items[clientIndex] = null; this.dispatcher.DecreaseProcessingCount(); Guid messageId = Utility.GetMessageIdFromMessage(item.Message); try { // Get the response message reply = client.EndProcessMessage(data.AsyncResult); data.ReplyMessage = reply; // following method needs to be called before setting bServiceInitalizationCompleted flag this.PostProcessMessage(reply); // mark bServiceInitalizationCompleted flag to true; if (!this.dispatcher.ServiceInitializationCompleted) { this.dispatcher.ServiceInitializationCompleted = true; // nofity that it is connected to service host this.dispatcher.OnServiceInstanceConnected(null); } BrokerTracing.EtwTrace.LogBackendResponseReceived(data.SessionId, taskId, messageId, reply.IsFault); } catch (EndpointNotFoundException e) { data.Exception = e; // TODO: need ExceptionHandler this.dispatcher.HandleEndpointNotFoundException(clientIndex, client, item, messageId, e); return; } catch (StorageException e) { data.Exception = e; // TODO: need ExceptionHandler this.dispatcher.HandleStorageException(dispatchTime, clientIndex, client, item, messageId, e); return; } catch (Exception e) { data.Exception = e; // TODO: need ExceptionHandler this.dispatcher.HandleException(dispatchTime, clientIndex, client, item, messageId, e); return; } if (reply.IsFault) { // If any fault message is received, consider passing binding data again. // TODO: pass binding data only on special error code this.dispatcher.PassBindingFlags[clientIndex] = true; } bool servicePreempted = false; if (reply.IsFault) { BrokerTracing.TraceVerbose( BrokerTracing.GenerateTraceString( "ResponseReceiver", "ReceiveResponse", taskId, clientIndex, client.ToString(), messageId, "The response is a fault message.")); // TODO: need ExceptionHandler data.ExceptionHandled = this.dispatcher.HandleFaultExceptionRetry(clientIndex, item, reply, dispatchTime, out servicePreempted); } else { // if it is a normal response, check if the CancelEvent happens when request is being processed. int index = reply.Headers.FindHeader(Constant.MessageHeaderPreemption, Constant.HpcHeaderNS); if (index >= 0 && index < reply.Headers.Count) { servicePreempted = true; int messageCount = reply.Headers.GetHeader <int>(index); BrokerTracing.TraceVerbose( BrokerTracing.GenerateTraceString( "ResponseReceiver", "ReceiveResponse", taskId, clientIndex, client.ToString(), messageId, string.Format("The count of processing message in the counterpart host is {0}.", messageCount))); // will remove the dispatcher if no processing message left on the host if (messageCount == 0) { // Simulate a SessionFault and reuse the method HandleServiceInstanceFailure, // and the message will be processed later as normal. BrokerTracing.TraceVerbose( BrokerTracing.GenerateTraceString( "ResponseReceiver", "ReceiveResponse", taskId, clientIndex, client.ToString(), messageId, "(Preemption) Call HandleServiceInstanceFailure.")); // TODO: need ServiceInstanceFailureHandler this.dispatcher.HandleServiceInstanceFailure(new SessionFault(SOAFaultCode.Service_Preempted, string.Empty)); } } } data.ServicePreempted = servicePreempted; // Debug Failure Test SimulateFailure.FailOperation(1); }