/// <summary> /// Check if we can locally accept this message. /// Redirects if it can't be accepted. /// </summary> /// <param name="message"></param> /// <param name="targetActivation"></param> private void ReceiveRequest(Message message, ActivationData targetActivation) { lock (targetActivation) { if (!ActivationMayAcceptRequest(targetActivation, message)) { // Check for deadlock before Enqueueing. if (schedulingOptions.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) { try { CheckDeadlock(message); } catch (DeadlockException exc) { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Deadlock"); logger.Warn(ErrorCode.Dispatcher_DetectedDeadlock, "Detected Application Deadlock: {0}", exc.Message); // We want to send DeadlockException back as an application exception, rather than as a system rejection. SendResponse(message, Response.ExceptionResponse(exc)); return; } } EnqueueRequest(message, targetActivation); } else { HandleIncomingRequest(message, targetActivation); } } }
/// <summary> /// Enqueue message for local handling after transaction completes /// </summary> /// <param name="message"></param> /// <param name="targetActivation"></param> private void EnqueueRequest(Message message, ActivationData targetActivation) { var overloadException = targetActivation.CheckOverloaded(logger); if (overloadException != null) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Overload2"); RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + targetActivation); return; } if (Message.WriteMessagingTraces) { message.AddTimestamp(Message.LifecycleTag.EnqueueWaiting); } bool enqueuedOk = targetActivation.EnqueueMessage(message); if (!enqueuedOk) { ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "EnqueueRequest"); } // Dont count this as end of processing. The message will come back after queueing via HandleIncomingRequest. #if DEBUG // This is a hot code path, so using #if to remove diags from Release version // Note: Caller already holds lock on activation if (logger.IsVerbose2) { logger.Verbose2(ErrorCode.Dispatcher_EnqueueMessage, "EnqueueMessage for {0}: targetActivation={1}", message.TargetActivation, targetActivation.DumpStatus()); } #endif }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> public void ReceiveMessage(IGrainContext target, Message message) { var activation = (ActivationData)target; _messagingTrace.OnDispatcherReceiveMessage(message); // Don't process messages that have already timed out if (message.IsExpired) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message); _messagingTrace.OnDropExpiredMessage(message, MessagingStatisticsGroup.Phase.Dispatch); return; } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, activation); } else // Request or OneWay { if (activation.State == ActivationState.Valid) { _activationCollector.TryRescheduleCollection(activation); } // Silo is always capable to accept a new request. It's up to the activation to handle its internal state. // If activation is shutting down, it will queue and later forward this request. ReceiveRequest(message, activation); } }
internal void OnDispatcherDetectedDeadlock(Message message, ActivationData targetActivation, DeadlockException exception) { if (this.IsEnabled(DispatcherDetectedDeadlockEventName)) { this.Write(DispatcherDetectedDeadlockEventName, new { Message = message, Activation = targetActivation, Exception = exception }); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message); LogDispatcherDetectedDeadlock(this, message, targetActivation, exception); }
internal void OnDispatcherReceiveInvalidActivation(Message message, ActivationState activationState) { if (this.IsEnabled(DispatcherReceiveInvalidActivationEventName)) { this.Write(DispatcherReceiveInvalidActivationEventName, new { Message = message, ActivationState = activationState }); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message); LogDispatcherReceiveInvalidActivation(this, activationState, message, null); }
internal void OnDispatcherSelectTargetFailed(Message message, Exception exception) { if (this.IsEnabled(DispatcherSelectTargetFailedEventName)) { this.Write(DispatcherSelectTargetFailedEventName, new { Message = message, Exception = exception }); } if (ShouldLogError(exception)) { LogDispatcherSelectTargetFailed(this, message, exception); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message);
/// <summary> /// Send an outgoing message, may complete synchronously /// - may buffer for transaction completion / commit if it ends a transaction /// - choose target placement address, maintaining send order /// - add ordering info and maintain send order /// /// </summary> /// <param name="message"></param> /// <param name="sendingActivation"></param> public Task AsyncSendMessage(Message message, ActivationData sendingActivation = null) { try { var messageAddressingTask = AddressMessage(message); if (messageAddressingTask.Status == TaskStatus.RanToCompletion) { TransportMessage(message, sendingActivation); } else { return(TransportMessageAferSending(messageAddressingTask, message, sendingActivation)); } } catch (Exception ex) { OnAddressingFailure(ex, message); } return(Task.CompletedTask); async Task TransportMessageAferSending(Task addressMessageTask, Message m, ActivationData activation) { try { await addressMessageTask; } catch (Exception ex) { OnAddressingFailure(ex, message); return; } TransportMessage(m, activation); } void OnAddressingFailure(Exception ex, Message m) { if (ShouldLogError(ex)) { logger.Error(ErrorCode.Dispatcher_SelectTarget_Failed, $"SelectTarget failed with {ex.Message}", ex); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "SelectTarget failed"); RejectMessage(m, Message.RejectionTypes.Unrecoverable, ex); } }
/// <summary> /// Enqueue message for local handling after transaction completes /// </summary> /// <param name="message"></param> /// <param name="targetActivation"></param> private void EnqueueRequest(Message message, ActivationData targetActivation) { var overloadException = targetActivation.CheckOverloaded(logger); if (overloadException != null) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Overload2"); RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + targetActivation); return; } switch (targetActivation.EnqueueMessage(message)) { case ActivationData.EnqueueMessageResult.Success: // Great, nothing to do break; case ActivationData.EnqueueMessageResult.ErrorInvalidActivation: ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "EnqueueRequest"); break; case ActivationData.EnqueueMessageResult.ErrorActivateFailed: ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "EnqueueRequest", rejectMessages: true); break; case ActivationData.EnqueueMessageResult.ErrorStuckActivation: // Avoid any new call to this activation catalog.DeactivateStuckActivation(targetActivation); ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "EnqueueRequest - blocked grain"); break; default: throw new ArgumentOutOfRangeException(); } // Dont count this as end of processing. The message will come back after queueing via HandleIncomingRequest. #if DEBUG // This is a hot code path, so using #if to remove diags from Release version // Note: Caller already holds lock on activation if (logger.IsEnabled(LogLevel.Trace)) { logger.Trace(ErrorCode.Dispatcher_EnqueueMessage, "EnqueueMessage for {0}: targetActivation={1}", message.TargetActivation, targetActivation.DumpStatus()); } #endif }
/// <summary> /// Send an outgoing message, may complete synchronously /// - may buffer for transaction completion / commit if it ends a transaction /// - choose target placement address, maintaining send order /// - add ordering info and maintain send order /// /// </summary> /// <param name="message"></param> /// <param name="sendingActivation"></param> public Task AsyncSendMessage(Message message, ActivationData sendingActivation = null) { Action <Exception> onAddressingFailure = ex => { if (ShouldLogError(ex)) { logger.Error(ErrorCode.Dispatcher_SelectTarget_Failed, $"SelectTarget failed with {ex.Message}", ex); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "SelectTarget failed"); RejectMessage(message, Message.RejectionTypes.Unrecoverable, ex); }; Func <Task, Task> transportMessageAfterAddressing = async addressMessageTask => { try { await addressMessageTask; } catch (Exception ex) { onAddressingFailure(ex); return; } TransportMessage(message, sendingActivation); }; try { var messageAddressingTask = AddressMessage(message); if (messageAddressingTask.Status == TaskStatus.RanToCompletion) { TransportMessage(message, sendingActivation); } else { return(transportMessageAfterAddressing(messageAddressingTask)); } } catch (Exception ex) { onAddressingFailure(ex); } return(Task.CompletedTask); }
private void ReceiveResponse(Message message, ActivationData targetActivation) { lock (targetActivation) { if (targetActivation.State == ActivationState.Invalid || targetActivation.State == ActivationState.FailedToActivate) { logger.Warn((int)ErrorCode.Dispatcher_Receive_InvalidActivation, "Response received for {State} activation {Message}", targetActivation.State, message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Invalid"); return; } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedOk(message); if (Transport.TryDeliverToProxy(message)) { return; } this.catalog.RuntimeClient.ReceiveResponse(message); } }
/// <summary> /// Send an outgoing message /// - may buffer for transaction completion / commit if it ends a transaction /// - choose target placement address, maintaining send order /// - add ordering info & maintain send order /// /// </summary> /// <param name="message"></param> /// <param name="sendingActivation"></param> public async Task AsyncSendMessage(Message message, ActivationData sendingActivation = null) { try { await AddressMessage(message); TransportMessage(message); } catch (Exception ex) { if (!(ex.GetBaseException() is KeyNotFoundException)) { logger.Error(ErrorCode.Dispatcher_SelectTarget_Failed, String.Format("SelectTarget failed with {0}", ex.Message), ex); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "SelectTarget failed"); RejectMessage(message, Message.RejectionTypes.Unrecoverable, ex); } }
private void ReceiveResponse(Message message, ActivationData targetActivation) { lock (targetActivation) { if (targetActivation.State == ActivationState.Invalid) { logger.Warn(ErrorCode.Dispatcher_Receive_InvalidActivation, "Response received for invalid activation {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Ivalid"); return; } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedOk(message); if (Transport.TryDeliverToProxy(message)) { return; } RuntimeClient.Current.ReceiveResponse(message); } }
/// <summary> /// Enqueue message for local handling after transaction completes /// </summary> /// <param name="message"></param> /// <param name="targetActivation"></param> private void EnqueueRequest(Message message, ActivationData targetActivation) { var overloadException = targetActivation.CheckOverloaded(); if (overloadException != null) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message); _dispatcher.RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + targetActivation); return; } switch (targetActivation.EnqueueMessage(message)) { case ActivationData.EnqueueMessageResult.Success: // Great, nothing to do break; case ActivationData.EnqueueMessageResult.ErrorInvalidActivation: _dispatcher.ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "EnqueueRequest"); break; case ActivationData.EnqueueMessageResult.ErrorActivateFailed: _dispatcher.ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "EnqueueRequest", rejectMessages: true); break; case ActivationData.EnqueueMessageResult.ErrorStuckActivation: // Avoid any new call to this activation _dispatcher.ProcessRequestToStuckActivation(message, targetActivation, "EnqueueRequest - blocked grain"); break; default: throw new ArgumentOutOfRangeException(); } // Dont count this as end of processing. The message will come back after queueing via HandleIncomingRequest. }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { EventSourceUtils.EmitEvent(message, OrleansDispatcherEvent.ReceiveMessageAction); MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message); // Don't process messages that have already timed out if (message.IsExpired) { logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired"); message.DropExpiredMessage(this.logger, MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target."); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType, message.RequestContextData, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { if (target.State == ActivationState.Valid) { this.activationCollector.TryRescheduleCollection(target); } // Silo is always capable to accept a new request. It's up to the activation to handle its internal state. // If activation is shutting down, it will queue and later forward this request. ReceiveRequest(message, target); } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = $"Error creating activation for {message.NewGrainType}. Message {message}"; logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } if (nea.IsStatelessWorker) { if (logger.IsEnabled(LogLevel.Debug)) { logger.Debug(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, $"Intermediate StatelessWorker NonExistentActivation for message {message}, Exception {ex}"); } } else { logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, $"Intermediate NonExistentActivation for message {message}, with Exception {ex}"); } ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context var origin = message.SendingSilo; scheduler.QueueWorkItem(new ClosureWorkItem( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async() => { try { await this.localGrainDirectory.UnregisterAfterNonexistingActivation( nonExistentActivation, origin); } catch (Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, $"Failed to un-register NonExistentActivation {nonExistentActivation}", exc); } }, "LocalGrainDirectory.UnregisterAfterNonexistingActivation"), catalog.SchedulingContext); ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn( ErrorCode.Dispatcher_NoTargetActivation, nonExistentActivation.Silo.IsClient ? "No target client {0} for response message: {1}. It's likely that the client recently disconnected." : "No target activation {0} for response message: {1}", nonExistentActivation, message); this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message); // Don't process messages that have already timed out if (message.IsExpired) { logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired"); message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target."); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } if (errorInjection && ShouldInjectError(message)) { if (logger.IsVerbose) { logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection"); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection"); RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection"); return; } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, message.GenericGrainType, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { if (SiloCanAcceptRequest(message)) { ReceiveRequest(message, target); } else if (message.MayResend(config.Globals)) { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Redirecting"); throw new NotImplementedException("RedirectRequest() is believed to be no longer necessary; please contact the Orleans team if you see this error."); } else { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Rejecting"); RejectMessage(message, Message.RejectionTypes.Transient, null, "Shutting down"); } } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message); logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } logger.Warn(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, String.Format("Intermediate warning for NonExistentActivation from Catalog.GetOrCreateActivation for message {0}", message), ex); ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero) { Scheduler.QueueWorkItem(new ClosureWorkItem( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async() => { try { await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync( nonExistentActivation); } catch (Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, String.Format("Failed to un-register NonExistentActivation {0}", nonExistentActivation), exc); } }, () => "LocalGrainDirectory.UnregisterConditionallyAsync"), catalog.SchedulingContext); } ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn(ErrorCode.Dispatcher_NoTargetActivation, "No target activation {0} for response message: {1}", nonExistentActivation, message); Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }
public void ReceiveMessage(Message msg) { this.messagingTrace.OnIncomingMessageAgentReceiveMessage(msg); // Find the activation it targets; first check for a system activation, then an app activation if (msg.TargetGrain.IsSystemTarget()) { SystemTarget target = this.activationDirectory.FindSystemTarget(msg.TargetActivation); if (target == null) { MessagingStatisticsGroup.OnRejectedMessage(msg); Message response = this.messageFactory.CreateRejectionResponse(msg, Message.RejectionTypes.Unrecoverable, string.Format("SystemTarget {0} not active on this silo. Msg={1}", msg.TargetGrain, msg)); this.messageCenter.SendMessage(response); this.logger.LogWarning( (int)ErrorCode.MessagingMessageFromUnknownActivation, "Received a message {Message} for an unknown SystemTarget: {Target}", msg, msg.TargetAddress); return; } target.ReceiveMessage(msg); } else if (messageCenter.TryDeliverToProxy(msg)) { return; } else { try { var targetActivation = catalog.GetOrCreateActivation( msg.TargetAddress, msg.IsNewPlacement, msg.RequestContextData); if (targetActivation is null) { // Activation does not exists and is not a new placement. if (msg.Direction == Message.Directions.Response) { logger.LogWarning( (int)ErrorCode.Dispatcher_NoTargetActivation, "No target activation {Activation} for response message: {Message}", msg.TargetActivation, msg); return; } else { logger.LogInformation( (int)ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, "Intermediate NonExistentActivation for message {Message}", msg); var nonExistentActivation = msg.TargetAddress; ProcessRequestToInvalidActivation(msg, nonExistentActivation, null, "Non-existent activation"); return; } } targetActivation.ReceiveMessage(msg); } catch (Exception ex) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(msg); logger.LogError( (int)ErrorCode.Dispatcher_ErrorCreatingActivation, ex, "Error creating activation for grain {TargetGrain} (interface: {InterfaceType}). Message {Message}", msg.TargetGrain, msg.InterfaceType, msg); this.RejectMessage(msg, Message.RejectionTypes.Transient, ex); } } }