/// <summary> /// Enqueue message for local handling after transaction completes /// </summary> /// <param name="message"></param> /// <param name="targetActivation"></param> private void EnqueueRequest(Message message, ActivationData targetActivation) { var overloadException = targetActivation.CheckOverloaded(logger); if (overloadException != null) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Overload2"); RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + targetActivation); return; } switch (targetActivation.EnqueueMessage(message)) { case ActivationData.EnqueueMessageResult.Success: // Great, nothing to do break; case ActivationData.EnqueueMessageResult.ErrorInvalidActivation: ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "EnqueueRequest"); break; case ActivationData.EnqueueMessageResult.ErrorStuckActivation: // Avoid any new call to this activation catalog.DeactivateStuckActivation(targetActivation); ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "EnqueueRequest - blocked grain"); break; default: throw new ArgumentOutOfRangeException(); } // Dont count this as end of processing. The message will come back after queueing via HandleIncomingRequest. #if DEBUG // This is a hot code path, so using #if to remove diags from Release version // Note: Caller already holds lock on activation if (logger.IsEnabled(LogLevel.Trace)) { logger.Trace(ErrorCode.Dispatcher_EnqueueMessage, "EnqueueMessage for {0}: targetActivation={1}", message.TargetActivation, targetActivation.DumpStatus()); } #endif }
private void ReceiveResponse(Message message, ActivationData targetActivation) { lock (targetActivation) { if (targetActivation.State == ActivationState.Invalid || targetActivation.State == ActivationState.FailedToActivate) { this.messagingTrace.OnDispatcherReceiveInvalidActivation(message, targetActivation.State); return; } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedOk(message); if (Transport.TryDeliverToProxy(message)) { return; } this.catalog.RuntimeClient.ReceiveResponse(message); } }
private void ReceiveResponse(Message message, ActivationData targetActivation) { lock (targetActivation) { if (targetActivation.State == ActivationState.Invalid || targetActivation.State == ActivationState.FailedToActivate) { logger.Warn((int)ErrorCode.Dispatcher_Receive_InvalidActivation, "Response received for {State} activation {Message}", targetActivation.State, message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Invalid"); return; } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedOk(message); if (Transport.TryDeliverToProxy(message)) { return; } this.catalog.RuntimeClient.ReceiveResponse(message); } }
/// <summary> /// Send an outgoing message /// - may buffer for transaction completion / commit if it ends a transaction /// - choose target placement address, maintaining send order /// - add ordering info & maintain send order /// /// </summary> /// <param name="message"></param> /// <param name="sendingActivation"></param> public async Task AsyncSendMessage(Message message, ActivationData sendingActivation = null) { try { await AddressMessage(message); TransportMessage(message); } catch (Exception ex) { if (!(ex.GetBaseException() is KeyNotFoundException)) { logger.Error(ErrorCode.Dispatcher_SelectTarget_Failed, String.Format("SelectTarget failed with {0}", ex.Message), ex); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "SelectTarget failed"); RejectMessage(message, Message.RejectionTypes.Unrecoverable, ex); } }
private void ReceiveResponse(Message message, ActivationData targetActivation) { lock (targetActivation) { if (targetActivation.State == ActivationState.Invalid) { logger.Warn(ErrorCode.Dispatcher_Receive_InvalidActivation, "Response received for invalid activation {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Ivalid"); return; } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedOk(message); if (Transport.TryDeliverToProxy(message)) { return; } RuntimeClient.Current.ReceiveResponse(message); } }
/// <summary> /// Handle an incoming message and queue/invoke appropriate handler /// </summary> /// <param name="message"></param> /// <param name="targetActivation"></param> public void HandleIncomingRequest(Message message, ActivationData targetActivation) { lock (targetActivation) { if (targetActivation.State == ActivationState.Invalid || targetActivation.State == ActivationState.FailedToActivate) { ProcessRequestToInvalidActivation( message, targetActivation.Address, targetActivation.ForwardingAddress, "HandleIncomingRequest", rejectMessages: targetActivation.State == ActivationState.FailedToActivate); return; } if (targetActivation.Grain.IsLegacyGrain() && message.IsUsingInterfaceVersions) { var request = (InvokeMethodRequest)message.BodyObject; var compatibilityDirector = compatibilityDirectorManager.GetDirector(request.InterfaceId); var currentVersion = this.grainTypeManager.GetLocalSupportedVersion(request.InterfaceId); if (!compatibilityDirector.IsCompatible(request.InterfaceVersion, currentVersion)) { catalog.DeactivateActivationOnIdle(targetActivation); ProcessRequestToInvalidActivation( message, targetActivation.Address, targetActivation.ForwardingAddress, "HandleIncomingRequest - Incompatible request"); return; } } // Now we can actually scheduler processing of this request targetActivation.RecordRunning(message, message.IsAlwaysInterleave); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedOk(message); this.messagingTrace.OnScheduleMessage(message); scheduler.QueueWorkItem(new InvokeWorkItem(targetActivation, message, this, this.invokeWorkItemLogger)); } }
/// <summary> /// Check if we can locally accept this message. /// Redirects if it can't be accepted. /// </summary> /// <param name="message"></param> /// <param name="targetActivation"></param> private void ReceiveRequest(Message message, ActivationData targetActivation) { lock (targetActivation) { if (targetActivation.State == ActivationState.Invalid) { ProcessRequestToInvalidActivation( message, targetActivation.Address, targetActivation.ForwardingAddress, "ReceiveRequest"); } else if (!ActivationMayAcceptRequest(targetActivation, message)) { // Check for deadlock before Enqueueing. if (config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) { try { CheckDeadlock(message); } catch (DeadlockException exc) { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Deadlock"); logger.Warn(ErrorCode.Dispatcher_DetectedDeadlock, "Detected Application Deadlock: {0}", exc.Message); // We want to send DeadlockException back as an application exception, rather than as a system rejection. SendResponse(message, Response.ExceptionResponse(exc)); return; } } EnqueueRequest(message, targetActivation); } else { HandleIncomingRequest(message, targetActivation); } } }
/// <summary> /// Handle an incoming message and queue/invoke appropriate handler /// </summary> /// <param name="message"></param> /// <param name="targetActivation"></param> public void HandleIncomingRequest(Message message, ActivationData targetActivation) { lock (targetActivation) { if (targetActivation.State == ActivationState.Invalid || targetActivation.State == ActivationState.FailedToActivate) { _dispatcher.ProcessRequestToInvalidActivation( message, targetActivation.Address, targetActivation.ForwardingAddress, "HandleIncomingRequest", rejectMessages: targetActivation.State == ActivationState.FailedToActivate); return; } if (message.InterfaceVersion > 0) { var compatibilityDirector = _compatibilityDirectorManager.GetDirector(message.InterfaceType); var currentVersion = _versionManifest.GetLocalVersion(message.InterfaceType); if (!compatibilityDirector.IsCompatible(message.InterfaceVersion, currentVersion)) { _catalog.DeactivateActivationOnIdle(targetActivation); _dispatcher.ProcessRequestToInvalidActivation( message, targetActivation.Address, targetActivation.ForwardingAddress, "HandleIncomingRequest - Incompatible request"); return; } } // Now we can actually scheduler processing of this request targetActivation.RecordRunning(message, message.IsAlwaysInterleave); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedOk(message); _messagingTrace.OnScheduleMessage(message); _scheduler.QueueWorkItem(new InvokeWorkItem(targetActivation, message, _catalog.RuntimeClient, this)); } }
/// <summary> /// Handle an incoming message and queue/invoke appropriate handler /// </summary> /// <param name="message"></param> /// <param name="targetActivation"></param> public void HandleIncomingRequest(Message message, ActivationData targetActivation) { lock (targetActivation) { if (targetActivation.State == ActivationState.Invalid) { ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "HandleIncomingRequest"); return; } // Now we can actually scheduler processing of this request targetActivation.RecordRunning(message); var context = new SchedulingContext(targetActivation); if (Message.WriteMessagingTraces) { message.AddTimestamp(Message.LifecycleTag.EnqueueWorkItem); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedOk(message); Scheduler.QueueWorkItem(new InvokeWorkItem(targetActivation, message, context), context); } }
/// <summary> /// Enqueue message for local handling after transaction completes /// </summary> /// <param name="message"></param> /// <param name="targetActivation"></param> private void EnqueueRequest(Message message, ActivationData targetActivation) { var overloadException = targetActivation.CheckOverloaded(); if (overloadException != null) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message); _dispatcher.RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + targetActivation); return; } switch (targetActivation.EnqueueMessage(message)) { case ActivationData.EnqueueMessageResult.Success: // Great, nothing to do break; case ActivationData.EnqueueMessageResult.ErrorInvalidActivation: _dispatcher.ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "EnqueueRequest"); break; case ActivationData.EnqueueMessageResult.ErrorActivateFailed: _dispatcher.ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "EnqueueRequest", rejectMessages: true); break; case ActivationData.EnqueueMessageResult.ErrorStuckActivation: // Avoid any new call to this activation _dispatcher.ProcessRequestToStuckActivation(message, targetActivation, "EnqueueRequest - blocked grain"); break; default: throw new ArgumentOutOfRangeException(); } // Dont count this as end of processing. The message will come back after queueing via HandleIncomingRequest. }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { EventSourceUtils.EmitEvent(message, OrleansDispatcherEvent.ReceiveMessageAction); MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message); // Don't process messages that have already timed out if (message.IsExpired) { logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired"); message.DropExpiredMessage(this.logger, MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target."); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType, message.RequestContextData, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { if (target.State == ActivationState.Valid) { this.activationCollector.TryRescheduleCollection(target); } // Silo is always capable to accept a new request. It's up to the activation to handle its internal state. // If activation is shutting down, it will queue and later forward this request. ReceiveRequest(message, target); } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = $"Error creating activation for {message.NewGrainType}. Message {message}"; logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } if (nea.IsStatelessWorker) { if (logger.IsEnabled(LogLevel.Debug)) { logger.Debug(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, $"Intermediate StatelessWorker NonExistentActivation for message {message}, Exception {ex}"); } } else { logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, $"Intermediate NonExistentActivation for message {message}, with Exception {ex}"); } ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context var origin = message.SendingSilo; scheduler.QueueWorkItem(new ClosureWorkItem( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async() => { try { await this.localGrainDirectory.UnregisterAfterNonexistingActivation( nonExistentActivation, origin); } catch (Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, $"Failed to un-register NonExistentActivation {nonExistentActivation}", exc); } }, "LocalGrainDirectory.UnregisterAfterNonexistingActivation"), catalog.SchedulingContext); ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn( ErrorCode.Dispatcher_NoTargetActivation, nonExistentActivation.Silo.IsClient ? "No target client {0} for response message: {1}. It's likely that the client recently disconnected." : "No target activation {0} for response message: {1}", nonExistentActivation, message); this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null) { bool forwardingSucceded = true; try { if (logger.IsEnabled(LogLevel.Information)) { logger.LogInformation( (int)ErrorCode.Messaging_Dispatcher_TryForward, "Trying to forward after {FailedOperation}, ForwardCount = {ForwardCount}. OldAddress = {OldAddress}, ForwardingAddress = {ForwardingAddress}, Message {Message}, Exception: {Exception}.", failedOperation, message.ForwardCount, oldAddress, forwardingAddress, message, exc); } // if this message is from a different cluster and hit a non-existing activation // in this cluster (which can happen due to stale cache or directory states) // we forward it back to the original silo it came from in the original cluster, // and target it to a fictional activation that is guaranteed to not exist. // This ensures that the GSI protocol creates a new instance there instead of here. if (forwardingAddress == null && message.TargetSilo != message.SendingSilo && !this.localGrainDirectory.IsSiloInCluster(message.SendingSilo)) { message.IsReturnedFromRemoteCluster = true; // marks message to force invalidation of stale directory entry forwardingAddress = ActivationAddress.NewActivationAddress(message.SendingSilo, message.TargetGrain); logger.Info(ErrorCode.Messaging_Dispatcher_ReturnToOriginCluster, $"Forwarding back to origin cluster, to fictional activation {message}"); } MessagingProcessingStatisticsGroup.OnDispatcherMessageReRouted(message); if (oldAddress != null) { message.AddToCacheInvalidationHeader(oldAddress); } forwardingSucceded = this.TryForwardMessage(message, forwardingAddress); } catch (Exception exc2) { forwardingSucceded = false; exc = exc2; } finally { var sentRejection = false; // If the message was a one-way message, send a cache invalidation response even if the message was successfully forwarded. if (message.Direction == Message.Directions.OneWay) { this.RejectMessage( message, Message.RejectionTypes.CacheInvalidation, exc, "OneWay message sent to invalid activation"); sentRejection = true; } if (!forwardingSucceded) { var str = $"Forwarding failed: tried to forward message {message} for {message.ForwardCount} times after {failedOperation} to invalid activation. Rejecting now."; logger.Warn(ErrorCode.Messaging_Dispatcher_TryForwardFailed, str, exc); if (!sentRejection) { RejectMessage(message, Message.RejectionTypes.Transient, exc, str); } } } }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message); // Don't process messages that have already timed out if (message.IsExpired) { logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired"); message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target."); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } if (errorInjection && ShouldInjectError(message)) { if (logger.IsVerbose) { logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection"); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection"); RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection"); return; } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, message.GenericGrainType, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { if (SiloCanAcceptRequest(message)) { ReceiveRequest(message, target); } else if (message.MayResend(config.Globals)) { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Redirecting"); throw new NotImplementedException("RedirectRequest() is believed to be no longer necessary; please contact the Orleans team if you see this error."); } else { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Rejecting"); RejectMessage(message, Message.RejectionTypes.Transient, null, "Shutting down"); } } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message); logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } logger.Warn(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, String.Format("Intermediate warning for NonExistentActivation from Catalog.GetOrCreateActivation for message {0}", message), ex); ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero) { Scheduler.QueueWorkItem(new ClosureWorkItem( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async() => { try { await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync( nonExistentActivation); } catch (Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, String.Format("Failed to un-register NonExistentActivation {0}", nonExistentActivation), exc); } }, () => "LocalGrainDirectory.UnregisterConditionallyAsync"), catalog.SchedulingContext); } ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn(ErrorCode.Dispatcher_NoTargetActivation, "No target activation {0} for response message: {1}", nonExistentActivation, message); Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }
public void ReceiveMessage(Message msg) { this.messagingTrace.OnIncomingMessageAgentReceiveMessage(msg); // Find the activation it targets; first check for a system activation, then an app activation if (msg.TargetGrain.IsSystemTarget()) { SystemTarget target = this.activationDirectory.FindSystemTarget(msg.TargetActivation); if (target == null) { MessagingStatisticsGroup.OnRejectedMessage(msg); Message response = this.messageFactory.CreateRejectionResponse(msg, Message.RejectionTypes.Unrecoverable, string.Format("SystemTarget {0} not active on this silo. Msg={1}", msg.TargetGrain, msg)); this.messageCenter.SendMessage(response); this.logger.LogWarning( (int)ErrorCode.MessagingMessageFromUnknownActivation, "Received a message {Message} for an unknown SystemTarget: {Target}", msg, msg.TargetAddress); return; } target.ReceiveMessage(msg); } else if (messageCenter.TryDeliverToProxy(msg)) { return; } else { try { var targetActivation = catalog.GetOrCreateActivation( msg.TargetAddress, msg.IsNewPlacement, msg.RequestContextData); if (targetActivation is null) { // Activation does not exists and is not a new placement. if (msg.Direction == Message.Directions.Response) { logger.LogWarning( (int)ErrorCode.Dispatcher_NoTargetActivation, "No target activation {Activation} for response message: {Message}", msg.TargetActivation, msg); return; } else { logger.LogInformation( (int)ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, "Intermediate NonExistentActivation for message {Message}", msg); var nonExistentActivation = msg.TargetAddress; ProcessRequestToInvalidActivation(msg, nonExistentActivation, null, "Non-existent activation"); return; } } targetActivation.ReceiveMessage(msg); } catch (Exception ex) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(msg); logger.LogError( (int)ErrorCode.Dispatcher_ErrorCreatingActivation, ex, "Error creating activation for grain {TargetGrain} (interface: {InterfaceType}). Message {Message}", msg.TargetGrain, msg.InterfaceType, msg); this.RejectMessage(msg, Message.RejectionTypes.Transient, ex); } } }