/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message); // Don't process messages that have already timed out if (message.IsExpired) { logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired"); message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target."); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } if (errorInjection && ShouldInjectError(message)) { if (logger.IsVerbose) { logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection"); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection"); RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection"); return; } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, message.GenericGrainType, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { if (SiloCanAcceptRequest(message)) { ReceiveRequest(message, target); } else if (message.MayResend(config.Globals)) { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Redirecting"); throw new NotImplementedException("RedirectRequest() is believed to be no longer necessary; please contact the Orleans team if you see this error."); } else { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Rejecting"); RejectMessage(message, Message.RejectionTypes.Transient, null, "Shutting down"); } } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message); logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } logger.Warn(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, String.Format("Intermediate warning for NonExistentActivation from Catalog.GetOrCreateActivation for message {0}", message), ex); ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero) { Scheduler.QueueWorkItem(new ClosureWorkItem( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async() => { try { await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync( nonExistentActivation); } catch (Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, String.Format("Failed to un-register NonExistentActivation {0}", nonExistentActivation), exc); } }, () => "LocalGrainDirectory.UnregisterConditionallyAsync"), catalog.SchedulingContext); } ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn(ErrorCode.Dispatcher_NoTargetActivation, "No target activation {0} for response message: {1}", nonExistentActivation, message); Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { EventSourceUtils.EmitEvent(message, OrleansDispatcherEvent.ReceiveMessageAction); MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message); // Don't process messages that have already timed out if (message.IsExpired) { logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired"); message.DropExpiredMessage(this.logger, MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target."); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType, message.RequestContextData, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { if (target.State == ActivationState.Valid) { this.activationCollector.TryRescheduleCollection(target); } // Silo is always capable to accept a new request. It's up to the activation to handle its internal state. // If activation is shutting down, it will queue and later forward this request. ReceiveRequest(message, target); } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = $"Error creating activation for {message.NewGrainType}. Message {message}"; logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } if (nea.IsStatelessWorker) { if (logger.IsEnabled(LogLevel.Debug)) { logger.Debug(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, $"Intermediate StatelessWorker NonExistentActivation for message {message}, Exception {ex}"); } } else { logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, $"Intermediate NonExistentActivation for message {message}, with Exception {ex}"); } ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context var origin = message.SendingSilo; scheduler.QueueWorkItem(new ClosureWorkItem( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async() => { try { await this.localGrainDirectory.UnregisterAfterNonexistingActivation( nonExistentActivation, origin); } catch (Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, $"Failed to un-register NonExistentActivation {nonExistentActivation}", exc); } }, "LocalGrainDirectory.UnregisterAfterNonexistingActivation"), catalog.SchedulingContext); ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn( ErrorCode.Dispatcher_NoTargetActivation, nonExistentActivation.Silo.IsClient ? "No target client {0} for response message: {1}. It's likely that the client recently disconnected." : "No target activation {0} for response message: {1}", nonExistentActivation, message); this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }
public void ReceiveMessage(Message msg) { this.messagingTrace.OnIncomingMessageAgentReceiveMessage(msg); // Find the activation it targets; first check for a system activation, then an app activation if (msg.TargetGrain.IsSystemTarget()) { SystemTarget target = this.activationDirectory.FindSystemTarget(msg.TargetActivation); if (target == null) { MessagingStatisticsGroup.OnRejectedMessage(msg); Message response = this.messageFactory.CreateRejectionResponse(msg, Message.RejectionTypes.Unrecoverable, string.Format("SystemTarget {0} not active on this silo. Msg={1}", msg.TargetGrain, msg)); this.messageCenter.SendMessage(response); this.logger.LogWarning( (int)ErrorCode.MessagingMessageFromUnknownActivation, "Received a message {Message} for an unknown SystemTarget: {Target}", msg, msg.TargetAddress); return; } target.ReceiveMessage(msg); } else if (messageCenter.TryDeliverToProxy(msg)) { return; } else { try { var targetActivation = catalog.GetOrCreateActivation( msg.TargetAddress, msg.IsNewPlacement, msg.RequestContextData); if (targetActivation is null) { // Activation does not exists and is not a new placement. if (msg.Direction == Message.Directions.Response) { logger.LogWarning( (int)ErrorCode.Dispatcher_NoTargetActivation, "No target activation {Activation} for response message: {Message}", msg.TargetActivation, msg); return; } else { logger.LogInformation( (int)ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, "Intermediate NonExistentActivation for message {Message}", msg); var nonExistentActivation = msg.TargetAddress; ProcessRequestToInvalidActivation(msg, nonExistentActivation, null, "Non-existent activation"); return; } } targetActivation.ReceiveMessage(msg); } catch (Exception ex) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(msg); logger.LogError( (int)ErrorCode.Dispatcher_ErrorCreatingActivation, ex, "Error creating activation for grain {TargetGrain} (interface: {InterfaceType}). Message {Message}", msg.TargetGrain, msg.InterfaceType, msg); this.RejectMessage(msg, Message.RejectionTypes.Transient, ex); } } }