internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null) { bool forwardingSucceded = true; try { this.messagingTrace.OnDispatcherForwarding(message, oldAddress, forwardingAddress, failedOperation, exc); // if this message is from a different cluster and hit a non-existing activation // in this cluster (which can happen due to stale cache or directory states) // we forward it back to the original silo it came from in the original cluster, // and target it to a fictional activation that is guaranteed to not exist. // This ensures that the GSI protocol creates a new instance there instead of here. if (forwardingAddress == null && message.TargetSilo != message.SendingSilo && !this.localGrainDirectory.IsSiloInCluster(message.SendingSilo)) { message.IsReturnedFromRemoteCluster = true; // marks message to force invalidation of stale directory entry forwardingAddress = ActivationAddress.NewActivationAddress(message.SendingSilo, message.TargetGrain); logger.Info(ErrorCode.Messaging_Dispatcher_ReturnToOriginCluster, $"Forwarding back to origin cluster, to fictional activation {message}"); } if (oldAddress != null) { message.AddToCacheInvalidationHeader(oldAddress); } forwardingSucceded = this.TryForwardMessage(message, forwardingAddress); } catch (Exception exc2) { forwardingSucceded = false; exc = exc2; } finally { var sentRejection = false; // If the message was a one-way message, send a cache invalidation response even if the message was successfully forwarded. if (message.Direction == Message.Directions.OneWay) { this.RejectMessage( message, Message.RejectionTypes.CacheInvalidation, exc, "OneWay message sent to invalid activation"); sentRejection = true; } if (!forwardingSucceded) { this.messagingTrace.OnDispatcherForwardingFailed(message, oldAddress, forwardingAddress, failedOperation, exc); if (!sentRejection) { var str = $"Forwarding failed: tried to forward message {message} for {message.ForwardCount} times after {failedOperation} to invalid activation. Rejecting now."; RejectMessage(message, Message.RejectionTypes.Transient, exc, str); } } } }
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null) { bool forwardingSucceded = true; try { logger.Info(ErrorCode.Messaging_Dispatcher_TryForward, String.Format("Trying to forward after {0}, ForwardCount = {1}. Message {2}.", failedOperation, message.ForwardCount, message)); MessagingProcessingStatisticsGroup.OnDispatcherMessageReRouted(message); if (oldAddress != null) { message.AddToCacheInvalidationHeader(oldAddress); } forwardingSucceded = InsideRuntimeClient.Current.TryForwardMessage(message, forwardingAddress); } catch (Exception exc2) { forwardingSucceded = false; exc = exc2; } finally { if (!forwardingSucceded) { var str = String.Format("Forwarding failed: tried to forward message {0} for {1} times after {2} to invalid activation. Rejecting now.", message, message.ForwardCount, failedOperation); logger.Warn(ErrorCode.Messaging_Dispatcher_TryForwardFailed, str, exc); RejectMessage(message, Message.RejectionTypes.Transient, exc, str); } } }
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null) { bool forwardingSucceded = true; try { if (logger.IsEnabled(LogLevel.Information)) { logger.LogInformation( (int)ErrorCode.Messaging_Dispatcher_TryForward, "Trying to forward after {FailedOperation}, ForwardCount = {ForwardCount}. OldAddress = {OldAddress}, ForwardingAddress = {ForwardingAddress}, Message {Message}, Exception: {Exception}.", failedOperation, message.ForwardCount, oldAddress, forwardingAddress, message, exc); } // if this message is from a different cluster and hit a non-existing activation // in this cluster (which can happen due to stale cache or directory states) // we forward it back to the original silo it came from in the original cluster, // and target it to a fictional activation that is guaranteed to not exist. // This ensures that the GSI protocol creates a new instance there instead of here. if (forwardingAddress == null && message.TargetSilo != message.SendingSilo && !this.localGrainDirectory.IsSiloInCluster(message.SendingSilo)) { message.IsReturnedFromRemoteCluster = true; // marks message to force invalidation of stale directory entry forwardingAddress = ActivationAddress.NewActivationAddress(message.SendingSilo, message.TargetGrain); logger.Info(ErrorCode.Messaging_Dispatcher_ReturnToOriginCluster, $"Forwarding back to origin cluster, to fictional activation {message}"); } MessagingProcessingStatisticsGroup.OnDispatcherMessageReRouted(message); if (oldAddress != null) { message.AddToCacheInvalidationHeader(oldAddress); } forwardingSucceded = this.TryForwardMessage(message, forwardingAddress); } catch (Exception exc2) { forwardingSucceded = false; exc = exc2; } finally { if (!forwardingSucceded) { var str = $"Forwarding failed: tried to forward message {message} for {message.ForwardCount} times after {failedOperation} to invalid activation. Rejecting now."; logger.Warn(ErrorCode.Messaging_Dispatcher_TryForwardFailed, str, exc); RejectMessage(message, Message.RejectionTypes.Transient, exc, str); } } }
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null) { bool forwardingSucceded = false; try { this.messagingTrace.OnDispatcherForwarding(message, oldAddress, forwardingAddress, failedOperation, exc); if (oldAddress != null) { message.AddToCacheInvalidationHeader(oldAddress); } forwardingSucceded = this.TryForwardMessage(message, forwardingAddress); } catch (Exception exc2) { forwardingSucceded = false; exc = exc2; } finally { var sentRejection = false; // If the message was a one-way message, send a cache invalidation response even if the message was successfully forwarded. if (message.Direction == Message.Directions.OneWay) { this.RejectMessage( message, Message.RejectionTypes.CacheInvalidation, exc, "OneWay message sent to invalid activation"); sentRejection = true; } if (!forwardingSucceded) { this.messagingTrace.OnDispatcherForwardingFailed(message, oldAddress, forwardingAddress, failedOperation, exc); if (!sentRejection) { var str = $"Forwarding failed: tried to forward message {message} for {message.ForwardCount} times after {failedOperation} to invalid activation. Rejecting now."; RejectMessage(message, Message.RejectionTypes.Transient, exc, str); } } } }
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null) { bool forwardingSucceded = true; try { logger.Info(ErrorCode.Messaging_Dispatcher_TryForward, String.Format("Trying to forward after {0}, ForwardCount = {1}. Message {2}.", failedOperation, message.ForwardCount, message)); // if this message is from a different cluster and hit a non-existing activation // in this cluster (which can happen due to stale cache or directory states) // we forward it back to the original silo it came from in the original cluster, // and target it to a fictional activation that is guaranteed to not exist. // This ensures that the GSI protocol creates a new instance there instead of here. if (forwardingAddress == null && message.TargetSilo != message.SendingSilo && !Silo.CurrentSilo.LocalGrainDirectory.IsSiloInCluster(message.SendingSilo)) { message.IsReturnedFromRemoteCluster = true; // marks message to force invalidation of stale directory entry forwardingAddress = ActivationAddress.NewActivationAddress(message.SendingSilo, message.TargetGrain); logger.Info(ErrorCode.Messaging_Dispatcher_ReturnToOriginCluster, String.Format("Forwarding back to origin cluster, to fictional activation {0}", message)); } MessagingProcessingStatisticsGroup.OnDispatcherMessageReRouted(message); if (oldAddress != null) { message.AddToCacheInvalidationHeader(oldAddress); } forwardingSucceded = InsideRuntimeClient.Current.TryForwardMessage(message, forwardingAddress); } catch (Exception exc2) { forwardingSucceded = false; exc = exc2; } finally { if (!forwardingSucceded) { var str = String.Format("Forwarding failed: tried to forward message {0} for {1} times after {2} to invalid activation. Rejecting now.", message, message.ForwardCount, failedOperation); logger.Warn(ErrorCode.Messaging_Dispatcher_TryForwardFailed, str, exc); RejectMessage(message, Message.RejectionTypes.Transient, exc, str); } } }