internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
bool forwardingSucceded = true;
try
{
this.messagingTrace.OnDispatcherForwarding(message, oldAddress, forwardingAddress, failedOperation, exc);
// if this message is from a different cluster and hit a non-existing activation
// in this cluster (which can happen due to stale cache or directory states)
// we forward it back to the original silo it came from in the original cluster,
// and target it to a fictional activation that is guaranteed to not exist.
// This ensures that the GSI protocol creates a new instance there instead of here.
if (forwardingAddress == null &&
message.TargetSilo != message.SendingSilo &&
!this.localGrainDirectory.IsSiloInCluster(message.SendingSilo))
{
message.IsReturnedFromRemoteCluster = true; // marks message to force invalidation of stale directory entry
forwardingAddress = ActivationAddress.NewActivationAddress(message.SendingSilo, message.TargetGrain);
logger.Info(ErrorCode.Messaging_Dispatcher_ReturnToOriginCluster, $"Forwarding back to origin cluster, to fictional activation {message}");
}
if (oldAddress != null)
{
message.AddToCacheInvalidationHeader(oldAddress);
}
forwardingSucceded = this.TryForwardMessage(message, forwardingAddress);
}
catch (Exception exc2)
{
forwardingSucceded = false;
exc = exc2;
}
finally
{
var sentRejection = false;
// If the message was a one-way message, send a cache invalidation response even if the message was successfully forwarded.
if (message.Direction == Message.Directions.OneWay)
{
this.RejectMessage(
message,
Message.RejectionTypes.CacheInvalidation,
exc,
"OneWay message sent to invalid activation");
sentRejection = true;
}
if (!forwardingSucceded)
{
this.messagingTrace.OnDispatcherForwardingFailed(message, oldAddress, forwardingAddress, failedOperation, exc);
if (!sentRejection)
{
var str = $"Forwarding failed: tried to forward message {message} for {message.ForwardCount} times after {failedOperation} to invalid activation. Rejecting now.";
RejectMessage(message, Message.RejectionTypes.Transient, exc, str);
}
}
}
}
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
bool forwardingSucceded = true;
try
{
logger.Info(ErrorCode.Messaging_Dispatcher_TryForward,
String.Format("Trying to forward after {0}, ForwardCount = {1}. Message {2}.", failedOperation, message.ForwardCount, message));
MessagingProcessingStatisticsGroup.OnDispatcherMessageReRouted(message);
if (oldAddress != null)
{
message.AddToCacheInvalidationHeader(oldAddress);
}
forwardingSucceded = InsideRuntimeClient.Current.TryForwardMessage(message, forwardingAddress);
}
catch (Exception exc2)
{
forwardingSucceded = false;
exc = exc2;
}
finally
{
if (!forwardingSucceded)
{
var str = String.Format("Forwarding failed: tried to forward message {0} for {1} times after {2} to invalid activation. Rejecting now.",
message, message.ForwardCount, failedOperation);
logger.Warn(ErrorCode.Messaging_Dispatcher_TryForwardFailed, str, exc);
RejectMessage(message, Message.RejectionTypes.Transient, exc, str);
}
}
}
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
bool forwardingSucceded = true;
try
{
if (logger.IsEnabled(LogLevel.Information))
{
logger.LogInformation(
(int)ErrorCode.Messaging_Dispatcher_TryForward,
"Trying to forward after {FailedOperation}, ForwardCount = {ForwardCount}. OldAddress = {OldAddress}, ForwardingAddress = {ForwardingAddress}, Message {Message}, Exception: {Exception}.",
failedOperation,
message.ForwardCount,
oldAddress,
forwardingAddress,
message,
exc);
}
// if this message is from a different cluster and hit a non-existing activation
// in this cluster (which can happen due to stale cache or directory states)
// we forward it back to the original silo it came from in the original cluster,
// and target it to a fictional activation that is guaranteed to not exist.
// This ensures that the GSI protocol creates a new instance there instead of here.
if (forwardingAddress == null &&
message.TargetSilo != message.SendingSilo &&
!this.localGrainDirectory.IsSiloInCluster(message.SendingSilo))
{
message.IsReturnedFromRemoteCluster = true; // marks message to force invalidation of stale directory entry
forwardingAddress = ActivationAddress.NewActivationAddress(message.SendingSilo, message.TargetGrain);
logger.Info(ErrorCode.Messaging_Dispatcher_ReturnToOriginCluster, $"Forwarding back to origin cluster, to fictional activation {message}");
}
MessagingProcessingStatisticsGroup.OnDispatcherMessageReRouted(message);
if (oldAddress != null)
{
message.AddToCacheInvalidationHeader(oldAddress);
}
forwardingSucceded = this.TryForwardMessage(message, forwardingAddress);
}
catch (Exception exc2)
{
forwardingSucceded = false;
exc = exc2;
}
finally
{
if (!forwardingSucceded)
{
var str = $"Forwarding failed: tried to forward message {message} for {message.ForwardCount} times after {failedOperation} to invalid activation. Rejecting now.";
logger.Warn(ErrorCode.Messaging_Dispatcher_TryForwardFailed, str, exc);
RejectMessage(message, Message.RejectionTypes.Transient, exc, str);
}
}
}
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
bool forwardingSucceded = false;
try
{
this.messagingTrace.OnDispatcherForwarding(message, oldAddress, forwardingAddress, failedOperation, exc);
if (oldAddress != null)
{
message.AddToCacheInvalidationHeader(oldAddress);
}
forwardingSucceded = this.TryForwardMessage(message, forwardingAddress);
}
catch (Exception exc2)
{
forwardingSucceded = false;
exc = exc2;
}
finally
{
var sentRejection = false;
// If the message was a one-way message, send a cache invalidation response even if the message was successfully forwarded.
if (message.Direction == Message.Directions.OneWay)
{
this.RejectMessage(
message,
Message.RejectionTypes.CacheInvalidation,
exc,
"OneWay message sent to invalid activation");
sentRejection = true;
}
if (!forwardingSucceded)
{
this.messagingTrace.OnDispatcherForwardingFailed(message, oldAddress, forwardingAddress, failedOperation, exc);
if (!sentRejection)
{
var str = $"Forwarding failed: tried to forward message {message} for {message.ForwardCount} times after {failedOperation} to invalid activation. Rejecting now.";
RejectMessage(message, Message.RejectionTypes.Transient, exc, str);
}
}
}
}
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
bool forwardingSucceded = true;
try
{
logger.Info(ErrorCode.Messaging_Dispatcher_TryForward,
String.Format("Trying to forward after {0}, ForwardCount = {1}. Message {2}.", failedOperation, message.ForwardCount, message));
MessagingProcessingStatisticsGroup.OnDispatcherMessageReRouted(message);
if (oldAddress != null)
{
message.AddToCacheInvalidationHeader(oldAddress);
}
forwardingSucceded = InsideRuntimeClient.Current.TryForwardMessage(message, forwardingAddress);
}
catch (Exception exc2)
{
forwardingSucceded = false;
exc = exc2;
}
finally
{
if (!forwardingSucceded)
{
var str = String.Format("Forwarding failed: tried to forward message {0} for {1} times after {2} to invalid activation. Rejecting now.",
message, message.ForwardCount, failedOperation);
logger.Warn(ErrorCode.Messaging_Dispatcher_TryForwardFailed, str, exc);
RejectMessage(message, Message.RejectionTypes.Transient, exc, str);
}
}
}
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
bool forwardingSucceded = true;
try
{
logger.Info(ErrorCode.Messaging_Dispatcher_TryForward,
String.Format("Trying to forward after {0}, ForwardCount = {1}. Message {2}.", failedOperation, message.ForwardCount, message));
// if this message is from a different cluster and hit a non-existing activation
// in this cluster (which can happen due to stale cache or directory states)
// we forward it back to the original silo it came from in the original cluster,
// and target it to a fictional activation that is guaranteed to not exist.
// This ensures that the GSI protocol creates a new instance there instead of here.
if (forwardingAddress == null
&& message.TargetSilo != message.SendingSilo
&& !Silo.CurrentSilo.LocalGrainDirectory.IsSiloInCluster(message.SendingSilo))
{
message.IsReturnedFromRemoteCluster = true; // marks message to force invalidation of stale directory entry
forwardingAddress = ActivationAddress.NewActivationAddress(message.SendingSilo, message.TargetGrain);
logger.Info(ErrorCode.Messaging_Dispatcher_ReturnToOriginCluster,
String.Format("Forwarding back to origin cluster, to fictional activation {0}", message));
}
MessagingProcessingStatisticsGroup.OnDispatcherMessageReRouted(message);
if (oldAddress != null)
{
message.AddToCacheInvalidationHeader(oldAddress);
}
forwardingSucceded = InsideRuntimeClient.Current.TryForwardMessage(message, forwardingAddress);
}
catch (Exception exc2)
{
forwardingSucceded = false;
exc = exc2;
}
finally
{
if (!forwardingSucceded)
{
var str = String.Format("Forwarding failed: tried to forward message {0} for {1} times after {2} to invalid activation. Rejecting now.",
message, message.ForwardCount, failedOperation);
logger.Warn(ErrorCode.Messaging_Dispatcher_TryForwardFailed, str, exc);
RejectMessage(message, Message.RejectionTypes.Transient, exc, str);
}
}
}
