public static PlacementResult IdentifySelection(ActivationAddress address)
{
return
new PlacementResult
{
Activation = address.Activation,
Silo = address.Silo
};
}
internal void ProcessRequestsToInvalidActivation(
List<Message> messages,
ActivationAddress oldAddress,
ActivationAddress forwardingAddress,
string failedOperation,
Exception exc = null)
{
// Just use this opportunity to invalidate local Cache Entry as well.
if (oldAddress != null)
{
Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(oldAddress);
}
logger.Info(ErrorCode.Messaging_Dispatcher_ForwardingRequests,
String.Format("Forwarding {0} requests to old address {1} after {2}.", messages.Count, oldAddress, failedOperation));
// IMPORTANT: do not do anything on activation context anymore, since this activation is invalid already.
Scheduler.QueueWorkItem(new ClosureWorkItem(
() =>
{
foreach (var message in messages)
{
TryForwardRequest(message, oldAddress, forwardingAddress, failedOperation, exc);
}
}
), catalog.SchedulingContext);
}
private void RerouteAllQueuedMessages(ActivationData activation, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
lock (activation)
{
List<Message> msgs = activation.DequeueAllWaitingMessages();
if (msgs == null || msgs.Count <= 0) return;
if (logger.IsVerbose) logger.Verbose(ErrorCode.Catalog_RerouteAllQueuedMessages, String.Format("RerouteAllQueuedMessages: {0} msgs from Invalid activation {1}.", msgs.Count(), activation));
dispatcher.ProcessRequestsToInvalidActivation(msgs, activation.Address, forwardingAddress, failedOperation, exc);
}
}
private async Task RegisterActivationInGrainDirectory(ActivationAddress address, bool singleActivationMode)
{
if (singleActivationMode)
{
ActivationAddress returnedAddress = await scheduler.RunOrQueueTask(() => directory.RegisterSingleActivationAsync(address), this.SchedulingContext);
if (address.Equals(returnedAddress)) return;
SiloAddress primaryDirectoryForGrain = directory.GetPrimaryForGrain(address.Grain);
var dae = new DuplicateActivationException
{
ActivationToUse = returnedAddress,
PrimaryDirectoryForGrain = primaryDirectoryForGrain
};
throw dae;
}
await scheduler.RunOrQueueTask(() => directory.RegisterAsync(address), this.SchedulingContext);
}
public DuplicateActivationException(ActivationAddress activationToUse, SiloAddress primaryDirectoryForGrain)
: base("DuplicateActivationException")
{
ActivationToUse = activationToUse;
PrimaryDirectoryForGrain = primaryDirectoryForGrain;
}
public NonExistentActivationException(string msg, ActivationAddress nonExistentActivation, bool isStatelessWorker)
: base(msg)
{
NonExistentActivation = nonExistentActivation;
IsStatelessWorker = isStatelessWorker;
}
internal void AddToCacheInvalidationHeader(ActivationAddress address)
{
var list = new List<ActivationAddress>();
if (CacheInvalidationHeader != null)
{
list.AddRange(CacheInvalidationHeader);
}
list.Add(address);
CacheInvalidationHeader = list;
}
public IGrainContext CreateContext(ActivationAddress address) => new StatelessWorkerGrainContext(address, _sharedContext, _innerActivator);
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
bool forwardingSucceded = true;
try
{
logger.Info(ErrorCode.Messaging_Dispatcher_TryForward,
String.Format("Trying to forward after {0}, ForwardCount = {1}. Message {2}.", failedOperation, message.ForwardCount, message));
MessagingProcessingStatisticsGroup.OnDispatcherMessageReRouted(message);
if (oldAddress != null)
{
message.AddToCacheInvalidationHeader(oldAddress);
}
forwardingSucceded = InsideRuntimeClient.Current.TryForwardMessage(message, forwardingAddress);
}
catch (Exception exc2)
{
forwardingSucceded = false;
exc = exc2;
}
finally
{
if (!forwardingSucceded)
{
var str = String.Format("Forwarding failed: tried to forward message {0} for {1} times after {2} to invalid activation. Rejecting now.",
message, message.ForwardCount, failedOperation);
logger.Warn(ErrorCode.Messaging_Dispatcher_TryForwardFailed, str, exc);
RejectMessage(message, Message.RejectionTypes.Transient, exc, str);
}
}
}
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
bool forwardingSucceded = true;
try
{
this.messagingTrace.OnDispatcherForwarding(message, oldAddress, forwardingAddress, failedOperation, exc);
// if this message is from a different cluster and hit a non-existing activation
// in this cluster (which can happen due to stale cache or directory states)
// we forward it back to the original silo it came from in the original cluster,
// and target it to a fictional activation that is guaranteed to not exist.
// This ensures that the GSI protocol creates a new instance there instead of here.
if (forwardingAddress == null &&
message.TargetSilo != message.SendingSilo &&
!this.localGrainDirectory.IsSiloInCluster(message.SendingSilo))
{
message.IsReturnedFromRemoteCluster = true; // marks message to force invalidation of stale directory entry
forwardingAddress = ActivationAddress.NewActivationAddress(message.SendingSilo, message.TargetGrain);
logger.Info(ErrorCode.Messaging_Dispatcher_ReturnToOriginCluster, $"Forwarding back to origin cluster, to fictional activation {message}");
}
if (oldAddress != null)
{
message.AddToCacheInvalidationHeader(oldAddress);
}
forwardingSucceded = this.TryForwardMessage(message, forwardingAddress);
}
catch (Exception exc2)
{
forwardingSucceded = false;
exc = exc2;
}
finally
{
var sentRejection = false;
// If the message was a one-way message, send a cache invalidation response even if the message was successfully forwarded.
if (message.Direction == Message.Directions.OneWay)
{
this.RejectMessage(
message,
Message.RejectionTypes.CacheInvalidation,
exc,
"OneWay message sent to invalid activation");
sentRejection = true;
}
if (!forwardingSucceded)
{
this.messagingTrace.OnDispatcherForwardingFailed(message, oldAddress, forwardingAddress, failedOperation, exc);
if (!sentRejection)
{
var str = $"Forwarding failed: tried to forward message {message} for {message.ForwardCount} times after {failedOperation} to invalid activation. Rejecting now.";
RejectMessage(message, Message.RejectionTypes.Transient, exc, str);
}
}
}
}
public bool Matches(ActivationAddress other)
{
return(Equals(Grain, other.Grain) && Equals(Activation, other.Activation));
}
private void ResendMessageImpl(Message message, ActivationAddress forwardingAddress = null)
{
if (logger.IsVerbose) logger.Verbose("Resend {0}", message);
message.TargetHistory = message.GetTargetHistory();
if (message.TargetGrain.IsSystemTarget)
{
dispatcher.SendSystemTargetMessage(message);
}
else if (forwardingAddress != null)
{
message.TargetAddress = forwardingAddress;
message.IsNewPlacement = false;
dispatcher.Transport.SendMessage(message);
}
else
{
message.TargetActivation = null;
message.TargetSilo = null;
message.ClearTargetAddress();
dispatcher.SendMessage(message);
}
}
/// <summary>
/// Receive a new message:
/// - validate order constraints, queue (or possibly redirect) if out of order
/// - validate transactions constraints
/// - invoke handler if ready, otherwise enqueue for later invocation
/// </summary>
/// <param name="message"></param>
public void ReceiveMessage(Message message)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message);
// Don't process messages that have already timed out
if (message.IsExpired)
{
logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired");
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch);
return;
}
// check if its targeted at a new activation
if (message.TargetGrain.IsSystemTarget)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target.");
throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
}
if (errorInjection && ShouldInjectError(message))
{
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection");
}
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection");
RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection");
return;
}
try
{
Task ignore;
ActivationData target = catalog.GetOrCreateActivation(
message.TargetAddress,
message.IsNewPlacement,
message.NewGrainType,
message.GenericGrainType,
out ignore);
if (ignore != null)
{
ignore.Ignore();
}
if (message.Direction == Message.Directions.Response)
{
ReceiveResponse(message, target);
}
else // Request or OneWay
{
if (SiloCanAcceptRequest(message))
{
ReceiveRequest(message, target);
}
else if (message.MayResend(config.Globals))
{
// Record that this message is no longer flowing through the system
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Redirecting");
throw new NotImplementedException("RedirectRequest() is believed to be no longer necessary; please contact the Orleans team if you see this error.");
}
else
{
// Record that this message is no longer flowing through the system
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Rejecting");
RejectMessage(message, Message.RejectionTypes.Transient, null, "Shutting down");
}
}
}
catch (Exception ex)
{
try
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation");
var nea = ex as Catalog.NonExistentActivationException;
if (nea == null)
{
var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message);
logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
throw new OrleansException(str, ex);
}
logger.Warn(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
String.Format("Intermediate warning for NonExistentActivation from Catalog.GetOrCreateActivation for message {0}", message), ex);
ActivationAddress nonExistentActivation = nea.NonExistentActivation;
if (message.Direction != Message.Directions.Response)
{
// Un-register the target activation so we don't keep getting spurious messages.
// The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
// this request snuck ahead of another request, with new placement requested, for the same activation.
// If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
// we want to make sure that we don't un-register the activation we just created.
// We would add a counter here, except that there's already a counter for this in the Catalog.
// Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero)
{
Scheduler.QueueWorkItem(new ClosureWorkItem(
// don't use message.TargetAddress, cause it may have been removed from the headers by this time!
async() =>
{
try
{
await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync(
nonExistentActivation);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
String.Format("Failed to un-register NonExistentActivation {0}",
nonExistentActivation), exc);
}
},
() => "LocalGrainDirectory.UnregisterConditionallyAsync"),
catalog.SchedulingContext);
}
ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
}
else
{
logger.Warn(ErrorCode.Dispatcher_NoTargetActivation,
"No target activation {0} for response message: {1}", nonExistentActivation, message);
Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
}
}
catch (Exception exc)
{
// Unable to create activation for this request - reject message
RejectMessage(message, Message.RejectionTypes.Transient, exc);
}
}
}
public ActivationAddress ToAddress(GrainId grainId)
{
return(ActivationAddress.GetAddress(Silo, grainId, Activation));
}
public ActivationData(ActivationAddress addr, string genericArguments, PlacementStrategy placedUsing, MultiClusterRegistrationStrategy registrationStrategy, IActivationCollector collector, TimeSpan ageLimit)
{
if (null == addr) throw new ArgumentNullException("addr");
if (null == placedUsing) throw new ArgumentNullException("placedUsing");
if (null == collector) throw new ArgumentNullException("collector");
logger = LogManager.GetLogger("ActivationData", LoggerType.Runtime);
ResetKeepAliveRequest();
Address = addr;
State = ActivationState.Create;
PlacedUsing = placedUsing;
RegistrationStrategy = registrationStrategy;
if (!Grain.IsSystemTarget && !Constants.IsSystemGrain(Grain))
{
this.collector = collector;
}
CollectionAgeLimit = ageLimit;
GrainReference = GrainReference.FromGrainId(addr.Grain, genericArguments,
Grain.IsSystemTarget ? addr.Silo : null);
}
internal bool TryForwardMessage(Message message, ActivationAddress forwardingAddress)
{
if (!message.MayForward(Config.Globals)) return false;
message.ForwardCount = message.ForwardCount + 1;
MessagingProcessingStatisticsGroup.OnIgcMessageForwared(message);
ResendMessageImpl(message, forwardingAddress);
return true;
}
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
bool forwardingSucceded = true;
try
{
logger.Info(ErrorCode.Messaging_Dispatcher_TryForward,
String.Format("Trying to forward after {0}, ForwardCount = {1}. Message {2}.", failedOperation, message.ForwardCount, message));
// if this message is from a different cluster and hit a non-existing activation
// in this cluster (which can happen due to stale cache or directory states)
// we forward it back to the original silo it came from in the original cluster,
// and target it to a fictional activation that is guaranteed to not exist.
// This ensures that the GSI protocol creates a new instance there instead of here.
if (forwardingAddress == null &&
message.TargetSilo != message.SendingSilo &&
!this.localGrainDirectory.IsSiloInCluster(message.SendingSilo))
{
message.IsReturnedFromRemoteCluster = true; // marks message to force invalidation of stale directory entry
forwardingAddress = ActivationAddress.NewActivationAddress(message.SendingSilo, message.TargetGrain);
logger.Info(ErrorCode.Messaging_Dispatcher_ReturnToOriginCluster,
String.Format("Forwarding back to origin cluster, to fictional activation {0}", message));
}
MessagingProcessingStatisticsGroup.OnDispatcherMessageReRouted(message);
if (oldAddress != null)
{
message.AddToCacheInvalidationHeader(oldAddress);
}
forwardingSucceded = this.TryForwardMessage(message, forwardingAddress);
}
catch (Exception exc2)
{
forwardingSucceded = false;
exc = exc2;
}
finally
{
if (!forwardingSucceded)
{
var str = String.Format("Forwarding failed: tried to forward message {0} for {1} times after {2} to invalid activation. Rejecting now.",
message, message.ForwardCount, failedOperation);
logger.Warn(ErrorCode.Messaging_Dispatcher_TryForwardFailed, str, exc);
RejectMessage(message, Message.RejectionTypes.Transient, exc, str);
}
}
}
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
bool forwardingSucceded = true;
try
{
logger.Info(ErrorCode.Messaging_Dispatcher_TryForward,
String.Format("Trying to forward after {0}, ForwardCount = {1}. Message {2}.", failedOperation, message.ForwardCount, message));
// if this message is from a different cluster and hit a non-existing activation
// in this cluster (which can happen due to stale cache or directory states)
// we forward it back to the original silo it came from in the original cluster,
// and target it to a fictional activation that is guaranteed to not exist.
// This ensures that the GSI protocol creates a new instance there instead of here.
if (forwardingAddress == null
&& message.TargetSilo != message.SendingSilo
&& !Silo.CurrentSilo.LocalGrainDirectory.IsSiloInCluster(message.SendingSilo))
{
message.IsReturnedFromRemoteCluster = true; // marks message to force invalidation of stale directory entry
forwardingAddress = ActivationAddress.NewActivationAddress(message.SendingSilo, message.TargetGrain);
logger.Info(ErrorCode.Messaging_Dispatcher_ReturnToOriginCluster,
String.Format("Forwarding back to origin cluster, to fictional activation {0}", message));
}
MessagingProcessingStatisticsGroup.OnDispatcherMessageReRouted(message);
if (oldAddress != null)
{
message.AddToCacheInvalidationHeader(oldAddress);
}
forwardingSucceded = InsideRuntimeClient.Current.TryForwardMessage(message, forwardingAddress);
}
catch (Exception exc2)
{
forwardingSucceded = false;
exc = exc2;
}
finally
{
if (!forwardingSucceded)
{
var str = String.Format("Forwarding failed: tried to forward message {0} for {1} times after {2} to invalid activation. Rejecting now.",
message, message.ForwardCount, failedOperation);
logger.Warn(ErrorCode.Messaging_Dispatcher_TryForwardFailed, str, exc);
RejectMessage(message, Message.RejectionTypes.Transient, exc, str);
}
}
}
/// <summary>
/// Receive a new message:
/// - validate order constraints, queue (or possibly redirect) if out of order
/// - validate transactions constraints
/// - invoke handler if ready, otherwise enqueue for later invocation
/// </summary>
/// <param name="message"></param>
public void ReceiveMessage(Message message)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message);
// Don't process messages that have already timed out
if (message.IsExpired)
{
logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired");
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch);
return;
}
// check if its targeted at a new activation
if (message.TargetGrain.IsSystemTarget)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target.");
throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
}
try
{
Task ignore;
ActivationData target = catalog.GetOrCreateActivation(
message.TargetAddress,
message.IsNewPlacement,
message.NewGrainType,
String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType,
message.RequestContextData,
out ignore);
if (ignore != null)
{
ignore.Ignore();
}
if (message.Direction == Message.Directions.Response)
{
ReceiveResponse(message, target);
}
else // Request or OneWay
{
if (target.State == ActivationState.Valid)
{
this.activationCollector.TryRescheduleCollection(target);
}
// Silo is always capable to accept a new request. It's up to the activation to handle its internal state.
// If activation is shutting down, it will queue and later forward this request.
ReceiveRequest(message, target);
}
}
catch (Exception ex)
{
try
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation");
var nea = ex as Catalog.NonExistentActivationException;
if (nea == null)
{
var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message);
logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
throw new OrleansException(str, ex);
}
if (nea.IsStatelessWorker)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
String.Format("Intermediate StatelessWorker NonExistentActivation for message {0}", message), ex);
}
}
else
{
logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
String.Format("Intermediate NonExistentActivation for message {0}", message), ex);
}
ActivationAddress nonExistentActivation = nea.NonExistentActivation;
if (message.Direction != Message.Directions.Response)
{
// Un-register the target activation so we don't keep getting spurious messages.
// The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
// this request snuck ahead of another request, with new placement requested, for the same activation.
// If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
// we want to make sure that we don't un-register the activation we just created.
// We would add a counter here, except that there's already a counter for this in the Catalog.
// Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
var origin = message.SendingSilo;
scheduler.QueueWorkItem(new ClosureWorkItem(
// don't use message.TargetAddress, cause it may have been removed from the headers by this time!
async() =>
{
try
{
await this.localGrainDirectory.UnregisterAfterNonexistingActivation(
nonExistentActivation, origin);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
String.Format("Failed to un-register NonExistentActivation {0}",
nonExistentActivation), exc);
}
},
"LocalGrainDirectory.UnregisterAfterNonexistingActivation"),
catalog.SchedulingContext);
ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
}
else
{
logger.Warn(
ErrorCode.Dispatcher_NoTargetActivation,
nonExistentActivation.Silo.IsClient
? "No target client {0} for response message: {1}. It's likely that the client recently disconnected."
: "No target activation {0} for response message: {1}",
nonExistentActivation,
message);
this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
}
}
catch (Exception exc)
{
// Unable to create activation for this request - reject message
RejectMessage(message, Message.RejectionTypes.Transient, exc);
}
}
}
public void ClearTargetAddress()
{
targetAddress = null;
}
private async Task AddressMessageAsync(Message message, PlacementTarget target, PlacementStrategy strategy, ActivationAddress targetAddress)
{
var placementResult = await placementDirectorsManager.SelectOrAddActivation(
message.SendingAddress, target, this.catalog, strategy);
SetMessageTargetPlacement(message, placementResult, targetAddress);
}
public DuplicateActivationException(ActivationAddress activationToUse)
: base("DuplicateActivationException")
{
ActivationToUse = activationToUse;
}
private void SetMessageTargetPlacement(Message message, PlacementResult placementResult, ActivationAddress targetAddress)
{
if (placementResult.IsNewPlacement && targetAddress.Grain.IsClient)
{
logger.Error(ErrorCode.Dispatcher_AddressMsg_UnregisteredClient, $"AddressMessage could not find target for client pseudo-grain {message}");
throw new KeyNotFoundException($"Attempting to send a message {message} to an unregistered client pseudo-grain {targetAddress.Grain}");
}
message.SetTargetPlacement(placementResult);
if (placementResult.IsNewPlacement)
{
CounterStatistic.FindOrCreate(StatisticNames.DISPATCHER_NEW_PLACEMENT).Increment();
}
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace(ErrorCode.Dispatcher_AddressMsg_SelectTarget, "AddressMessage Placement SelectTarget {0}", message);
}
}
/// <summary> Write a <c>ActivationAddress</c> value to the stream. </summary>
internal void Write(ActivationAddress addr)
{
Write(addr.Silo ?? SiloAddress.Zero);
// GrainId must not be null
Write(addr.Grain);
Write(addr.Activation ?? ActivationId.Zero);
}
internal void OnDispatcherForwarding(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exception)
{
if (this.IsEnabled(DispatcherForwardingEventName))
{
this.Write(DispatcherForwardingEventName, new { Message = message, OldAddress = oldAddress, ForwardingAddress = forwardingAddress, FailedOperation = failedOperation, Exception = exception });
}
if (this.IsEnabled(LogLevel.Information))
{
LogDispatcherForwarding(this, message, oldAddress, forwardingAddress, failedOperation, message.ForwardCount, exception);
}
MessagingProcessingStatisticsGroup.OnDispatcherMessageForwared(message);
}
/// <summary>
/// If activation already exists, use it
/// Otherwise, create an activation of an existing grain by reading its state.
/// Return immediately using a dummy that will queue messages.
/// Concurrently start creating and initializing the real activation and replace it when it is ready.
/// </summary>
/// <param name="address">Grain's activation address</param>
/// <param name="newPlacement">Creation of new activation was requested by the placement director.</param>
/// <param name="grainType">The type of grain to be activated or created</param>
/// <param name="genericArguments">Specific generic type of grain to be activated or created</param>
/// <param name="activatedPromise"></param>
/// <returns></returns>
public ActivationData GetOrCreateActivation(
ActivationAddress address,
bool newPlacement,
string grainType,
string genericArguments,
out Task activatedPromise)
{
ActivationData result;
activatedPromise = TaskDone.Done;
lock (activations)
{
if (TryGetActivationData(address.Activation, out result))
{
ActivationCollector.TryRescheduleCollection(result);
return result;
}
if (newPlacement && !SiloStatusOracle.CurrentStatus.IsTerminating())
{
// create a dummy activation that will queue up messages until the real data arrives
PlacementStrategy placement;
int typeCode = address.Grain.GetTypeCode();
string actualGrainType = null;
if (typeCode != 0) // special case for Membership grain.
GetGrainTypeInfo(typeCode, out actualGrainType, out placement);
else
placement = SystemPlacement.Singleton;
if (string.IsNullOrEmpty(grainType))
{
grainType = actualGrainType;
}
// We want to do this (RegisterMessageTarget) under the same lock that we tested TryGetActivationData. They both access ActivationDirectory.
result = new ActivationData(
address,
genericArguments,
placement,
ActivationCollector,
config.Application.GetCollectionAgeLimit(grainType));
RegisterMessageTarget(result);
}
} // End lock
// Did not find and did not start placing new
if (result == null)
{
var msg = String.Format("Non-existent activation: {0}, grain type: {1}.",
address.ToFullString(), grainType);
if (logger.IsVerbose) logger.Verbose(ErrorCode.CatalogNonExistingActivation2, msg);
CounterStatistic.FindOrCreate(StatisticNames.CATALOG_ACTIVATION_NON_EXISTENT_ACTIVATIONS).Increment();
throw new NonExistentActivationException(msg) { NonExistentActivation = address };
}
SetupActivationInstance(result, grainType, genericArguments);
activatedPromise = InitActivation(result, grainType, genericArguments);
return result;
}
internal void OnDispatcherForwardingFailed(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exception)
{
if (this.IsEnabled(DispatcherForwardingFailedEventName))
{
this.Write(DispatcherForwardingFailedEventName, new { Message = message, OldAddress = oldAddress, ForwardingAddress = forwardingAddress, FailedOperation = failedOperation, Exception = exception });
}
LogDispatcherForwardingFailed(this, message, oldAddress, forwardingAddress, failedOperation, message.ForwardCount, exception);
}
internal void ProcessRequestToInvalidActivation(
Message message,
ActivationAddress oldAddress,
ActivationAddress forwardingAddress,
string failedOperation,
Exception exc = null)
{
// Just use this opportunity to invalidate local Cache Entry as well.
if (oldAddress != null)
{
Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(oldAddress);
}
// IMPORTANT: do not do anything on activation context anymore, since this activation is invalid already.
Scheduler.QueueWorkItem(new ClosureWorkItem(
() => TryForwardRequest(message, oldAddress, forwardingAddress, failedOperation, exc)),
catalog.SchedulingContext);
}
internal void OnDispatcherForwardingMultiple(int messageCount, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exception)
{
if (this.IsEnabled(DispatcherForwardingMultipleEventName))
{
this.Write(DispatcherForwardingMultipleEventName, new { MessageCount = messageCount, OldAddress = oldAddress, ForwardingAddress = forwardingAddress, FailedOperation = failedOperation, Exception = exception });
}
if (this.IsEnabled(LogLevel.Information))
{
LogDispatcherForwardingMultiple(this, messageCount, oldAddress, forwardingAddress, failedOperation, exception);
}
}
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
bool forwardingSucceded = true;
try
{
logger.Info(ErrorCode.Messaging_Dispatcher_TryForward,
String.Format("Trying to forward after {0}, ForwardCount = {1}. Message {2}.", failedOperation, message.ForwardCount, message));
MessagingProcessingStatisticsGroup.OnDispatcherMessageReRouted(message);
if (oldAddress != null)
{
message.AddToCacheInvalidationHeader(oldAddress);
}
forwardingSucceded = InsideRuntimeClient.Current.TryForwardMessage(message, forwardingAddress);
}
catch (Exception exc2)
{
forwardingSucceded = false;
exc = exc2;
}
finally
{
if (!forwardingSucceded)
{
var str = String.Format("Forwarding failed: tried to forward message {0} for {1} times after {2} to invalid activation. Rejecting now.",
message, message.ForwardCount, failedOperation);
logger.Warn(ErrorCode.Messaging_Dispatcher_TryForwardFailed, str, exc);
RejectMessage(message, Message.RejectionTypes.Transient, exc, str);
}
}
}
internal void TryForwardRequest(Message message, ActivationAddress oldAddress, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
bool forwardingSucceded = false;
try
{
this.messagingTrace.OnDispatcherForwarding(message, oldAddress, forwardingAddress, failedOperation, exc);
if (oldAddress != null)
{
message.AddToCacheInvalidationHeader(oldAddress);
}
forwardingSucceded = this.TryForwardMessage(message, forwardingAddress);
}
catch (Exception exc2)
{
forwardingSucceded = false;
exc = exc2;
}
finally
{
var sentRejection = false;
// If the message was a one-way message, send a cache invalidation response even if the message was successfully forwarded.
if (message.Direction == Message.Directions.OneWay)
{
this.RejectMessage(
message,
Message.RejectionTypes.CacheInvalidation,
exc,
"OneWay message sent to invalid activation");
sentRejection = true;
}
if (!forwardingSucceded)
{
this.messagingTrace.OnDispatcherForwardingFailed(message, oldAddress, forwardingAddress, failedOperation, exc);
if (!sentRejection)
{
var str = $"Forwarding failed: tried to forward message {message} for {message.ForwardCount} times after {failedOperation} to invalid activation. Rejecting now.";
RejectMessage(message, Message.RejectionTypes.Transient, exc, str);
}
}
}
}
public void ClearTargetAddress()
{
targetAddress = null;
}
public bool Matches(ActivationAddress other)
{
return Equals(Grain, other.Grain) && Equals(Activation, other.Activation);
}
private void ResendMessageImpl(Message message, ActivationAddress forwardingAddress = null)
{
if (logger.IsVerbose) logger.Verbose("Resend {0}", message);
message.SetMetadata(Message.Metadata.TARGET_HISTORY, message.GetTargetHistory());
if (message.TargetGrain.IsSystemTarget)
{
dispatcher.SendSystemTargetMessage(message);
}
else if (forwardingAddress != null)
{
message.TargetAddress = forwardingAddress;
message.RemoveHeader(Message.Header.IS_NEW_PLACEMENT);
dispatcher.Transport.SendMessage(message);
}
else
{
message.RemoveHeader(Message.Header.TARGET_ACTIVATION);
message.RemoveHeader(Message.Header.TARGET_SILO);
dispatcher.SendMessage(message);
}
}
private ActivationAddress GetClientActivationAddress(GrainId clientId)
{
return(ActivationAddress.GetAddress(myAddress, clientId, ActivationId.GetActivationId(clientId)));
}