private GrainTimer(Func<object, Task> asyncCallback, object state, TimeSpan dueTime, TimeSpan period, string name)
{
var ctxt = RuntimeContext.Current.ActivationContext;
activationData = (ActivationData) RuntimeClient.Current.CurrentActivationData;
this.Name = name;
this.asyncCallback = asyncCallback;
timer = new AsyncTaskSafeTimer(
stateObj => TimerTick(stateObj, ctxt),
state);
this.dueTime = dueTime;
timerFrequency = period;
previousTickTime = DateTime.UtcNow;
totalNumTicks = 0;
}
/// <summary>
/// Handle an incoming message and queue/invoke appropriate handler
/// </summary>
/// <param name="message"></param>
/// <param name="targetActivation"></param>
public void HandleIncomingRequest(Message message, ActivationData targetActivation)
{
lock (targetActivation)
{
if (targetActivation.State == ActivationState.Invalid)
{
ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "HandleIncomingRequest");
return;
}
// Now we can actually scheduler processing of this request
targetActivation.RecordRunning(message);
var context = new SchedulingContext(targetActivation);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedOk(message);
Scheduler.QueueWorkItem(new InvokeWorkItem(targetActivation, message, context), context);
}
}
/// <summary>
/// Determine if the activation is able to currently accept the given message
/// - always accept responses
/// For other messages, require that:
/// - activation is properly initialized
/// - the message would not cause a reentrancy conflict
/// </summary>
/// <param name="targetActivation"></param>
/// <param name="incoming"></param>
/// <returns></returns>
private bool ActivationMayAcceptRequest(ActivationData targetActivation, Message incoming)
{
if (!targetActivation.State.Equals(ActivationState.Valid)) return false;
if (!targetActivation.IsCurrentlyExecuting) return true;
return CanInterleave(targetActivation, incoming);
}
private void ReceiveResponse(Message message, ActivationData targetActivation)
{
lock (targetActivation)
{
if (targetActivation.State == ActivationState.Invalid)
{
logger.Warn(ErrorCode.Dispatcher_Receive_InvalidActivation,
"Response received for invalid activation {0}", message);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Ivalid");
return;
}
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedOk(message);
if (Transport.TryDeliverToProxy(message)) return;
RuntimeClient.Current.ReceiveResponse(message);
}
}
private async Task<ActivationData> CallGrainDeactivateAndCleanupStreams(ActivationData activation)
{
try
{
var grainTypeName = activation.GrainInstanceType.FullName;
// Note: This call is being made from within Scheduler.Queue wrapper, so we are already executing on worker thread
if (logger.IsVerbose) logger.Verbose(ErrorCode.Catalog_BeforeCallingDeactivate, "About to call {1} grain's OnDeactivateAsync() method {0}", activation, grainTypeName);
// Call OnDeactivateAsync inline, but within try-catch wrapper to safely capture any exceptions thrown from called function
try
{
// just check in case this activation data is already Invalid or not here at all.
ActivationData ignore;
if (TryGetActivationData(activation.ActivationId, out ignore) &&
activation.State == ActivationState.Deactivating)
{
await activation.GrainInstance.OnDeactivateAsync();
}
if (logger.IsVerbose) logger.Verbose(ErrorCode.Catalog_AfterCallingDeactivate, "Returned from calling {1} grain's OnDeactivateAsync() method {0}", activation, grainTypeName);
}
catch (Exception exc)
{
logger.Error(ErrorCode.Catalog_ErrorCallingDeactivate,
string.Format("Error calling grain's OnDeactivateAsync() method - Grain type = {1} Activation = {0}", activation, grainTypeName), exc);
}
if (activation.IsUsingStreams)
{
try
{
await activation.DeactivateStreamResources();
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Catalog_DeactivateStreamResources_Exception, String.Format("DeactivateStreamResources Grain type = {0} Activation = {1} failed.", grainTypeName, activation), exc);
}
}
}
catch(Exception exc)
{
logger.Error(ErrorCode.Catalog_FinishGrainDeactivateAndCleanupStreams_Exception, String.Format("CallGrainDeactivateAndCleanupStreams Activation = {0} failed.", activation), exc);
}
return activation;
}
private void RerouteAllQueuedMessages(ActivationData activation, ActivationAddress forwardingAddress, string failedOperation, Exception exc = null)
{
lock (activation)
{
List<Message> msgs = activation.DequeueAllWaitingMessages();
if (msgs == null || msgs.Count <= 0) return;
if (logger.IsVerbose) logger.Verbose(ErrorCode.Catalog_RerouteAllQueuedMessages, String.Format("RerouteAllQueuedMessages: {0} msgs from Invalid activation {1}.", msgs.Count(), activation));
dispatcher.ProcessRequestsToInvalidActivation(msgs, activation.Address, forwardingAddress, failedOperation, exc);
}
}
/// <summary>
/// Deletes activation immediately regardless of active transactions etc.
/// For use by grain delete, transaction abort, etc.
/// </summary>
/// <param name="activation"></param>
private void DestroyActivationVoid(ActivationData activation)
{
StartDestroyActivations(new List<ActivationData> { activation });
}
internal Silo(string name, SiloType siloType, ClusterConfiguration config, ILocalDataStore keyStore)
{
SystemStatus.Current = SystemStatus.Creating;
CurrentSilo = this;
var startTime = DateTime.UtcNow;
this.siloType = siloType;
Name = name;
siloTerminatedEvent = new ManualResetEvent(false);
OrleansConfig = config;
globalConfig = config.Globals;
config.OnConfigChange("Defaults", () => nodeConfig = config.GetConfigurationForNode(name));
if (!TraceLogger.IsInitialized)
{
TraceLogger.Initialize(nodeConfig);
}
config.OnConfigChange("Defaults/Tracing", () => TraceLogger.Initialize(nodeConfig, true), false);
LimitManager.Initialize(nodeConfig);
ActivationData.Init(config);
StatisticsCollector.Initialize(nodeConfig);
SerializationManager.Initialize(globalConfig.UseStandardSerializer);
initTimeout = globalConfig.MaxJoinAttemptTime;
if (Debugger.IsAttached)
{
initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), globalConfig.MaxJoinAttemptTime);
stopTimeout = initTimeout;
}
IPEndPoint here = nodeConfig.Endpoint;
int generation = nodeConfig.Generation;
if (generation == 0)
{
generation = SiloAddress.AllocateNewGeneration();
nodeConfig.Generation = generation;
}
TraceLogger.MyIPEndPoint = here;
logger = TraceLogger.GetLogger("Silo", TraceLogger.LoggerType.Runtime);
logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on {1} at {2}, gen {3} --------------", siloType, nodeConfig.DNSHostName, here, generation);
logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with runtime Version='{1}' Config= " + Environment.NewLine + "{2}", name, RuntimeVersion.Current, config.ToString(name));
if (keyStore != null)
{
// Re-establish reference to shared local key store in this app domain
LocalDataStoreInstance.LocalDataStore = keyStore;
}
healthCheckParticipants = new List <IHealthCheckParticipant>();
BufferPool.InitGlobalBufferPool(globalConfig);
PlacementStrategy.Initialize(globalConfig);
UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnobservedExceptionHandler);
AppDomain.CurrentDomain.UnhandledException +=
(obj, ev) => DomainUnobservedExceptionHandler(obj, (Exception)ev.ExceptionObject);
typeManager = new GrainTypeManager(here.Address.Equals(IPAddress.Loopback));
// Performance metrics
siloStatistics = new SiloStatisticsManager(globalConfig, nodeConfig);
config.OnConfigChange("Defaults/LoadShedding", () => siloStatistics.MetricsTable.NodeConfig = nodeConfig, false);
// The scheduler
scheduler = new OrleansTaskScheduler(globalConfig, nodeConfig);
healthCheckParticipants.Add(scheduler);
// Initialize the message center
var mc = new MessageCenter(here, generation, globalConfig, siloStatistics.MetricsTable);
if (nodeConfig.IsGatewayNode)
{
mc.InstallGateway(nodeConfig.ProxyGatewayEndpoint);
}
messageCenter = mc;
// Now the router/directory service
// This has to come after the message center //; note that it then gets injected back into the message center.;
localGrainDirectory = new LocalGrainDirectory(this);
// Now the activation directory.
// This needs to know which router to use so that it can keep the global directory in synch with the local one.
activationDirectory = new ActivationDirectory();
// Now the consistent ring provider
RingProvider = GlobalConfig.UseVirtualBucketsConsistentRing ?
(IConsistentRingProvider) new VirtualBucketsRingProvider(SiloAddress, GlobalConfig.NumVirtualBucketsConsistentRing)
: new ConsistentRingProvider(SiloAddress);
Action <Dispatcher> setDispatcher;
catalog = new Catalog(Constants.CatalogId, SiloAddress, Name, LocalGrainDirectory, typeManager, scheduler, activationDirectory, config, out setDispatcher);
var dispatcher = new Dispatcher(scheduler, messageCenter, catalog, config);
setDispatcher(dispatcher);
RuntimeClient.Current = new InsideRuntimeClient(
dispatcher,
catalog,
LocalGrainDirectory,
SiloAddress,
config,
RingProvider,
typeManager);
messageCenter.RerouteHandler = InsideRuntimeClient.Current.RerouteMessage;
messageCenter.SniffIncomingMessage = InsideRuntimeClient.Current.SniffIncomingMessage;
siloStatistics.MetricsTable.Scheduler = scheduler;
siloStatistics.MetricsTable.ActivationDirectory = activationDirectory;
siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector;
siloStatistics.MetricsTable.MessageCenter = messageCenter;
DeploymentLoadPublisher.CreateDeploymentLoadPublisher(this, globalConfig);
PlacementDirectorsManager.CreatePlacementDirectorsManager(globalConfig);
// Now the incoming message agents
incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, dispatcher);
incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, dispatcher);
incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, dispatcher);
membershipFactory = new MembershipFactory();
reminderFactory = new LocalReminderServiceFactory();
SystemStatus.Current = SystemStatus.Created;
StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
() => TraceLogger.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.
TestHookup = new TestHookups(this);
logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
}
private async Task RegisterActivationInGrainDirectoryAndValidate(ActivationData activation)
{
ActivationAddress address = activation.Address;
bool singleActivationMode = !activation.IsStatelessWorker;
if (singleActivationMode)
{
ActivationAddress returnedAddress = await scheduler.RunOrQueueTask(() => directory.RegisterSingleActivationAsync(address), this.SchedulingContext);
if (address.Equals(returnedAddress)) return;
SiloAddress primaryDirectoryForGrain = directory.GetPrimaryForGrain(address.Grain);
var dae = new DuplicateActivationException
{
ActivationToUse = returnedAddress,
PrimaryDirectoryForGrain = primaryDirectoryForGrain
};
throw dae;
}
else
{
StatelessWorkerPlacement stPlacement = activation.PlacedUsing as StatelessWorkerPlacement;
int maxNumLocalActivations = stPlacement.MaxLocal;
lock (activations)
{
List<ActivationData> local;
if (!LocalLookup(address.Grain, out local) || local.Count <= maxNumLocalActivations)
return;
var id = StatelessWorkerDirector.PickRandom(local).Address;
var dae = new DuplicateActivationException
{
ActivationToUse = id,
};
throw dae;
}
}
// We currently don't have any other case for multiple activations except for StatelessWorker.
//await scheduler.RunOrQueueTask(() => directory.RegisterAsync(address), this.SchedulingContext);
}
private void DeactivateActivationImpl(ActivationData data, StatisticName statisticName)
{
bool promptly = false;
bool alreadBeingDestroyed = false;
lock (data)
{
if (data.State == ActivationState.Valid)
{
// Change the ActivationData state here, since we're about to give up the lock.
data.PrepareForDeactivation(); // Don't accept any new messages
ActivationCollector.TryCancelCollection(data);
if (!data.IsCurrentlyExecuting)
{
promptly = true;
}
else // busy, so destroy later.
{
data.AddOnInactive(() => DestroyActivationVoid(data));
}
}
else if (data.State == ActivationState.Create)
{
throw new InvalidOperationException(String.Format(
"Activation {0} has called DeactivateOnIdle from within a constructor, which is not allowed.",
data.ToString()));
}
else if (data.State == ActivationState.Activating)
{
throw new InvalidOperationException(String.Format(
"Activation {0} has called DeactivateOnIdle from within OnActivateAsync, which is not allowed.",
data.ToString()));
}
else
{
alreadBeingDestroyed = true;
}
}
logger.Info(ErrorCode.Catalog_ShutdownActivations_2,
"DeactivateActivationOnIdle: {0} {1}.", data.ToString(), promptly ? "promptly" : (alreadBeingDestroyed ? "already being destroyed or invalid" : "later when become idle"));
CounterStatistic.FindOrCreate(statisticName).Increment();
if (promptly)
{
DestroyActivationVoid(data); // Don't await or Ignore, since we are in this activation context and it may have alraedy been destroyed!
}
}
// To be called fro within Activation context.
// To be used only if an activation is stuck for a long time, since it can lead to a duplicate activation
internal void DeactivateStuckActivation(ActivationData activationData)
{
DeactivateActivationImpl(activationData, StatisticNames.CATALOG_ACTIVATION_SHUTDOWN_VIA_DEACTIVATE_STUCK_ACTIVATION);
// The unregistration is normally done in the regular deactivation process, but since this activation seems
// stuck (it might never run the deactivation process), we remove it from the directory directly
scheduler.RunOrQueueTask(
() => directory.UnregisterAsync(activationData.Address, UnregistrationCause.Force),
SchedulingContext)
.Ignore();
}
// this is a compatibility method for portions of the code base that don't use
// async/await yet, which is almost everything. there's no liability to discarding the
// Task returned by AsyncSendMessage()
internal void SendMessage(Message message, ActivationData sendingActivation = null)
{
AsyncSendMessage(message, sendingActivation).Ignore();
}
/// <summary>
/// Receive a new message:
/// - validate order constraints, queue (or possibly redirect) if out of order
/// - validate transactions constraints
/// - invoke handler if ready, otherwise enqueue for later invocation
/// </summary>
/// <param name="message"></param>
public void ReceiveMessage(Message message)
{
EventSourceUtils.EmitEvent(message, OrleansDispatcherEvent.ReceiveMessageAction);
MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message);
// Don't process messages that have already timed out
if (message.IsExpired)
{
logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired");
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch);
return;
}
// check if its targeted at a new activation
if (message.TargetGrain.IsSystemTarget)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target.");
throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
}
try
{
Task ignore;
ActivationData target = catalog.GetOrCreateActivation(
message.TargetAddress,
message.IsNewPlacement,
message.NewGrainType,
String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType,
message.RequestContextData,
out ignore);
if (ignore != null)
{
ignore.Ignore();
}
if (message.Direction == Message.Directions.Response)
{
ReceiveResponse(message, target);
}
else // Request or OneWay
{
if (target.State == ActivationState.Valid)
{
this.activationCollector.TryRescheduleCollection(target);
}
// Silo is always capable to accept a new request. It's up to the activation to handle its internal state.
// If activation is shutting down, it will queue and later forward this request.
ReceiveRequest(message, target);
}
}
catch (Exception ex)
{
try
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation");
var nea = ex as Catalog.NonExistentActivationException;
if (nea == null)
{
var str = $"Error creating activation for {message.NewGrainType}. Message {message}";
logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
throw new OrleansException(str, ex);
}
if (nea.IsStatelessWorker)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
$"Intermediate StatelessWorker NonExistentActivation for message {message}, Exception {ex}");
}
}
else
{
logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
$"Intermediate NonExistentActivation for message {message}, with Exception {ex}");
}
ActivationAddress nonExistentActivation = nea.NonExistentActivation;
if (message.Direction != Message.Directions.Response)
{
// Un-register the target activation so we don't keep getting spurious messages.
// The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
// this request snuck ahead of another request, with new placement requested, for the same activation.
// If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
// we want to make sure that we don't un-register the activation we just created.
// We would add a counter here, except that there's already a counter for this in the Catalog.
// Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
var origin = message.SendingSilo;
scheduler.QueueWorkItem(new ClosureWorkItem(
// don't use message.TargetAddress, cause it may have been removed from the headers by this time!
async() =>
{
try
{
await this.localGrainDirectory.UnregisterAfterNonexistingActivation(
nonExistentActivation, origin);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
$"Failed to un-register NonExistentActivation {nonExistentActivation}", exc);
}
},
"LocalGrainDirectory.UnregisterAfterNonexistingActivation"),
catalog.SchedulingContext);
ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
}
else
{
logger.Warn(
ErrorCode.Dispatcher_NoTargetActivation,
nonExistentActivation.Silo.IsClient
? "No target client {0} for response message: {1}. It's likely that the client recently disconnected."
: "No target activation {0} for response message: {1}",
nonExistentActivation,
message);
this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
}
}
catch (Exception exc)
{
// Unable to create activation for this request - reject message
RejectMessage(message, Message.RejectionTypes.Transient, exc);
}
}
}
private void SendRequestMessage(
GrainReference target,
Message message,
TaskCompletionSource <object> context,
Action <Message, TaskCompletionSource <object> > callback,
string debugContext,
InvokeMethodOptions options,
string genericArguments = null)
{
// fill in sender
if (message.SendingSilo == null)
{
message.SendingSilo = MySilo;
}
if (!String.IsNullOrEmpty(genericArguments))
{
message.GenericGrainType = genericArguments;
}
SchedulingContext schedulingContext = RuntimeContext.Current != null ?
RuntimeContext.Current.ActivationContext as SchedulingContext : null;
ActivationData sendingActivation = null;
if (schedulingContext == null)
{
throw new InvalidExpressionException(
String.Format("Trying to send a message {0} on a silo not from within grain and not from within system target (RuntimeContext is not set to SchedulingContext) "
+ "RuntimeContext.Current={1} TaskScheduler.Current={2}",
message,
RuntimeContext.Current == null ? "null" : RuntimeContext.Current.ToString(),
TaskScheduler.Current));
}
switch (schedulingContext.ContextType)
{
case SchedulingContextType.SystemThread:
throw new ArgumentException(
String.Format("Trying to send a message {0} on a silo not from within grain and not from within system target (RuntimeContext is of SchedulingContextType.SystemThread type)", message), "context");
case SchedulingContextType.Activation:
message.SendingActivation = schedulingContext.Activation.ActivationId;
message.SendingGrain = schedulingContext.Activation.Grain;
sendingActivation = schedulingContext.Activation;
break;
case SchedulingContextType.SystemTarget:
message.SendingActivation = schedulingContext.SystemTarget.ActivationId;
message.SendingGrain = schedulingContext.SystemTarget.GrainId;
break;
}
// fill in destination
var targetGrainId = target.GrainId;
message.TargetGrain = targetGrainId;
if (targetGrainId.IsSystemTarget)
{
SiloAddress targetSilo = (target.SystemTargetSilo ?? MySilo);
message.TargetSilo = targetSilo;
message.TargetActivation = ActivationId.GetSystemActivation(targetGrainId, targetSilo);
message.Category = targetGrainId.Equals(Constants.MembershipOracleId) ?
Message.Categories.Ping : Message.Categories.System;
}
if (target.IsObserverReference)
{
message.TargetObserverId = target.ObserverId;
}
if (debugContext != null)
{
message.DebugContext = debugContext;
}
var oneWay = (options & InvokeMethodOptions.OneWay) != 0;
if (context == null && !oneWay)
{
logger.Warn(ErrorCode.IGC_SendRequest_NullContext, "Null context {0}: {1}", message, new StackTrace());
}
if (message.IsExpirableMessage(Config.Globals))
{
message.Expiration = DateTime.UtcNow + ResponseTimeout + Constants.MAXIMUM_CLOCK_SKEW;
}
if (!oneWay)
{
var callbackData = new CallbackData(
callback,
TryResendMessage,
context,
message,
() => UnRegisterCallback(message.Id),
Config.Globals);
callbacks.TryAdd(message.Id, callbackData);
callbackData.StartTimer(ResponseTimeout);
}
if (targetGrainId.IsSystemTarget)
{
// Messages to system targets bypass the task system and get sent "in-line"
dispatcher.TransportMessage(message);
}
else
{
dispatcher.SendMessage(message, sendingActivation);
}
}
/// <summary>
/// Send an outgoing message
/// - may buffer for transaction completion / commit if it ends a transaction
/// - choose target placement address, maintaining send order
/// - add ordering info & maintain send order
///
/// </summary>
/// <param name="message"></param>
/// <param name="sendingActivation"></param>
public async Task AsyncSendMessage(Message message, ActivationData sendingActivation = null)
{
try
{
await AddressMessage(message);
TransportMessage(message);
}
catch (Exception ex)
{
if (!(ex.GetBaseException() is KeyNotFoundException))
{
logger.Error(ErrorCode.Dispatcher_SelectTarget_Failed,
String.Format("SelectTarget failed with {0}", ex.Message),
ex);
}
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "SelectTarget failed");
RejectMessage(message, Message.RejectionTypes.Unrecoverable, ex);
}
}
/// <summary>
/// Invoked when an activation has finished a transaction and may be ready for additional transactions
/// </summary>
/// <param name="activation">The activation that has just completed processing this message</param>
/// <param name="message">The message that has just completed processing.
/// This will be <c>null</c> for the case of completion of Activate/Deactivate calls.</param>
internal void OnActivationCompletedRequest(ActivationData activation, Message message)
{
lock (activation)
{
#if DEBUG
// This is a hot code path, so using #if to remove diags from Release version
if (logger.IsVerbose2)
{
logger.Verbose2(ErrorCode.Dispatcher_OnActivationCompletedRequest_Waiting,
"OnActivationCompletedRequest {0}: Activation={1}", activation.ActivationId, activation.DumpStatus());
}
#endif
activation.ResetRunning(message);
// ensure inactive callbacks get run even with transactions disabled
if (!activation.IsCurrentlyExecuting)
activation.RunOnInactive();
// Run message pump to see if there is a new request arrived to be processed
RunMessagePump(activation);
}
}
/// <summary>
/// Invoke the activate method on a newly created activation
/// </summary>
/// <param name="activation"></param>
/// <returns></returns>
private Task InvokeActivate(ActivationData activation)
{
// NOTE: This should only be called with the correct schedulering context for the activation to be invoked.
lock (activation)
{
activation.SetState(ActivationState.Activating);
}
return scheduler.QueueTask(() => CallGrainActivate(activation), new SchedulingContext(activation)); // Target grain's scheduler context);
// ActivationData will transition out of ActivationState.Activating via Dispatcher.OnActivationCompletedRequest
}
// To be called fro within Activation context.
// Cannot be awaitable, since after DestroyActivation is done the activation is in Invalid state and cannot await any Task.
internal void DeactivateActivationOnIdle(ActivationData data)
{
bool promptly = false;
bool alreadBeingDestroyed = false;
lock (data)
{
if (data.State == ActivationState.Valid)
{
// Change the ActivationData state here, since we're about to give up the lock.
data.PrepareForDeactivation(); // Don't accept any new messages
if (!data.IsCurrentlyExecuting)
{
promptly = true;
}
else // busy, so destroy later.
{
data.AddOnInactive(() => DestroyActivationVoid(data));
}
}
else
{
alreadBeingDestroyed = true;
}
}
logger.Info(ErrorCode.Catalog_ShutdownActivations_2,
"DeactivateActivationOnIdle: 1 {0}.", promptly ? "promptly" : (alreadBeingDestroyed ? "already being destroyed or invalid" : "later when become idle"));
CounterStatistic.FindOrCreate(StatisticNames.CATALOG_ACTIVATION_SHUTDOWN_VIA_DEACTIVATE_ON_IDLE).Increment();
if (promptly)
{
DestroyActivationVoid(data); // Don't await or Ignore, since we are in this activation context and it may have alraedy been destroyed!
}
}
/// <summary>
/// Register a new object to which messages can be delivered with the local lookup table and scheduler.
/// </summary>
/// <param name="activation"></param>
public void RegisterMessageTarget(ActivationData activation)
{
var context = new SchedulingContext(activation);
scheduler.RegisterWorkContext(context);
activations.RecordNewTarget(activation);
activationsCreated.Increment();
}
private void DestroyActivationAsync(ActivationData activation, MultiTaskCompletionSource tcs)
{
StartDestroyActivations(new List<ActivationData> { activation }, tcs);
}
/// <summary>
/// Unregister message target and stop delivering messages to it
/// </summary>
/// <param name="activation"></param>
public void UnregisterMessageTarget(ActivationData activation)
{
activations.RemoveTarget(activation);
// this should be removed once we've refactored the deactivation code path. For now safe to keep.
ActivationCollector.TryCancelCollection(activation);
activationsDestroyed.Increment();
scheduler.UnregisterWorkContext(new SchedulingContext(activation));
if (activation.GrainInstance == null) return;
var grainTypeName = TypeUtils.GetFullName(activation.GrainInstanceType);
activations.DecrementGrainCounter(grainTypeName);
activation.SetGrainInstance(null);
}
private async Task CallGrainActivate(ActivationData activation)
{
var grainTypeName = activation.GrainInstanceType.FullName;
// Note: This call is being made from within Scheduler.Queue wrapper, so we are already executing on worker thread
if (logger.IsVerbose) logger.Verbose(ErrorCode.Catalog_BeforeCallingActivate, "About to call {1} grain's OnActivateAsync() method {0}", activation, grainTypeName);
// Call OnActivateAsync inline, but within try-catch wrapper to safely capture any exceptions thrown from called function
try
{
await activation.GrainInstance.OnActivateAsync();
if (logger.IsVerbose) logger.Verbose(ErrorCode.Catalog_AfterCallingActivate, "Returned from calling {1} grain's OnActivateAsync() method {0}", activation, grainTypeName);
lock (activation)
{
if (activation.State == ActivationState.Activating)
{
activation.SetState(ActivationState.Valid); // Activate calls on this activation are finished
}
if (!activation.IsCurrentlyExecuting)
{
activation.RunOnInactive();
}
// Run message pump to see if there is a new request is queued to be processed
dispatcher.RunMessagePump(activation);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.Catalog_ErrorCallingActivate,
string.Format("Error calling grain's AsyncActivate method - Grain type = {1} Activation = {0}", activation, grainTypeName), exc);
activation.SetState(ActivationState.Invalid); // Mark this activation as unusable
activationsFailedToActivate.Increment();
throw;
}
}
/// <summary>
/// If activation already exists, use it
/// Otherwise, create an activation of an existing grain by reading its state.
/// Return immediately using a dummy that will queue messages.
/// Concurrently start creating and initializing the real activation and replace it when it is ready.
/// </summary>
/// <param name="address">Grain's activation address</param>
/// <param name="newPlacement">Creation of new activation was requested by the placement director.</param>
/// <param name="grainType">The type of grain to be activated or created</param>
/// <param name="genericArguments">Specific generic type of grain to be activated or created</param>
/// <param name="activatedPromise"></param>
/// <returns></returns>
public ActivationData GetOrCreateActivation(
ActivationAddress address,
bool newPlacement,
string grainType,
string genericArguments,
out Task activatedPromise)
{
ActivationData result;
activatedPromise = TaskDone.Done;
lock (activations)
{
if (TryGetActivationData(address.Activation, out result))
{
ActivationCollector.TryRescheduleCollection(result);
return result;
}
if (newPlacement && !SiloStatusOracle.CurrentStatus.IsTerminating())
{
// create a dummy activation that will queue up messages until the real data arrives
PlacementStrategy placement;
int typeCode = address.Grain.GetTypeCode();
string actualGrainType = null;
if (typeCode != 0) // special case for Membership grain.
GetGrainTypeInfo(typeCode, out actualGrainType, out placement);
else
placement = SystemPlacement.Singleton;
if (string.IsNullOrEmpty(grainType))
{
grainType = actualGrainType;
}
// We want to do this (RegisterMessageTarget) under the same lock that we tested TryGetActivationData. They both access ActivationDirectory.
result = new ActivationData(
address,
genericArguments,
placement,
ActivationCollector,
config.Application.GetCollectionAgeLimit(grainType));
RegisterMessageTarget(result);
}
} // End lock
// Did not find and did not start placing new
if (result == null)
{
var msg = String.Format("Non-existent activation: {0}, grain type: {1}.",
address.ToFullString(), grainType);
if (logger.IsVerbose) logger.Verbose(ErrorCode.CatalogNonExistingActivation2, msg);
CounterStatistic.FindOrCreate(StatisticNames.CATALOG_ACTIVATION_NON_EXISTENT_ACTIVATIONS).Increment();
throw new NonExistentActivationException(msg) { NonExistentActivation = address };
}
SetupActivationInstance(result, grainType, genericArguments);
activatedPromise = InitActivation(result, grainType, genericArguments);
return result;
}
private void SetupActivationInstance(ActivationData result, string grainType, string genericInterface)
{
var genericArguments = String.IsNullOrEmpty(genericInterface) ? null
: TypeUtils.GenericTypeArgsString(genericInterface);
lock (result)
{
if (result.GrainInstance == null)
{
CreateGrainInstance(grainType, result, genericArguments);
}
}
}
private async Task InitActivation(ActivationData activation, string grainType, string genericInterface)
{
// We've created a dummy activation, which we'll eventually return, but in the meantime we'll queue up (or perform promptly)
// the operations required to turn the "dummy" activation into a real activation
ActivationAddress address = activation.Address;
int initStage = 0;
// A chain of promises that will have to complete in order to complete the activation
// Register with the grain directory, register with the store if necessary and call the Activate method on the new activation.
try
{
initStage = 1;
await RegisterActivationInGrainDirectory(address, !activation.IsMultiActivationGrain);
initStage = 2;
await SetupActivationState(activation, grainType);
initStage = 3;
await InvokeActivate(activation);
ActivationCollector.ScheduleCollection(activation);
// Success!! Log the result, and start processing messages
if (logger.IsVerbose) logger.Verbose("InitActivation is done: {0}", address);
}
catch (Exception ex)
{
lock (activation)
{
activation.SetState(ActivationState.Invalid);
try
{
UnregisterMessageTarget(activation);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Catalog_UnregisterMessageTarget4, String.Format("UnregisterMessageTarget failed on {0}.", activation), exc);
}
switch (initStage)
{
case 1: // failed to RegisterActivationInGrainDirectory
ActivationAddress target = null;
Exception dupExc;
// Failure!! Could it be that this grain uses single activation placement, and there already was an activation?
if (Utils.TryFindException(ex, typeof (DuplicateActivationException), out dupExc))
{
target = ((DuplicateActivationException) dupExc).ActivationToUse;
CounterStatistic.FindOrCreate(StatisticNames.CATALOG_ACTIVATION_DUPLICATE_ACTIVATIONS)
.Increment();
}
activation.ForwardingAddress = target;
if (target != null)
{
// If this was a duplicate, it's not an error, just a race.
// Forward on all of the pending messages, and then forget about this activation.
logger.Info(ErrorCode.Catalog_DuplicateActivation,
"Tried to create a duplicate activation {0}, but we'll use {1} instead. " +
"GrainInstanceType is {2}. " +
"Primary Directory partition for this grain is {3}, " +
"full activation address is {4}. We have {5} messages to forward.",
address,
target,
activation.GrainInstanceType,
((DuplicateActivationException) dupExc).PrimaryDirectoryForGrain,
address.ToFullString(),
activation.WaitingCount);
RerouteAllQueuedMessages(activation, target, "Duplicate activation", ex);
}
else
{
logger.Warn(ErrorCode.Runtime_Error_100064,
String.Format("Failed to RegisterActivationInGrainDirectory for {0}.",
activation), ex);
// Need to undo the registration we just did earlier
scheduler.RunOrQueueTask(() => directory.UnregisterAsync(address),
SchedulingContext).Ignore();
RerouteAllQueuedMessages(activation, null,
"Failed RegisterActivationInGrainDirectory", ex);
}
break;
case 2: // failed to setup persistent state
logger.Warn(ErrorCode.Catalog_Failed_SetupActivationState,
String.Format("Failed to SetupActivationState for {0}.", activation), ex);
// Need to undo the registration we just did earlier
scheduler.RunOrQueueTask(() => directory.UnregisterAsync(address),
SchedulingContext).Ignore();
RerouteAllQueuedMessages(activation, null, "Failed SetupActivationState", ex);
break;
case 3: // failed to InvokeActivate
logger.Warn(ErrorCode.Catalog_Failed_InvokeActivate,
String.Format("Failed to InvokeActivate for {0}.", activation), ex);
// Need to undo the registration we just did earlier
scheduler.RunOrQueueTask(() => directory.UnregisterAsync(address),
SchedulingContext).Ignore();
RerouteAllQueuedMessages(activation, null, "Failed InvokeActivate", ex);
break;
}
}
throw;
}
}
/// <summary>
/// Check if we can locally accept this message.
/// Redirects if it can't be accepted.
/// </summary>
/// <param name="message"></param>
/// <param name="targetActivation"></param>
private void ReceiveRequest(Message message, ActivationData targetActivation)
{
lock (targetActivation)
{
if (targetActivation.State == ActivationState.Invalid)
{
ProcessRequestToInvalidActivation(
message,
targetActivation.Address,
targetActivation.ForwardingAddress,
"ReceiveRequest");
}
else if (!ActivationMayAcceptRequest(targetActivation, message))
{
// Check for deadlock before Enqueueing.
if (config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget)
{
try
{
CheckDeadlock(message);
}
catch (DeadlockException exc)
{
// Record that this message is no longer flowing through the system
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Deadlock");
logger.Warn(ErrorCode.Dispatcher_DetectedDeadlock,
"Detected Application Deadlock: {0}", exc.Message);
// We want to send DeadlockException back as an application exception, rather than as a system rejection.
SendResponse(message, Response.ExceptionResponse(exc));
return;
}
}
EnqueueRequest(message, targetActivation);
}
else
{
HandleIncomingRequest(message, targetActivation);
}
}
}
/// <summary>
/// Perform just the prompt, local part of creating an activation object
/// Caller is responsible for registering locally, registering with store and calling its activate routine
/// </summary>
/// <param name="grainTypeName"></param>
/// <param name="data"></param>
/// <param name="genericArguments"></param>
/// <returns></returns>
private void CreateGrainInstance(string grainTypeName, ActivationData data, string genericArguments)
{
string grainClassName;
var interfaceToClassMap = GrainTypeManager.GetGrainInterfaceToClassMap();
if (!interfaceToClassMap.TryGetValue(grainTypeName, out grainClassName))
{
// Lookup from grain type code
var typeCode = data.Grain.GetTypeCode();
if (typeCode != 0)
{
PlacementStrategy unused;
GetGrainTypeInfo(typeCode, out grainClassName, out unused, genericArguments);
}
else
{
grainClassName = grainTypeName;
}
}
GrainTypeData grainTypeData = GrainTypeManager[grainClassName];
Type grainType = grainTypeData.Type;
Type stateObjectType = grainTypeData.StateObjectType;
lock (data)
{
var grain = (Grain) Activator.CreateInstance(grainType);
grain.Identity = data.Identity;
grain.Runtime = grainRuntime;
data.SetGrainInstance(grain);
if (stateObjectType != null)
{
SetupStorageProvider(data);
var state = (GrainState)Activator.CreateInstance(stateObjectType);
state.InitState(null);
data.GrainInstance.GrainState = state;
data.GrainInstance.Storage = new GrainStateStorageBridge(data.GrainTypeName, data.GrainInstance, data.StorageProvider);
}
}
activations.IncrementGrainCounter(grainClassName);
data.GrainInstance.Data = data;
if (logger.IsVerbose) logger.Verbose("CreateGrainInstance {0}{1}", data.Grain, data.ActivationId);
}
/// <summary>
/// Whether an incoming message can interleave
/// </summary>
/// <param name="targetActivation"></param>
/// <param name="incoming"></param>
/// <returns></returns>
public bool CanInterleave(ActivationData targetActivation, Message incoming)
{
bool canInterleave =
catalog.IsReentrantGrain(targetActivation.ActivationId)
|| incoming.IsAlwaysInterleave
|| targetActivation.Running == null
|| (targetActivation.Running.IsReadOnly && incoming.IsReadOnly);
return canInterleave;
}
private void SetupStorageProvider(ActivationData data)
{
var grainTypeName = data.GrainInstanceType.FullName;
// Get the storage provider name, using the default if not specified.
var attrs = data.GrainInstanceType.GetCustomAttributes(typeof(StorageProviderAttribute), true);
var attr = attrs.FirstOrDefault() as StorageProviderAttribute;
var storageProviderName = attr != null ? attr.ProviderName : Constants.DEFAULT_STORAGE_PROVIDER_NAME;
IStorageProvider provider;
if (storageProviderManager == null || storageProviderManager.GetNumLoadedProviders() == 0)
{
var errMsg = string.Format("No storage providers found loading grain type {0}", grainTypeName);
logger.Error(ErrorCode.Provider_CatalogNoStorageProvider_1, errMsg);
throw new BadProviderConfigException(errMsg);
}
if (string.IsNullOrWhiteSpace(storageProviderName))
{
// Use default storage provider
provider = storageProviderManager.GetDefaultProvider();
}
else
{
// Look for MemoryStore provider as special case name
bool caseInsensitive = Constants.MEMORY_STORAGE_PROVIDER_NAME.Equals(storageProviderName, StringComparison.OrdinalIgnoreCase);
storageProviderManager.TryGetProvider(storageProviderName, out provider, caseInsensitive);
if (provider == null)
{
var errMsg = string.Format(
"Cannot find storage provider with Name={0} for grain type {1}", storageProviderName,
grainTypeName);
logger.Error(ErrorCode.Provider_CatalogNoStorageProvider_2, errMsg);
throw new BadProviderConfigException(errMsg);
}
}
data.StorageProvider = provider;
if (logger.IsVerbose2)
{
string msg = string.Format("Assigned storage provider with Name={0} to grain type {1}",
storageProviderName, grainTypeName);
logger.Verbose2(ErrorCode.Provider_CatalogStorageProviderAllocated, msg);
}
}
/// <summary>
/// Enqueue message for local handling after transaction completes
/// </summary>
/// <param name="message"></param>
/// <param name="targetActivation"></param>
private void EnqueueRequest(Message message, ActivationData targetActivation)
{
var overloadException = targetActivation.CheckOverloaded(logger);
if (overloadException != null)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Overload2");
RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + targetActivation);
return;
}
bool enqueuedOk = targetActivation.EnqueueMessage(message);
if (!enqueuedOk)
{
ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "EnqueueRequest");
}
// Dont count this as end of processing. The message will come back after queueing via HandleIncomingRequest.
#if DEBUG
// This is a hot code path, so using #if to remove diags from Release version
// Note: Caller already holds lock on activation
if (logger.IsVerbose2) logger.Verbose2(ErrorCode.Dispatcher_EnqueueMessage,
"EnqueueMessage for {0}: targetActivation={1}", message.TargetActivation, targetActivation.DumpStatus());
#endif
}
private async Task SetupActivationState(ActivationData result, string grainType)
{
var state = result.GrainInstance.GrainState;
if (result.StorageProvider != null && state != null)
{
var sw = Stopwatch.StartNew();
// Populate state data
try
{
var grainRef = result.GrainReference;
await scheduler.RunOrQueueTask(() =>
result.StorageProvider.ReadStateAsync(grainType, grainRef, state),
new SchedulingContext(result));
sw.Stop();
StorageStatisticsGroup.OnStorageActivate(result.StorageProvider, grainType, result.GrainReference, sw.Elapsed);
result.GrainInstance.GrainState = state;
}
catch (Exception ex)
{
StorageStatisticsGroup.OnStorageActivateError(result.StorageProvider, grainType, result.GrainReference);
sw.Stop();
if (!(ex.GetBaseException() is KeyNotFoundException))
throw;
result.GrainInstance.GrainState = state; // Just keep original empty state object
}
}
}
// this is a compatibility method for portions of the code base that don't use
// async/await yet, which is almost everything. there's no liability to discarding the
// Task returned by AsyncSendMessage()
internal void SendMessage(Message message, ActivationData sendingActivation = null)
{
AsyncSendMessage(message, sendingActivation).Ignore();
}
/// <summary>
/// Try to get runtime data for an activation
/// </summary>
/// <param name="activationId"></param>
/// <param name="data"></param>
/// <returns></returns>
public bool TryGetActivationData(ActivationId activationId, out ActivationData data)
{
data = null;
if (activationId.IsSystem) return false;
data = activations.FindTarget(activationId);
return data != null;
}
internal void RunMessagePump(ActivationData activation)
{
// Note: this method must be called while holding lock (activation)
#if DEBUG
// This is a hot code path, so using #if to remove diags from Release version
// Note: Caller already holds lock on activation
if (logger.IsVerbose2)
{
logger.Verbose2(ErrorCode.Dispatcher_ActivationEndedTurn_Waiting,
"RunMessagePump {0}: Activation={1}", activation.ActivationId, activation.DumpStatus());
}
#endif
// don't run any messages if activation is not ready or deactivating
if (!activation.State.Equals(ActivationState.Valid)) return;
bool runLoop;
do
{
runLoop = false;
var nextMessage = activation.PeekNextWaitingMessage();
if (nextMessage == null) continue;
if (!ActivationMayAcceptRequest(activation, nextMessage)) continue;
activation.DequeueNextWaitingMessage();
// we might be over-writing an already running read only request.
HandleIncomingRequest(nextMessage, activation);
runLoop = true;
}
while (runLoop);
}
/// <summary>
/// Receive a new message:
/// - validate order constraints, queue (or possibly redirect) if out of order
/// - validate transactions constraints
/// - invoke handler if ready, otherwise enqueue for later invocation
/// </summary>
/// <param name="message"></param>
public void ReceiveMessage(Message message)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message);
// Don't process messages that have already timed out
if (message.IsExpired)
{
logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired");
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch);
return;
}
// check if its targeted at a new activation
if (message.TargetGrain.IsSystemTarget)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target.");
throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
}
if (errorInjection && ShouldInjectError(message))
{
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection");
}
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection");
RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection");
return;
}
try
{
Task ignore;
ActivationData target = catalog.GetOrCreateActivation(
message.TargetAddress,
message.IsNewPlacement,
message.NewGrainType,
message.GenericGrainType,
message.RequestContextData,
out ignore);
if (ignore != null)
{
ignore.Ignore();
}
if (message.Direction == Message.Directions.Response)
{
ReceiveResponse(message, target);
}
else // Request or OneWay
{
// Silo is always capable to accept a new request. It's up to the activation to handle its internal state.
// If activation is shutting down, it will queue and later forward this request.
ReceiveRequest(message, target);
}
}
catch (Exception ex)
{
try
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation");
var nea = ex as Catalog.NonExistentActivationException;
if (nea == null)
{
var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message);
logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
throw new OrleansException(str, ex);
}
if (nea.IsStatelessWorker)
{
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
String.Format("Intermediate StatelessWorker NonExistentActivation for message {0}", message), ex);
}
}
else
{
logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
String.Format("Intermediate NonExistentActivation for message {0}", message), ex);
}
ActivationAddress nonExistentActivation = nea.NonExistentActivation;
if (message.Direction != Message.Directions.Response)
{
// Un-register the target activation so we don't keep getting spurious messages.
// The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
// this request snuck ahead of another request, with new placement requested, for the same activation.
// If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
// we want to make sure that we don't un-register the activation we just created.
// We would add a counter here, except that there's already a counter for this in the Catalog.
// Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero)
{
Scheduler.QueueWorkItem(new ClosureWorkItem(
// don't use message.TargetAddress, cause it may have been removed from the headers by this time!
async() =>
{
try
{
await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync(
nonExistentActivation);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
String.Format("Failed to un-register NonExistentActivation {0}",
nonExistentActivation), exc);
}
},
() => "LocalGrainDirectory.UnregisterConditionallyAsync"),
catalog.SchedulingContext);
}
ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
}
else
{
logger.Warn(ErrorCode.Dispatcher_NoTargetActivation,
"No target activation {0} for response message: {1}", nonExistentActivation, message);
Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
}
}
catch (Exception exc)
{
// Unable to create activation for this request - reject message
RejectMessage(message, Message.RejectionTypes.Transient, exc);
}
}
}
