private void SendResponse(Message request, Response response)
{
// Don't process messages that have already timed out
if (request.IsExpired)
{
request.DropExpiredMessage(MessagingStatisticsGroup.Phase.Respond);
return;
}
this.Dispatcher.SendResponse(request, response);
}
public void SendResponse(Message request, Response response)
{
EventSourceUtils.EmitEvent(request, OrleansInsideRuntimeClientEvent.SendResponseAction);
// Don't process messages that have already timed out
if (request.IsExpired)
{
request.DropExpiredMessage(MessagingStatisticsGroup.Phase.Respond);
return;
}
this.Dispatcher.SendResponse(request, response);
}
/// <summary>
/// Receive a new message:
/// - validate order constraints, queue (or possibly redirect) if out of order
/// - validate transactions constraints
/// - invoke handler if ready, otherwise enqueue for later invocation
/// </summary>
/// <param name="message"></param>
public void ReceiveMessage(Message message)
{
EventSourceUtils.EmitEvent(message, OrleansDispatcherEvent.ReceiveMessageAction);
MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message);
// Don't process messages that have already timed out
if (message.IsExpired)
{
logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired");
message.DropExpiredMessage(this.logger, MessagingStatisticsGroup.Phase.Dispatch);
return;
}
// check if its targeted at a new activation
if (message.TargetGrain.IsSystemTarget)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target.");
throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
}
try
{
Task ignore;
ActivationData target = catalog.GetOrCreateActivation(
message.TargetAddress,
message.IsNewPlacement,
message.NewGrainType,
String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType,
message.RequestContextData,
out ignore);
if (ignore != null)
{
ignore.Ignore();
}
if (message.Direction == Message.Directions.Response)
{
ReceiveResponse(message, target);
}
else // Request or OneWay
{
if (target.State == ActivationState.Valid)
{
this.activationCollector.TryRescheduleCollection(target);
}
// Silo is always capable to accept a new request. It's up to the activation to handle its internal state.
// If activation is shutting down, it will queue and later forward this request.
ReceiveRequest(message, target);
}
}
catch (Exception ex)
{
try
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation");
var nea = ex as Catalog.NonExistentActivationException;
if (nea == null)
{
var str = $"Error creating activation for {message.NewGrainType}. Message {message}";
logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
throw new OrleansException(str, ex);
}
if (nea.IsStatelessWorker)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
$"Intermediate StatelessWorker NonExistentActivation for message {message}, Exception {ex}");
}
}
else
{
logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
$"Intermediate NonExistentActivation for message {message}, with Exception {ex}");
}
ActivationAddress nonExistentActivation = nea.NonExistentActivation;
if (message.Direction != Message.Directions.Response)
{
// Un-register the target activation so we don't keep getting spurious messages.
// The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
// this request snuck ahead of another request, with new placement requested, for the same activation.
// If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
// we want to make sure that we don't un-register the activation we just created.
// We would add a counter here, except that there's already a counter for this in the Catalog.
// Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
var origin = message.SendingSilo;
scheduler.QueueWorkItem(new ClosureWorkItem(
// don't use message.TargetAddress, cause it may have been removed from the headers by this time!
async() =>
{
try
{
await this.localGrainDirectory.UnregisterAfterNonexistingActivation(
nonExistentActivation, origin);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
$"Failed to un-register NonExistentActivation {nonExistentActivation}", exc);
}
},
"LocalGrainDirectory.UnregisterAfterNonexistingActivation"),
catalog.SchedulingContext);
ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
}
else
{
logger.Warn(
ErrorCode.Dispatcher_NoTargetActivation,
nonExistentActivation.Silo.IsClient
? "No target client {0} for response message: {1}. It's likely that the client recently disconnected."
: "No target activation {0} for response message: {1}",
nonExistentActivation,
message);
this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
}
}
catch (Exception exc)
{
// Unable to create activation for this request - reject message
RejectMessage(message, Message.RejectionTypes.Transient, exc);
}
}
}
public async Task Invoke(IAddressable target, IInvokable invokable, Message message)
{
try
{
// Don't process messages that have already timed out
if (message.IsExpired)
{
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke);
return;
}
RequestContext.Import(message.RequestContextData);
if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget)
{
UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message));
// RequestContext is automatically saved in the msg upon send and propagated to the next hop
// in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message);
}
object resultObject;
try
{
var request = (InvokeMethodRequest)message.GetDeserializedBody(this.SerializationManager);
if (request.Arguments != null)
{
CancellationSourcesExtension.RegisterCancellationTokens(target, request, logger, this);
}
var invoker = invokable.GetInvoker(typeManager, request.InterfaceId, message.GenericGrainType);
if (invoker is IGrainExtensionMethodInvoker &&
!(target is IGrainExtension))
{
// We are trying the invoke a grain extension method on a grain
// -- most likely reason is that the dynamic extension is not installed for this grain
// So throw a specific exception here rather than a general InvalidCastException
var error = String.Format(
"Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}",
target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName);
var exc = new GrainExtensionNotInstalledException(error);
string extraDebugInfo = null;
#if DEBUG
extraDebugInfo = Utils.GetStackTrace();
#endif
logger.Warn(ErrorCode.Stream_ExtensionNotInstalled,
string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc);
throw exc;
}
#pragma warning disable 618
var invokeInterceptor = this.CurrentStreamProviderRuntime?.GetInvokeInterceptor();
#pragma warning restore 618
var requestInvoker = new GrainMethodInvoker(target, request, invoker, siloInterceptors, interfaceToImplementationMapping, invokeInterceptor);
await requestInvoker.Invoke();
resultObject = requestInvoker.Result;
}
catch (Exception exc1)
{
if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay)
{
invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message, exc1);
}
if (exc1 is InconsistentStateException && target is Grain)
{
var activation = ((Grain)target).Data;
invokeExceptionLogger.Info($"Deactivating {activation} due to inconsistent state.");
this.DeactivateOnIdle(activation.ActivationId);
}
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc1);
}
return;
}
if (message.Direction == Message.Directions.OneWay)
{
return;
}
SafeSendResponse(message, resultObject);
}
catch (Exception exc2)
{
logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2);
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc2);
}
}
}
private static bool ExpireMessageIfExpired(Message message, MessagingStatisticsGroup.Phase phase)
{
if (message.IsExpired)
{
message.DropExpiredMessage(phase);
return true;
}
return false;
}
internal async Task Invoke(IAddressable target, IInvokable invokable, Message message)
{
try
{
// Don't process messages that have already timed out
if (message.IsExpired)
{
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke);
return;
}
//MessagingProcessingStatisticsGroup.OnRequestProcessed(message, "Invoked");
if (Message.WriteMessagingTraces)
{
message.AddTimestamp(Message.LifecycleTag.InvokeIncoming);
}
RequestContext.Import(message.RequestContextData);
if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget)
{
UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message));
// RequestContext is automatically saved in the msg upon send and propagated to the next hop
// in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message);
}
var invoker = invokable.GetInvoker(message.InterfaceId, message.GenericGrainType);
object resultObject;
try
{
var request = (InvokeMethodRequest)message.BodyObject;
if (invoker is IGrainExtensionMethodInvoker &&
!(target is IGrainExtension))
{
// We are trying the invoke a grain extension method on a grain
// -- most likely reason is that the dynamic extension is not installed for this grain
// So throw a specific exception here rather than a general InvalidCastException
var error = String.Format(
"Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}",
target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName);
var exc = new GrainExtensionNotInstalledException(error);
string extraDebugInfo = null;
#if DEBUG
extraDebugInfo = new StackTrace().ToString();
#endif
logger.Warn(ErrorCode.Stream_ExtensionNotInstalled,
string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc);
throw exc;
}
resultObject = await invoker.Invoke(target, request.InterfaceId, request.MethodId, request.Arguments);
}
catch (Exception exc1)
{
if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay)
{
invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message, exc1);
}
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc1);
}
return;
}
if (message.Direction == Message.Directions.OneWay)
{
return;
}
SafeSendResponse(message, resultObject);
}
catch (Exception exc2)
{
logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2);
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc2);
}
}
}
public async Task Invoke(IAddressable target, IInvokable invokable, Message message)
{
try
{
// Don't process messages that have already timed out
if (message.IsExpired)
{
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke);
return;
}
RequestContextExtensions.Import(message.RequestContextData);
if (schedulingOptions.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget)
{
UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext));
// RequestContext is automatically saved in the msg upon send and propagated to the next hop
// in RuntimeClient.CreateMessage -> RequestContextExtensions.ExportToMessage(message);
}
bool startNewTransaction = false;
ITransactionInfo transactionInfo = message.TransactionInfo;
if (message.IsTransactionRequired && transactionInfo == null)
{
// TODO: this should be a configurable parameter
var transactionTimeout = Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(10);
// Start a new transaction
transactionInfo = await this.transactionAgent.Value.StartTransaction(message.IsReadOnly, transactionTimeout);
startNewTransaction = true;
}
if (transactionInfo != null)
{
TransactionContext.SetTransactionInfo(transactionInfo);
}
object resultObject;
try
{
var request = (InvokeMethodRequest)message.GetDeserializedBody(this.serializationManager);
if (request.Arguments != null)
{
CancellationSourcesExtension.RegisterCancellationTokens(target, request, this.loggerFactory, logger, this, this.cancellationTokenRuntime);
}
var invoker = invokable.GetInvoker(typeManager, request.InterfaceId, message.GenericGrainType);
if (invoker is IGrainExtensionMethodInvoker &&
!(target is IGrainExtension))
{
// We are trying the invoke a grain extension method on a grain
// -- most likely reason is that the dynamic extension is not installed for this grain
// So throw a specific exception here rather than a general InvalidCastException
var error = String.Format(
"Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}",
target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName);
var exc = new GrainExtensionNotInstalledException(error);
string extraDebugInfo = null;
#if DEBUG
extraDebugInfo = Utils.GetStackTrace();
#endif
logger.Warn(ErrorCode.Stream_ExtensionNotInstalled,
string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc);
throw exc;
}
var requestInvoker = new GrainMethodInvoker(target, request, invoker, GrainCallFilters, interfaceToImplementationMapping);
await requestInvoker.Invoke();
resultObject = requestInvoker.Result;
}
catch (Exception exc1)
{
if (invokeExceptionLogger.IsEnabled(LogLevel.Debug) || message.Direction == Message.Directions.OneWay)
{
invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message, exc1);
}
transactionInfo = TransactionContext.GetTransactionInfo();
if (transactionInfo != null)
{
// Must abort the transaction on exceptions
transactionInfo.IsAborted = true;
if (startNewTransaction)
{
var abortException = (exc1 as OrleansTransactionAbortedException) ??
new OrleansTransactionAbortedException(transactionInfo.TransactionId.ToString(), exc1);
this.transactionAgent.Value.Abort(transactionInfo, abortException);
exc1 = abortException;
}
}
// If a grain allowed an inconsistent state exception to escape and the exception originated from
// this activation, then deactivate it.
var ise = exc1 as InconsistentStateException;
if (ise != null && ise.IsSourceActivation)
{
// Mark the exception so that it doesn't deactivate any other activations.
ise.IsSourceActivation = false;
var activation = (target as Grain)?.Data;
if (activation != null)
{
invokeExceptionLogger.Info($"Deactivating {activation} due to inconsistent state.");
this.DeactivateOnIdle(activation.ActivationId);
}
}
if (message.Direction != Message.Directions.OneWay)
{
TransactionContext.Clear();
SafeSendExceptionResponse(message, exc1);
}
return;
}
transactionInfo = TransactionContext.GetTransactionInfo();
if (transactionInfo != null && !transactionInfo.ReconcilePending(out var numberOrphans))
{
var abortException = new OrleansOrphanCallException(transactionInfo.TransactionId.ToString(), numberOrphans);
// Can't exit before the transaction completes.
TransactionContext.GetTransactionInfo().IsAborted = true;
if (startNewTransaction)
{
this.transactionAgent.Value.Abort(TransactionContext.GetTransactionInfo(), abortException);
}
if (message.Direction != Message.Directions.OneWay)
{
TransactionContext.Clear();
SafeSendExceptionResponse(message, abortException);
}
return;
}
if (startNewTransaction)
{
// This request started the transaction, so we try to commit before returning.
await this.transactionAgent.Value.Commit(transactionInfo);
TransactionContext.Clear();
}
if (message.Direction == Message.Directions.OneWay)
{
return;
}
SafeSendResponse(message, resultObject);
}
catch (Exception exc2)
{
logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2);
try
{
if (exc2 is OrleansTransactionInDoubtException)
{
this.logger.LogError(exc2, "Transaction failed due to in doubt transaction");
}
else if (TransactionContext.GetTransactionInfo() != null)
{
// Must abort the transaction on exceptions
TransactionContext.GetTransactionInfo().IsAborted = true;
var abortException = (exc2 as OrleansTransactionAbortedException) ??
new OrleansTransactionAbortedException(TransactionContext.GetTransactionInfo().TransactionId.ToString(), exc2);
this.transactionAgent.Value.Abort(TransactionContext.GetTransactionInfo(), abortException);
}
}
finally
{
TransactionContext.Clear();
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc2);
}
}
}
finally
{
TransactionContext.Clear();
}
}
internal async Task Invoke(IAddressable target, IInvokable invokable, Message message)
{
try
{
// Don't process messages that have already timed out
if (message.IsExpired)
{
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke);
return;
}
RequestContext.Import(message.RequestContextData);
if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget)
{
UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message));
// RequestContext is automatically saved in the msg upon send and propagated to the next hop
// in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message);
}
object resultObject;
try
{
var request = (InvokeMethodRequest) message.BodyObject;
var invoker = invokable.GetInvoker(request.InterfaceId, message.GenericGrainType);
if (invoker is IGrainExtensionMethodInvoker
&& !(target is IGrainExtension))
{
// We are trying the invoke a grain extension method on a grain
// -- most likely reason is that the dynamic extension is not installed for this grain
// So throw a specific exception here rather than a general InvalidCastException
var error = String.Format(
"Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}",
target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName);
var exc = new GrainExtensionNotInstalledException(error);
string extraDebugInfo = null;
#if DEBUG
extraDebugInfo = new StackTrace().ToString();
#endif
logger.Warn(ErrorCode.Stream_ExtensionNotInstalled,
string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc);
throw exc;
}
// If the target has a grain-level interceptor or there is a silo-level interceptor, intercept the call.
var shouldCallSiloWideInterceptor = SiloProviderRuntime.Instance.GetInvokeInterceptor() != null && target is IGrain;
var intercepted = target as IGrainInvokeInterceptor;
if (intercepted != null || shouldCallSiloWideInterceptor)
{
// Fetch the method info for the intercepted call.
var implementationInvoker =
invocationMethodInfoMap.GetInterceptedMethodInvoker(target.GetType(), request.InterfaceId,
invoker);
var methodInfo = implementationInvoker.GetMethodInfo(request.MethodId);
if (shouldCallSiloWideInterceptor)
{
// There is a silo-level interceptor and possibly a grain-level interceptor.
var runtime = SiloProviderRuntime.Instance;
resultObject =
await runtime.CallInvokeInterceptor(methodInfo, request, target, implementationInvoker);
}
else
{
// The grain has an interceptor, but there is no silo-wide interceptor.
resultObject = await intercepted.Invoke(methodInfo, request, invoker);
}
}
else
{
// The call is not intercepted.
resultObject = await invoker.Invoke(target, request);
}
}
catch (Exception exc1)
{
if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay)
{
invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message, exc1);
}
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc1);
}
return;
}
if (message.Direction == Message.Directions.OneWay) return;
SafeSendResponse(message, resultObject);
}
catch (Exception exc2)
{
logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2);
if (message.Direction != Message.Directions.OneWay)
SafeSendExceptionResponse(message, exc2);
}
}
private void SendResponse(Message request, Response response)
{
// Don't process messages that have already timed out
if (request.IsExpired)
{
request.DropExpiredMessage(MessagingStatisticsGroup.Phase.Respond);
return;
}
dispatcher.SendResponse(request, response);
}
/// <summary>
/// Receive a new message:
/// - validate order constraints, queue (or possibly redirect) if out of order
/// - validate transactions constraints
/// - invoke handler if ready, otherwise enqueue for later invocation
/// </summary>
/// <param name="message"></param>
public void ReceiveMessage(Message message)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message);
// Don't process messages that have already timed out
if (message.IsExpired)
{
logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired");
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch);
return;
}
// check if its targeted at a new activation
if (message.TargetGrain.IsSystemTarget)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target.");
throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
}
if (errorInjection && ShouldInjectError(message))
{
if (logger.IsVerbose)
{
logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection");
}
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection");
RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection");
return;
}
try
{
Task ignore;
ActivationData target = catalog.GetOrCreateActivation(
message.TargetAddress,
message.IsNewPlacement,
message.NewGrainType,
message.GenericGrainType,
out ignore);
if (ignore != null)
{
ignore.Ignore();
}
if (message.Direction == Message.Directions.Response)
{
ReceiveResponse(message, target);
}
else // Request or OneWay
{
if (SiloCanAcceptRequest(message))
{
ReceiveRequest(message, target);
}
else if (message.MayResend(config.Globals))
{
// Record that this message is no longer flowing through the system
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Redirecting");
throw new NotImplementedException("RedirectRequest() is believed to be no longer necessary; please contact the Orleans team if you see this error.");
}
else
{
// Record that this message is no longer flowing through the system
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Rejecting");
RejectMessage(message, Message.RejectionTypes.Transient, null, "Shutting down");
}
}
}
catch (Exception ex)
{
try
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation");
var nea = ex as Catalog.NonExistentActivationException;
if (nea == null)
{
var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message);
logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
throw new OrleansException(str, ex);
}
logger.Warn(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
String.Format("Intermediate warning for NonExistentActivation from Catalog.GetOrCreateActivation for message {0}", message), ex);
ActivationAddress nonExistentActivation = nea.NonExistentActivation;
if (message.Direction != Message.Directions.Response)
{
// Un-register the target activation so we don't keep getting spurious messages.
// The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
// this request snuck ahead of another request, with new placement requested, for the same activation.
// If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
// we want to make sure that we don't un-register the activation we just created.
// We would add a counter here, except that there's already a counter for this in the Catalog.
// Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero)
{
Scheduler.QueueWorkItem(new ClosureWorkItem(
// don't use message.TargetAddress, cause it may have been removed from the headers by this time!
async() =>
{
try
{
await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync(
nonExistentActivation);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
String.Format("Failed to un-register NonExistentActivation {0}",
nonExistentActivation), exc);
}
},
() => "LocalGrainDirectory.UnregisterConditionallyAsync"),
catalog.SchedulingContext);
}
ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
}
else
{
logger.Warn(ErrorCode.Dispatcher_NoTargetActivation,
"No target activation {0} for response message: {1}", nonExistentActivation, message);
Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
}
}
catch (Exception exc)
{
// Unable to create activation for this request - reject message
RejectMessage(message, Message.RejectionTypes.Transient, exc);
}
}
}
/// <summary>
/// Receive a new message:
/// - validate order constraints, queue (or possibly redirect) if out of order
/// - validate transactions constraints
/// - invoke handler if ready, otherwise enqueue for later invocation
/// </summary>
/// <param name="message"></param>
public void ReceiveMessage(Message message)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message);
// Don't process messages that have already timed out
if (message.IsExpired)
{
logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired");
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch);
return;
}
// check if its targeted at a new activation
if (message.TargetGrain.IsSystemTarget)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target.");
throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
}
if (errorInjection && ShouldInjectError(message))
{
if (logger.IsVerbose) logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection");
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection");
RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection");
return;
}
try
{
Task ignore;
ActivationData target = catalog.GetOrCreateActivation(
message.TargetAddress,
message.IsNewPlacement,
message.NewGrainType,
message.GenericGrainType,
message.RequestContextData,
out ignore);
if (ignore != null)
{
ignore.Ignore();
}
if (message.Direction == Message.Directions.Response)
{
ReceiveResponse(message, target);
}
else // Request or OneWay
{
// Silo is always capable to accept a new request. It's up to the activation to handle its internal state.
// If activation is shutting down, it will queue and later forward this request.
ReceiveRequest(message, target);
}
}
catch (Exception ex)
{
try
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation");
var nea = ex as Catalog.NonExistentActivationException;
if (nea == null)
{
var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message);
logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
throw new OrleansException(str, ex);
}
if (nea.IsStatelessWorker)
{
if (logger.IsVerbose) logger.Verbose(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
String.Format("Intermediate StatelessWorker NonExistentActivation for message {0}", message), ex);
}
else
{
logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
String.Format("Intermediate NonExistentActivation for message {0}", message), ex);
}
ActivationAddress nonExistentActivation = nea.NonExistentActivation;
if (message.Direction != Message.Directions.Response)
{
// Un-register the target activation so we don't keep getting spurious messages.
// The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
// this request snuck ahead of another request, with new placement requested, for the same activation.
// If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
// we want to make sure that we don't un-register the activation we just created.
// We would add a counter here, except that there's already a counter for this in the Catalog.
// Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero)
{
Scheduler.QueueWorkItem(new ClosureWorkItem(
// don't use message.TargetAddress, cause it may have been removed from the headers by this time!
async () =>
{
try
{
await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync(
nonExistentActivation);
}
catch(Exception exc)
{
logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
String.Format("Failed to un-register NonExistentActivation {0}",
nonExistentActivation), exc);
}
},
() => "LocalGrainDirectory.UnregisterConditionallyAsync"),
catalog.SchedulingContext);
}
ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
}
else
{
logger.Warn(ErrorCode.Dispatcher_NoTargetActivation,
"No target activation {0} for response message: {1}", nonExistentActivation, message);
Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
}
}
catch (Exception exc)
{
// Unable to create activation for this request - reject message
RejectMessage(message, Message.RejectionTypes.Transient, exc);
}
}
}
internal async Task Invoke(IAddressable target, IInvokable invokable, Message message)
{
try
{
// Don't process messages that have already timed out
if (message.IsExpired)
{
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke);
return;
}
//MessagingProcessingStatisticsGroup.OnRequestProcessed(message, "Invoked");
if (Message.WriteMessagingTraces)
message.AddTimestamp(Message.LifecycleTag.InvokeIncoming);
RequestContext.ImportFromMessage(message);
if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget)
{
UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message));
// RequestContext is automatically saved in the msg upon send and propagated to the next hop
// in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message);
}
var invoker = invokable.GetInvoker(message.InterfaceId, message.GenericGrainType);
object resultObject;
try
{
var request = (InvokeMethodRequest) message.BodyObject;
if (invoker is IGrainExtensionMethodInvoker
&& !(target is IGrainExtension))
{
// We are trying the invoke a grain extension method on a grain
// -- most likely reason is that the dynamic extension is not installed for this grain
// So throw a specific exception here rather than a general InvalidCastException
var error = String.Format(
"Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}",
target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName);
var exc = new GrainExtensionNotInstalledException(error);
string extraDebugInfo = null;
#if DEBUG
extraDebugInfo = new StackTrace().ToString();
#endif
logger.Warn(ErrorCode.Stream_ExtensionNotInstalled,
string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc);
throw exc;
}
resultObject = await invoker.Invoke(target, request.InterfaceId, request.MethodId, request.Arguments);
}
catch (Exception exc1)
{
if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay)
{
invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message, exc1);
}
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc1);
}
return;
}
if (message.Direction == Message.Directions.OneWay) return;
SafeSendResponse(message, resultObject);
}
catch (Exception exc2)
{
logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2);
if (message.Direction != Message.Directions.OneWay)
SafeSendExceptionResponse(message, exc2);
}
}
internal async Task Invoke(IAddressable target, IInvokable invokable, Message message)
{
try
{
// Don't process messages that have already timed out
if (message.IsExpired)
{
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke);
return;
}
RequestContext.Import(message.RequestContextData);
if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget)
{
UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message));
// RequestContext is automatically saved in the msg upon send and propagated to the next hop
// in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message);
}
object resultObject;
try
{
var request = (InvokeMethodRequest)message.BodyObject;
var invoker = invokable.GetInvoker(request.InterfaceId, message.GenericGrainType);
if (invoker is IGrainExtensionMethodInvoker &&
!(target is IGrainExtension))
{
// We are trying the invoke a grain extension method on a grain
// -- most likely reason is that the dynamic extension is not installed for this grain
// So throw a specific exception here rather than a general InvalidCastException
var error = String.Format(
"Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}",
target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName);
var exc = new GrainExtensionNotInstalledException(error);
string extraDebugInfo = null;
#if DEBUG
extraDebugInfo = new StackTrace().ToString();
#endif
logger.Warn(ErrorCode.Stream_ExtensionNotInstalled,
string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc);
throw exc;
}
// If the target has a grain-level interceptor or there is a silo-level interceptor, intercept the call.
var shouldCallSiloWideInterceptor = SiloProviderRuntime.Instance.GetInvokeInterceptor() != null && target is IGrain;
var intercepted = target as IGrainInvokeInterceptor;
if (intercepted != null || shouldCallSiloWideInterceptor)
{
// Fetch the method info for the intercepted call.
var implementationInvoker =
invocationMethodInfoMap.GetInterceptedMethodInvoker(target.GetType(), request.InterfaceId,
invoker);
var methodInfo = implementationInvoker.GetMethodInfo(request.MethodId);
if (shouldCallSiloWideInterceptor)
{
// There is a silo-level interceptor and possibly a grain-level interceptor.
var runtime = SiloProviderRuntime.Instance;
resultObject =
await runtime.CallInvokeInterceptor(methodInfo, request, target, implementationInvoker);
}
else
{
// The grain has an interceptor, but there is no silo-wide interceptor.
resultObject = await intercepted.Invoke(methodInfo, request, invoker);
}
}
else
{
// The call is not intercepted.
// TODO: this await here is just grabbing the first exception of the AggregateException. As a framework we shouldn't do that.
resultObject = await invoker.Invoke(target, request);
}
}
catch (Exception exc1)
{
if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay)
{
invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message, exc1);
}
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc1);
}
return;
}
if (message.Direction == Message.Directions.OneWay)
{
return;
}
SafeSendResponse(message, resultObject);
}
catch (Exception exc2)
{
logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2);
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc2);
}
}
}
/// <summary>
/// Handles the an InvokeMethodRequest message on given target.
/// </summary>
/// <remarks>
/// Options when we receive a InvokeMethodRequest
/// ----------------------------------------------
/// 1) Intercepted RPC
/// 2) Request to start a reactive computation for this method invocation
/// 3) KeepAlive request of the reactive computation for this method invocation
/// 4) Normal Application RPC
/// 5) System RPC
/// </remarks>
/// <returns></returns>
internal async Task Invoke(IAddressable target, IInvokable invokable, Message message)
{
try
{
// Don't process messages that have already timed out
if (message.IsExpired)
{
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke);
return;
}
RequestContext.Import(message.RequestContextData);
if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget)
{
UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message));
// RequestContext is automatically saved in the msg upon send and propagated to the next hop
// in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message);
}
object resultObject;
try
{
var request = (InvokeMethodRequest)message.BodyObject;
if (request.Arguments != null)
{
CancellationSourcesExtension.RegisterCancellationTokens(target, request, logger);
}
// Get the invoker for this invocation
IGrainMethodInvoker invoker = GetGrainMethodInvoker(target, invokable, message, request);
// Check whether this call should be intercepted
var siloWideInterceptor = SiloProviderRuntime.Instance.GetInvokeInterceptor();
var grainWithInterceptor = target as IGrainInvokeInterceptor;
// Silo-wide interceptors do not operate on system targets.
var hasSiloWideInterceptor = siloWideInterceptor != null && target is IGrain;
var hasGrainLevelInterceptor = grainWithInterceptor != null;
// Normal Application Call = an application call made from grain to grain or from client to grain.
// Some system-related calls such as the IReminderTable are encoded as Application messages, therefore we needed this check.
var NormalApplicationCall = message.Category == Message.Categories.Application && message.TargetGrain.IsGrain && (message.SendingGrain.IsGrain || message.SendingGrain.IsClient);
// 1) If the target has a grain-level interceptor or there is a silo-level interceptor, intercept the
// call.
if (hasGrainLevelInterceptor || hasSiloWideInterceptor)
{
resultObject = await InvokeWithInterceptors(target, request, invoker, hasSiloWideInterceptor, siloWideInterceptor, hasGrainLevelInterceptor, grainWithInterceptor);
}
else if (NormalApplicationCall)
{
// 2) Request to start a reactive computation for this method invocation
if (message.IsRcExecute())
{
HandleReactiveComputationExecute(target, request, message, invoker);
return; // Does not expect a return (OneWay Message)
}
// 3) Refresh Reactive Computation Subscription
else if (message.IsRcKeepAlive())
{
HandleReactiveComputationExecute(target, request, message, invoker, true);
return; // Does not expect a return (OneWay Message)
}
// 4) Normal application RPC call
else
{
// Invoke the method
resultObject = await invoker.Invoke(target, request);
// Check if there are any active reactive computations in this grain that require recomputation after this call
InsideRcManager.RecomputeSummaries();
}
}
// 5) System RPC call
else
{
resultObject = await invoker.Invoke(target, request);
}
}
catch (Exception exc1)
{
if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay)
{
invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message, exc1);
}
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc1);
}
return;
}
if (message.Direction == Message.Directions.OneWay)
{
return;
}
SafeSendResponse(message, resultObject);
}
catch (Exception exc2)
{
logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2);
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc2);
}
}
}
