private void SendResponse(Message request, Response response) { // Don't process messages that have already timed out if (request.IsExpired) { request.DropExpiredMessage(MessagingStatisticsGroup.Phase.Respond); return; } this.Dispatcher.SendResponse(request, response); }
public void SendResponse(Message request, Response response) { EventSourceUtils.EmitEvent(request, OrleansInsideRuntimeClientEvent.SendResponseAction); // Don't process messages that have already timed out if (request.IsExpired) { request.DropExpiredMessage(MessagingStatisticsGroup.Phase.Respond); return; } this.Dispatcher.SendResponse(request, response); }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { EventSourceUtils.EmitEvent(message, OrleansDispatcherEvent.ReceiveMessageAction); MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message); // Don't process messages that have already timed out if (message.IsExpired) { logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired"); message.DropExpiredMessage(this.logger, MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target."); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType, message.RequestContextData, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { if (target.State == ActivationState.Valid) { this.activationCollector.TryRescheduleCollection(target); } // Silo is always capable to accept a new request. It's up to the activation to handle its internal state. // If activation is shutting down, it will queue and later forward this request. ReceiveRequest(message, target); } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = $"Error creating activation for {message.NewGrainType}. Message {message}"; logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } if (nea.IsStatelessWorker) { if (logger.IsEnabled(LogLevel.Debug)) { logger.Debug(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, $"Intermediate StatelessWorker NonExistentActivation for message {message}, Exception {ex}"); } } else { logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, $"Intermediate NonExistentActivation for message {message}, with Exception {ex}"); } ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context var origin = message.SendingSilo; scheduler.QueueWorkItem(new ClosureWorkItem( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async() => { try { await this.localGrainDirectory.UnregisterAfterNonexistingActivation( nonExistentActivation, origin); } catch (Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, $"Failed to un-register NonExistentActivation {nonExistentActivation}", exc); } }, "LocalGrainDirectory.UnregisterAfterNonexistingActivation"), catalog.SchedulingContext); ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn( ErrorCode.Dispatcher_NoTargetActivation, nonExistentActivation.Silo.IsClient ? "No target client {0} for response message: {1}. It's likely that the client recently disconnected." : "No target activation {0} for response message: {1}", nonExistentActivation, message); this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }
public async Task Invoke(IAddressable target, IInvokable invokable, Message message) { try { // Don't process messages that have already timed out if (message.IsExpired) { message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke); return; } RequestContext.Import(message.RequestContextData); if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) { UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message)); // RequestContext is automatically saved in the msg upon send and propagated to the next hop // in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message); } object resultObject; try { var request = (InvokeMethodRequest)message.GetDeserializedBody(this.SerializationManager); if (request.Arguments != null) { CancellationSourcesExtension.RegisterCancellationTokens(target, request, logger, this); } var invoker = invokable.GetInvoker(typeManager, request.InterfaceId, message.GenericGrainType); if (invoker is IGrainExtensionMethodInvoker && !(target is IGrainExtension)) { // We are trying the invoke a grain extension method on a grain // -- most likely reason is that the dynamic extension is not installed for this grain // So throw a specific exception here rather than a general InvalidCastException var error = String.Format( "Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}", target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName); var exc = new GrainExtensionNotInstalledException(error); string extraDebugInfo = null; #if DEBUG extraDebugInfo = Utils.GetStackTrace(); #endif logger.Warn(ErrorCode.Stream_ExtensionNotInstalled, string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc); throw exc; } #pragma warning disable 618 var invokeInterceptor = this.CurrentStreamProviderRuntime?.GetInvokeInterceptor(); #pragma warning restore 618 var requestInvoker = new GrainMethodInvoker(target, request, invoker, siloInterceptors, interfaceToImplementationMapping, invokeInterceptor); await requestInvoker.Invoke(); resultObject = requestInvoker.Result; } catch (Exception exc1) { if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay) { invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException, "Exception during Grain method call of message: " + message, exc1); } if (exc1 is InconsistentStateException && target is Grain) { var activation = ((Grain)target).Data; invokeExceptionLogger.Info($"Deactivating {activation} due to inconsistent state."); this.DeactivateOnIdle(activation.ActivationId); } if (message.Direction != Message.Directions.OneWay) { SafeSendExceptionResponse(message, exc1); } return; } if (message.Direction == Message.Directions.OneWay) { return; } SafeSendResponse(message, resultObject); } catch (Exception exc2) { logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2); if (message.Direction != Message.Directions.OneWay) { SafeSendExceptionResponse(message, exc2); } } }
private static bool ExpireMessageIfExpired(Message message, MessagingStatisticsGroup.Phase phase) { if (message.IsExpired) { message.DropExpiredMessage(phase); return true; } return false; }
internal async Task Invoke(IAddressable target, IInvokable invokable, Message message) { try { // Don't process messages that have already timed out if (message.IsExpired) { message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke); return; } //MessagingProcessingStatisticsGroup.OnRequestProcessed(message, "Invoked"); if (Message.WriteMessagingTraces) { message.AddTimestamp(Message.LifecycleTag.InvokeIncoming); } RequestContext.Import(message.RequestContextData); if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) { UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message)); // RequestContext is automatically saved in the msg upon send and propagated to the next hop // in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message); } var invoker = invokable.GetInvoker(message.InterfaceId, message.GenericGrainType); object resultObject; try { var request = (InvokeMethodRequest)message.BodyObject; if (invoker is IGrainExtensionMethodInvoker && !(target is IGrainExtension)) { // We are trying the invoke a grain extension method on a grain // -- most likely reason is that the dynamic extension is not installed for this grain // So throw a specific exception here rather than a general InvalidCastException var error = String.Format( "Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}", target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName); var exc = new GrainExtensionNotInstalledException(error); string extraDebugInfo = null; #if DEBUG extraDebugInfo = new StackTrace().ToString(); #endif logger.Warn(ErrorCode.Stream_ExtensionNotInstalled, string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc); throw exc; } resultObject = await invoker.Invoke(target, request.InterfaceId, request.MethodId, request.Arguments); } catch (Exception exc1) { if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay) { invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException, "Exception during Grain method call of message: " + message, exc1); } if (message.Direction != Message.Directions.OneWay) { SafeSendExceptionResponse(message, exc1); } return; } if (message.Direction == Message.Directions.OneWay) { return; } SafeSendResponse(message, resultObject); } catch (Exception exc2) { logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2); if (message.Direction != Message.Directions.OneWay) { SafeSendExceptionResponse(message, exc2); } } }
public async Task Invoke(IAddressable target, IInvokable invokable, Message message) { try { // Don't process messages that have already timed out if (message.IsExpired) { message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke); return; } RequestContextExtensions.Import(message.RequestContextData); if (schedulingOptions.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) { UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext)); // RequestContext is automatically saved in the msg upon send and propagated to the next hop // in RuntimeClient.CreateMessage -> RequestContextExtensions.ExportToMessage(message); } bool startNewTransaction = false; ITransactionInfo transactionInfo = message.TransactionInfo; if (message.IsTransactionRequired && transactionInfo == null) { // TODO: this should be a configurable parameter var transactionTimeout = Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(10); // Start a new transaction transactionInfo = await this.transactionAgent.Value.StartTransaction(message.IsReadOnly, transactionTimeout); startNewTransaction = true; } if (transactionInfo != null) { TransactionContext.SetTransactionInfo(transactionInfo); } object resultObject; try { var request = (InvokeMethodRequest)message.GetDeserializedBody(this.serializationManager); if (request.Arguments != null) { CancellationSourcesExtension.RegisterCancellationTokens(target, request, this.loggerFactory, logger, this, this.cancellationTokenRuntime); } var invoker = invokable.GetInvoker(typeManager, request.InterfaceId, message.GenericGrainType); if (invoker is IGrainExtensionMethodInvoker && !(target is IGrainExtension)) { // We are trying the invoke a grain extension method on a grain // -- most likely reason is that the dynamic extension is not installed for this grain // So throw a specific exception here rather than a general InvalidCastException var error = String.Format( "Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}", target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName); var exc = new GrainExtensionNotInstalledException(error); string extraDebugInfo = null; #if DEBUG extraDebugInfo = Utils.GetStackTrace(); #endif logger.Warn(ErrorCode.Stream_ExtensionNotInstalled, string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc); throw exc; } var requestInvoker = new GrainMethodInvoker(target, request, invoker, GrainCallFilters, interfaceToImplementationMapping); await requestInvoker.Invoke(); resultObject = requestInvoker.Result; } catch (Exception exc1) { if (invokeExceptionLogger.IsEnabled(LogLevel.Debug) || message.Direction == Message.Directions.OneWay) { invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException, "Exception during Grain method call of message: " + message, exc1); } transactionInfo = TransactionContext.GetTransactionInfo(); if (transactionInfo != null) { // Must abort the transaction on exceptions transactionInfo.IsAborted = true; if (startNewTransaction) { var abortException = (exc1 as OrleansTransactionAbortedException) ?? new OrleansTransactionAbortedException(transactionInfo.TransactionId.ToString(), exc1); this.transactionAgent.Value.Abort(transactionInfo, abortException); exc1 = abortException; } } // If a grain allowed an inconsistent state exception to escape and the exception originated from // this activation, then deactivate it. var ise = exc1 as InconsistentStateException; if (ise != null && ise.IsSourceActivation) { // Mark the exception so that it doesn't deactivate any other activations. ise.IsSourceActivation = false; var activation = (target as Grain)?.Data; if (activation != null) { invokeExceptionLogger.Info($"Deactivating {activation} due to inconsistent state."); this.DeactivateOnIdle(activation.ActivationId); } } if (message.Direction != Message.Directions.OneWay) { TransactionContext.Clear(); SafeSendExceptionResponse(message, exc1); } return; } transactionInfo = TransactionContext.GetTransactionInfo(); if (transactionInfo != null && !transactionInfo.ReconcilePending(out var numberOrphans)) { var abortException = new OrleansOrphanCallException(transactionInfo.TransactionId.ToString(), numberOrphans); // Can't exit before the transaction completes. TransactionContext.GetTransactionInfo().IsAborted = true; if (startNewTransaction) { this.transactionAgent.Value.Abort(TransactionContext.GetTransactionInfo(), abortException); } if (message.Direction != Message.Directions.OneWay) { TransactionContext.Clear(); SafeSendExceptionResponse(message, abortException); } return; } if (startNewTransaction) { // This request started the transaction, so we try to commit before returning. await this.transactionAgent.Value.Commit(transactionInfo); TransactionContext.Clear(); } if (message.Direction == Message.Directions.OneWay) { return; } SafeSendResponse(message, resultObject); } catch (Exception exc2) { logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2); try { if (exc2 is OrleansTransactionInDoubtException) { this.logger.LogError(exc2, "Transaction failed due to in doubt transaction"); } else if (TransactionContext.GetTransactionInfo() != null) { // Must abort the transaction on exceptions TransactionContext.GetTransactionInfo().IsAborted = true; var abortException = (exc2 as OrleansTransactionAbortedException) ?? new OrleansTransactionAbortedException(TransactionContext.GetTransactionInfo().TransactionId.ToString(), exc2); this.transactionAgent.Value.Abort(TransactionContext.GetTransactionInfo(), abortException); } } finally { TransactionContext.Clear(); if (message.Direction != Message.Directions.OneWay) { SafeSendExceptionResponse(message, exc2); } } } finally { TransactionContext.Clear(); } }
internal async Task Invoke(IAddressable target, IInvokable invokable, Message message) { try { // Don't process messages that have already timed out if (message.IsExpired) { message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke); return; } RequestContext.Import(message.RequestContextData); if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) { UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message)); // RequestContext is automatically saved in the msg upon send and propagated to the next hop // in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message); } object resultObject; try { var request = (InvokeMethodRequest) message.BodyObject; var invoker = invokable.GetInvoker(request.InterfaceId, message.GenericGrainType); if (invoker is IGrainExtensionMethodInvoker && !(target is IGrainExtension)) { // We are trying the invoke a grain extension method on a grain // -- most likely reason is that the dynamic extension is not installed for this grain // So throw a specific exception here rather than a general InvalidCastException var error = String.Format( "Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}", target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName); var exc = new GrainExtensionNotInstalledException(error); string extraDebugInfo = null; #if DEBUG extraDebugInfo = new StackTrace().ToString(); #endif logger.Warn(ErrorCode.Stream_ExtensionNotInstalled, string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc); throw exc; } // If the target has a grain-level interceptor or there is a silo-level interceptor, intercept the call. var shouldCallSiloWideInterceptor = SiloProviderRuntime.Instance.GetInvokeInterceptor() != null && target is IGrain; var intercepted = target as IGrainInvokeInterceptor; if (intercepted != null || shouldCallSiloWideInterceptor) { // Fetch the method info for the intercepted call. var implementationInvoker = invocationMethodInfoMap.GetInterceptedMethodInvoker(target.GetType(), request.InterfaceId, invoker); var methodInfo = implementationInvoker.GetMethodInfo(request.MethodId); if (shouldCallSiloWideInterceptor) { // There is a silo-level interceptor and possibly a grain-level interceptor. var runtime = SiloProviderRuntime.Instance; resultObject = await runtime.CallInvokeInterceptor(methodInfo, request, target, implementationInvoker); } else { // The grain has an interceptor, but there is no silo-wide interceptor. resultObject = await intercepted.Invoke(methodInfo, request, invoker); } } else { // The call is not intercepted. resultObject = await invoker.Invoke(target, request); } } catch (Exception exc1) { if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay) { invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException, "Exception during Grain method call of message: " + message, exc1); } if (message.Direction != Message.Directions.OneWay) { SafeSendExceptionResponse(message, exc1); } return; } if (message.Direction == Message.Directions.OneWay) return; SafeSendResponse(message, resultObject); } catch (Exception exc2) { logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2); if (message.Direction != Message.Directions.OneWay) SafeSendExceptionResponse(message, exc2); } }
private void SendResponse(Message request, Response response) { // Don't process messages that have already timed out if (request.IsExpired) { request.DropExpiredMessage(MessagingStatisticsGroup.Phase.Respond); return; } dispatcher.SendResponse(request, response); }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message); // Don't process messages that have already timed out if (message.IsExpired) { logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired"); message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target."); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } if (errorInjection && ShouldInjectError(message)) { if (logger.IsVerbose) { logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection"); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection"); RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection"); return; } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, message.GenericGrainType, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { if (SiloCanAcceptRequest(message)) { ReceiveRequest(message, target); } else if (message.MayResend(config.Globals)) { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Redirecting"); throw new NotImplementedException("RedirectRequest() is believed to be no longer necessary; please contact the Orleans team if you see this error."); } else { // Record that this message is no longer flowing through the system MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Rejecting"); RejectMessage(message, Message.RejectionTypes.Transient, null, "Shutting down"); } } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message); logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } logger.Warn(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, String.Format("Intermediate warning for NonExistentActivation from Catalog.GetOrCreateActivation for message {0}", message), ex); ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero) { Scheduler.QueueWorkItem(new ClosureWorkItem( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async() => { try { await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync( nonExistentActivation); } catch (Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, String.Format("Failed to un-register NonExistentActivation {0}", nonExistentActivation), exc); } }, () => "LocalGrainDirectory.UnregisterConditionallyAsync"), catalog.SchedulingContext); } ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn(ErrorCode.Dispatcher_NoTargetActivation, "No target activation {0} for response message: {1}", nonExistentActivation, message); Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message); // Don't process messages that have already timed out if (message.IsExpired) { logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired"); message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target."); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } if (errorInjection && ShouldInjectError(message)) { if (logger.IsVerbose) logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection"); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection"); RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection"); return; } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, message.GenericGrainType, message.RequestContextData, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { // Silo is always capable to accept a new request. It's up to the activation to handle its internal state. // If activation is shutting down, it will queue and later forward this request. ReceiveRequest(message, target); } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message); logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } if (nea.IsStatelessWorker) { if (logger.IsVerbose) logger.Verbose(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, String.Format("Intermediate StatelessWorker NonExistentActivation for message {0}", message), ex); } else { logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, String.Format("Intermediate NonExistentActivation for message {0}", message), ex); } ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero) { Scheduler.QueueWorkItem(new ClosureWorkItem( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async () => { try { await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync( nonExistentActivation); } catch(Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, String.Format("Failed to un-register NonExistentActivation {0}", nonExistentActivation), exc); } }, () => "LocalGrainDirectory.UnregisterConditionallyAsync"), catalog.SchedulingContext); } ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn(ErrorCode.Dispatcher_NoTargetActivation, "No target activation {0} for response message: {1}", nonExistentActivation, message); Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }
internal async Task Invoke(IAddressable target, IInvokable invokable, Message message) { try { // Don't process messages that have already timed out if (message.IsExpired) { message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke); return; } //MessagingProcessingStatisticsGroup.OnRequestProcessed(message, "Invoked"); if (Message.WriteMessagingTraces) message.AddTimestamp(Message.LifecycleTag.InvokeIncoming); RequestContext.ImportFromMessage(message); if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) { UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message)); // RequestContext is automatically saved in the msg upon send and propagated to the next hop // in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message); } var invoker = invokable.GetInvoker(message.InterfaceId, message.GenericGrainType); object resultObject; try { var request = (InvokeMethodRequest) message.BodyObject; if (invoker is IGrainExtensionMethodInvoker && !(target is IGrainExtension)) { // We are trying the invoke a grain extension method on a grain // -- most likely reason is that the dynamic extension is not installed for this grain // So throw a specific exception here rather than a general InvalidCastException var error = String.Format( "Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}", target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName); var exc = new GrainExtensionNotInstalledException(error); string extraDebugInfo = null; #if DEBUG extraDebugInfo = new StackTrace().ToString(); #endif logger.Warn(ErrorCode.Stream_ExtensionNotInstalled, string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc); throw exc; } resultObject = await invoker.Invoke(target, request.InterfaceId, request.MethodId, request.Arguments); } catch (Exception exc1) { if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay) { invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException, "Exception during Grain method call of message: " + message, exc1); } if (message.Direction != Message.Directions.OneWay) { SafeSendExceptionResponse(message, exc1); } return; } if (message.Direction == Message.Directions.OneWay) return; SafeSendResponse(message, resultObject); } catch (Exception exc2) { logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2); if (message.Direction != Message.Directions.OneWay) SafeSendExceptionResponse(message, exc2); } }
internal async Task Invoke(IAddressable target, IInvokable invokable, Message message) { try { // Don't process messages that have already timed out if (message.IsExpired) { message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke); return; } RequestContext.Import(message.RequestContextData); if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) { UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message)); // RequestContext is automatically saved in the msg upon send and propagated to the next hop // in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message); } object resultObject; try { var request = (InvokeMethodRequest)message.BodyObject; var invoker = invokable.GetInvoker(request.InterfaceId, message.GenericGrainType); if (invoker is IGrainExtensionMethodInvoker && !(target is IGrainExtension)) { // We are trying the invoke a grain extension method on a grain // -- most likely reason is that the dynamic extension is not installed for this grain // So throw a specific exception here rather than a general InvalidCastException var error = String.Format( "Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}", target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName); var exc = new GrainExtensionNotInstalledException(error); string extraDebugInfo = null; #if DEBUG extraDebugInfo = new StackTrace().ToString(); #endif logger.Warn(ErrorCode.Stream_ExtensionNotInstalled, string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc); throw exc; } // If the target has a grain-level interceptor or there is a silo-level interceptor, intercept the call. var shouldCallSiloWideInterceptor = SiloProviderRuntime.Instance.GetInvokeInterceptor() != null && target is IGrain; var intercepted = target as IGrainInvokeInterceptor; if (intercepted != null || shouldCallSiloWideInterceptor) { // Fetch the method info for the intercepted call. var implementationInvoker = invocationMethodInfoMap.GetInterceptedMethodInvoker(target.GetType(), request.InterfaceId, invoker); var methodInfo = implementationInvoker.GetMethodInfo(request.MethodId); if (shouldCallSiloWideInterceptor) { // There is a silo-level interceptor and possibly a grain-level interceptor. var runtime = SiloProviderRuntime.Instance; resultObject = await runtime.CallInvokeInterceptor(methodInfo, request, target, implementationInvoker); } else { // The grain has an interceptor, but there is no silo-wide interceptor. resultObject = await intercepted.Invoke(methodInfo, request, invoker); } } else { // The call is not intercepted. // TODO: this await here is just grabbing the first exception of the AggregateException. As a framework we shouldn't do that. resultObject = await invoker.Invoke(target, request); } } catch (Exception exc1) { if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay) { invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException, "Exception during Grain method call of message: " + message, exc1); } if (message.Direction != Message.Directions.OneWay) { SafeSendExceptionResponse(message, exc1); } return; } if (message.Direction == Message.Directions.OneWay) { return; } SafeSendResponse(message, resultObject); } catch (Exception exc2) { logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2); if (message.Direction != Message.Directions.OneWay) { SafeSendExceptionResponse(message, exc2); } } }
/// <summary> /// Handles the an InvokeMethodRequest message on given target. /// </summary> /// <remarks> /// Options when we receive a InvokeMethodRequest /// ---------------------------------------------- /// 1) Intercepted RPC /// 2) Request to start a reactive computation for this method invocation /// 3) KeepAlive request of the reactive computation for this method invocation /// 4) Normal Application RPC /// 5) System RPC /// </remarks> /// <returns></returns> internal async Task Invoke(IAddressable target, IInvokable invokable, Message message) { try { // Don't process messages that have already timed out if (message.IsExpired) { message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke); return; } RequestContext.Import(message.RequestContextData); if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) { UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message)); // RequestContext is automatically saved in the msg upon send and propagated to the next hop // in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message); } object resultObject; try { var request = (InvokeMethodRequest)message.BodyObject; if (request.Arguments != null) { CancellationSourcesExtension.RegisterCancellationTokens(target, request, logger); } // Get the invoker for this invocation IGrainMethodInvoker invoker = GetGrainMethodInvoker(target, invokable, message, request); // Check whether this call should be intercepted var siloWideInterceptor = SiloProviderRuntime.Instance.GetInvokeInterceptor(); var grainWithInterceptor = target as IGrainInvokeInterceptor; // Silo-wide interceptors do not operate on system targets. var hasSiloWideInterceptor = siloWideInterceptor != null && target is IGrain; var hasGrainLevelInterceptor = grainWithInterceptor != null; // Normal Application Call = an application call made from grain to grain or from client to grain. // Some system-related calls such as the IReminderTable are encoded as Application messages, therefore we needed this check. var NormalApplicationCall = message.Category == Message.Categories.Application && message.TargetGrain.IsGrain && (message.SendingGrain.IsGrain || message.SendingGrain.IsClient); // 1) If the target has a grain-level interceptor or there is a silo-level interceptor, intercept the // call. if (hasGrainLevelInterceptor || hasSiloWideInterceptor) { resultObject = await InvokeWithInterceptors(target, request, invoker, hasSiloWideInterceptor, siloWideInterceptor, hasGrainLevelInterceptor, grainWithInterceptor); } else if (NormalApplicationCall) { // 2) Request to start a reactive computation for this method invocation if (message.IsRcExecute()) { HandleReactiveComputationExecute(target, request, message, invoker); return; // Does not expect a return (OneWay Message) } // 3) Refresh Reactive Computation Subscription else if (message.IsRcKeepAlive()) { HandleReactiveComputationExecute(target, request, message, invoker, true); return; // Does not expect a return (OneWay Message) } // 4) Normal application RPC call else { // Invoke the method resultObject = await invoker.Invoke(target, request); // Check if there are any active reactive computations in this grain that require recomputation after this call InsideRcManager.RecomputeSummaries(); } } // 5) System RPC call else { resultObject = await invoker.Invoke(target, request); } } catch (Exception exc1) { if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay) { invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException, "Exception during Grain method call of message: " + message, exc1); } if (message.Direction != Message.Directions.OneWay) { SafeSendExceptionResponse(message, exc1); } return; } if (message.Direction == Message.Directions.OneWay) { return; } SafeSendResponse(message, resultObject); } catch (Exception exc2) { logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2); if (message.Direction != Message.Directions.OneWay) { SafeSendExceptionResponse(message, exc2); } } }