Exemplo n.º 1
0
        public void DoCallback(Message response)
        {
            if (this.IsCompleted)
            {
                return;
            }

            EventSourceUtils.EmitEvent(this.Message, OrleansCallBackDataEvent.DoCallbackAction);

            if (Interlocked.CompareExchange(ref this.completed, 1, 0) == 0)
            {
                var requestStatistics = this.shared.RequestStatistics;
                if (response.Result == Message.ResponseTypes.Rejection && response.RejectionType == Message.RejectionTypes.Transient)
                {
                    if (this.shared.ShouldResend(this.Message))
                    {
                        return;
                    }
                }

                if (requestStatistics.CollectApplicationRequestsStats)
                {
                    this.stopwatch.Stop();
                }

                if (requestStatistics.CollectApplicationRequestsStats)
                {
                    requestStatistics.OnAppRequestsEnd(this.stopwatch.Elapsed);
                }

                // do callback outside the CallbackData lock. Just not a good practice to hold a lock for this unrelated operation.
                this.shared.ResponseCallback(response, this.context);
            }
        }
Exemplo n.º 2
0
        public void SendResponse(Message request, Response response)
        {
            EventSourceUtils.EmitEvent(request, OrleansInsideRuntimeClientEvent.SendResponseAction);
            // Don't process messages that have already timed out
            if (request.IsExpired)
            {
                request.DropExpiredMessage(MessagingStatisticsGroup.Phase.Respond);
                return;
            }

            this.Dispatcher.SendResponse(request, response);
        }
Exemplo n.º 3
0
        public void SendRequest(
            GrainReference target,
            InvokeMethodRequest request,
            TaskCompletionSource <object> context,
            string debugContext,
            InvokeMethodOptions options,
            string genericArguments = null)
        {
            var message = this.messageFactory.CreateMessage(request, options);

            EventSourceUtils.EmitEvent(message, OrleansInsideRuntimeClientEvent.SendRequestAction);
            SendRequestMessage(target, message, context, debugContext, options, genericArguments);
        }
Exemplo n.º 4
0
        public void OnTargetSiloFail()
        {
            if (this.IsCompleted)
            {
                return;
            }
            EventSourceUtils.EmitEvent(this.Message, OrleansCallBackDataEvent.OnTargetSiloFailAction);
            var    msg            = this.Message;
            var    messageHistory = msg.GetTargetHistory();
            string errorMsg       =
                $"The target silo became unavailable for message: {msg}. Target History is: {messageHistory}. See {Constants.TroubleshootingHelpLink} for troubleshooting help.";

            this.shared.Logger.Warn(ErrorCode.Runtime_Error_100157, "{0} About to break its promise.", errorMsg);

            var error = Message.CreatePromptExceptionResponse(msg, new SiloUnavailableException(errorMsg));

            OnFail(msg, error, "On silo fail - Resend {0} for {1}");
        }
Exemplo n.º 5
0
        public void OnTimeout(TimeSpan timeout)
        {
            if (this.IsCompleted)
            {
                return;
            }

            EventSourceUtils.EmitEvent(this.Message, OrleansCallBackDataEvent.OnTimeoutAction);

            var msg = this.Message; // Local working copy

            string messageHistory = msg.GetTargetHistory();
            string errorMsg       = $"Response did not arrive on time in {timeout} for message: {msg}. Target History is: {messageHistory}.";

            this.shared.Logger.Warn(ErrorCode.Runtime_Error_100157, "{0} About to break its promise.", errorMsg);

            var error = Message.CreatePromptExceptionResponse(msg, new TimeoutException(errorMsg));

            OnFail(msg, error, "OnTimeout - Resend {0} for {1}", true);
        }
Exemplo n.º 6
0
        /// <summary>
        /// Receive a new message:
        /// - validate order constraints, queue (or possibly redirect) if out of order
        /// - validate transactions constraints
        /// - invoke handler if ready, otherwise enqueue for later invocation
        /// </summary>
        /// <param name="message"></param>
        public void ReceiveMessage(Message message)
        {
            EventSourceUtils.EmitEvent(message, OrleansDispatcherEvent.ReceiveMessageAction);
            MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message);
            // Don't process messages that have already timed out
            if (message.IsExpired)
            {
                logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message);
                MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired");
                message.DropExpiredMessage(this.logger, MessagingStatisticsGroup.Phase.Dispatch);
                return;
            }

            // check if its targeted at a new activation
            if (message.TargetGrain.IsSystemTarget)
            {
                MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target.");
                throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
            }

            try
            {
                Task           ignore;
                ActivationData target = catalog.GetOrCreateActivation(
                    message.TargetAddress,
                    message.IsNewPlacement,
                    message.NewGrainType,
                    String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType,
                    message.RequestContextData,
                    out ignore);

                if (ignore != null)
                {
                    ignore.Ignore();
                }

                if (message.Direction == Message.Directions.Response)
                {
                    ReceiveResponse(message, target);
                }
                else // Request or OneWay
                {
                    if (target.State == ActivationState.Valid)
                    {
                        this.activationCollector.TryRescheduleCollection(target);
                    }
                    // Silo is always capable to accept a new request. It's up to the activation to handle its internal state.
                    // If activation is shutting down, it will queue and later forward this request.
                    ReceiveRequest(message, target);
                }
            }
            catch (Exception ex)
            {
                try
                {
                    MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation");

                    var nea = ex as Catalog.NonExistentActivationException;
                    if (nea == null)
                    {
                        var str = $"Error creating activation for {message.NewGrainType}. Message {message}";
                        logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
                        throw new OrleansException(str, ex);
                    }

                    if (nea.IsStatelessWorker)
                    {
                        if (logger.IsEnabled(LogLevel.Debug))
                        {
                            logger.Debug(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
                                         $"Intermediate StatelessWorker NonExistentActivation for message {message}, Exception {ex}");
                        }
                    }
                    else
                    {
                        logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
                                    $"Intermediate NonExistentActivation for message {message}, with Exception {ex}");
                    }

                    ActivationAddress nonExistentActivation = nea.NonExistentActivation;

                    if (message.Direction != Message.Directions.Response)
                    {
                        // Un-register the target activation so we don't keep getting spurious messages.
                        // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
                        // this request snuck ahead of another request, with new placement requested, for the same activation.
                        // If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
                        // we want to make sure that we don't un-register the activation we just created.
                        // We would add a counter here, except that there's already a counter for this in the Catalog.
                        // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
                        var origin = message.SendingSilo;
                        scheduler.QueueWorkItem(new ClosureWorkItem(
                                                    // don't use message.TargetAddress, cause it may have been removed from the headers by this time!
                                                    async() =>
                        {
                            try
                            {
                                await this.localGrainDirectory.UnregisterAfterNonexistingActivation(
                                    nonExistentActivation, origin);
                            }
                            catch (Exception exc)
                            {
                                logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
                                            $"Failed to un-register NonExistentActivation {nonExistentActivation}", exc);
                            }
                        },
                                                    "LocalGrainDirectory.UnregisterAfterNonexistingActivation"),
                                                catalog.SchedulingContext);

                        ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
                    }
                    else
                    {
                        logger.Warn(
                            ErrorCode.Dispatcher_NoTargetActivation,
                            nonExistentActivation.Silo.IsClient
                                ? "No target client {0} for response message: {1}. It's likely that the client recently disconnected."
                                : "No target activation {0} for response message: {1}",
                            nonExistentActivation,
                            message);

                        this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
                    }
                }
                catch (Exception exc)
                {
                    // Unable to create activation for this request - reject message
                    RejectMessage(message, Message.RejectionTypes.Transient, exc);
                }
            }
        }
Exemplo n.º 7
0
        public void ReceiveResponse(Message message)
        {
            EventSourceUtils.EmitEvent(message, OrleansInsideRuntimeClientEvent.ReceiveResponseAction);
            if (message.Result == Message.ResponseTypes.Rejection)
            {
                if (!message.TargetSilo.Matches(this.MySilo))
                {
                    // gatewayed message - gateway back to sender
                    if (logger.IsEnabled(LogLevel.Trace))
                    {
                        logger.Trace(ErrorCode.Dispatcher_NoCallbackForRejectionResp, "No callback for rejection response message: {0}", message);
                    }
                    this.Dispatcher.Transport.SendMessage(message);
                    return;
                }

                if (logger.IsEnabled(LogLevel.Debug))
                {
                    logger.Debug(ErrorCode.Dispatcher_HandleMsg, "HandleMessage {0}", message);
                }
                switch (message.RejectionType)
                {
                case Message.RejectionTypes.DuplicateRequest:
                    // try to remove from callbackData, just in case it is still there.
                    break;

                case Message.RejectionTypes.Overloaded:
                    break;

                case Message.RejectionTypes.Unrecoverable:
                // fall through & reroute
                case Message.RejectionTypes.Transient:
                    if (message.CacheInvalidationHeader == null)
                    {
                        // Remove from local directory cache. Note that SendingGrain is the original target, since message is the rejection response.
                        // If CacheMgmtHeader is present, we already did this. Otherwise, we left this code for backward compatability.
                        // It should be retired as we move to use CacheMgmtHeader in all relevant places.
                        this.Directory.InvalidateCacheEntry(message.SendingAddress);
                    }
                    break;

                case Message.RejectionTypes.CacheInvalidation when message.HasCacheInvalidationHeader:
                    // The message targeted an invalid (eg, defunct) activation and this response serves only to invalidate this silo's activation cache.
                    return;

                default:
                    logger.Error(ErrorCode.Dispatcher_InvalidEnum_RejectionType,
                                 "Missing enum in switch: " + message.RejectionType);
                    break;
                }
            }

            CallbackData callbackData;
            bool         found = callbacks.TryRemove(message.Id, out callbackData);

            if (found)
            {
                if (message.TransactionInfo != null)
                {
                    // NOTE: Not clear if thread-safe, revise
                    callbackData.TransactionInfo.Join(message.TransactionInfo);
                }
                // IMPORTANT: we do not schedule the response callback via the scheduler, since the only thing it does
                // is to resolve/break the resolver. The continuations/waits that are based on this resolution will be scheduled as work items.
                callbackData.DoCallback(message);
            }
            else
            {
                if (logger.IsEnabled(LogLevel.Debug))
                {
                    logger.Debug(ErrorCode.Dispatcher_NoCallbackForResp,
                                 "No callback for response message: " + message);
                }
            }
        }