internal RequestInvocationHistory(Message message) { GrainId = message.TargetGrain; ActivationId = message.TargetActivation; InterfaceId = message.InterfaceId; MethodId = message.MethodId; }
internal static void OnDispatcherMessageReceive(Message msg) { ISchedulingContext context = RuntimeContext.Current != null ? RuntimeContext.Current.ActivationContext : null; dispatcherMessagesProcessingReceivedPerDirection[(int)msg.Direction].Increment(); dispatcherMessagesReceivedTotal.Increment(); if (context == null) { dispatcherReceivedByContext[0].Increment(); } else if (context.ContextType == SchedulingContextType.Activation) { dispatcherReceivedByContext[1].Increment(); } }
public Message Message { get; set; } // might hold metadata used by response pipeline public CallbackData(Action<Message, TaskCompletionSource<object>> callback, Func<Message, bool> resendFunc, TaskCompletionSource<object> ctx, Message msg, Action unregisterDelegate, Action<CallbackData> onTimeout = null) { // We are never called without a callback func, but best to double check. if (callback == null) throw new ArgumentNullException("callback"); // We are never called without a resend func, but best to double check. if (resendFunc == null) throw new ArgumentNullException("resendFunc"); this.callback = callback; this.resendFunc = resendFunc; this.context = ctx; this.Message = msg; this.unregister = unregisterDelegate; this.alreadyFired = false; this.onTimeout = onTimeout; }
public Task Pong(IDestination @from, Message message) { pongs++; if (pings < repeats) { actor.Ping(this, msg); pings++; } else if (pongs >= repeats) { subscribers.Notify(x => x.Done(pings, pongs)); } return TaskDone.Done; }
internal static Message CreateMessage(InvokeMethodRequest request, InvokeMethodOptions options) { var message = new Message( Message.Categories.Application, (options & InvokeMethodOptions.OneWay) != 0 ? Message.Directions.OneWay : Message.Directions.Request) { Id = CorrelationId.GetNext(), InterfaceId = request.InterfaceId, MethodId = request.MethodId, IsReadOnly = (options & InvokeMethodOptions.ReadOnly) != 0, IsUnordered = (options & InvokeMethodOptions.Unordered) != 0, BodyObject = request }; if ((options & InvokeMethodOptions.AlwaysInterleave) != 0) message.IsAlwaysInterleave = true; RequestContext.ExportToMessage(message); return message; }
public Message Message { get; set; } // might hold metadata used by response pipeline public CallbackData( Action<Message, TaskCompletionSource<object>> callback, Func<Message, bool> resendFunc, TaskCompletionSource<object> ctx, Message msg, Action<Message> unregisterDelegate, IMessagingConfiguration config) { // We are never called without a callback func, but best to double check. if (callback == null) throw new ArgumentNullException(nameof(callback)); // We are never called without a resend func, but best to double check. if (resendFunc == null) throw new ArgumentNullException(nameof(resendFunc)); this.callback = callback; this.resendFunc = resendFunc; context = ctx; Message = msg; unregister = unregisterDelegate; alreadyFired = false; this.config = config; }
public void HandleResponse(Message response) { running = response; InsideRuntimeClient.Current.ReceiveResponse(response); }
private bool TryResendMessage(Message message) { if (!message.MayResend(config)) { return false; } if (logger.IsVerbose) logger.Verbose("Resend {0}", message); message.ResendCount = message.ResendCount + 1; message.SetMetadata(Message.Metadata.TARGET_HISTORY, message.GetTargetHistory()); if (!message.TargetGrain.IsSystemTarget) { message.RemoveHeader(Message.Header.TARGET_ACTIVATION); message.RemoveHeader(Message.Header.TARGET_SILO); } transport.SendMessage(message); return true; }
/// <summary> /// Determine if the activation is able to currently accept the given message /// - always accept responses /// For other messages, require that: /// - activation is properly initialized /// - the message would not cause a reentrancy conflict /// </summary> /// <param name="targetActivation"></param> /// <param name="incoming"></param> /// <returns></returns> private bool ActivationMayAcceptRequest(ActivationData targetActivation, Message incoming) { if (targetActivation.State != ActivationState.Valid) return false; if (!targetActivation.IsCurrentlyExecuting) return true; return CanInterleave(targetActivation, incoming); }
public Message WaitMessage(Message.Categories type, CancellationToken ct) { try { if (ct.IsCancellationRequested) { return null; } // Don't pass CancellationToken to Take. It causes too much spinning. Message msg = PendingInboundMessages.Take(); #if TRACK_DETAILED_STATS if (StatisticsCollector.CollectQueueStats) { queueTracking.OnDeQueueRequest(msg); } #endif return msg; } #if !NETSTANDARD catch (ThreadAbortException exc) { // Silo may be shutting-down, so downgrade to verbose log logger.Verbose(ErrorCode.ProxyClient_ThreadAbort, "Received thread abort exception -- exiting. {0}", exc); Thread.ResetAbort(); return null; } #endif catch (OperationCanceledException exc) { logger.Verbose(ErrorCode.ProxyClient_OperationCancelled, "Received operation cancelled exception -- exiting. {0}", exc); return null; } catch (ObjectDisposedException exc) { logger.Verbose(ErrorCode.ProxyClient_OperationCancelled, "Received Object Disposed exception -- exiting. {0}", exc); return null; } catch (InvalidOperationException exc) { logger.Verbose(ErrorCode.ProxyClient_OperationCancelled, "Received Invalid Operation exception -- exiting. {0}", exc); return null; } catch (Exception ex) { logger.Error(ErrorCode.ProxyClient_ReceiveError, "Unexpected error getting an inbound message", ex); return null; } }
/// <summary> /// Directly send a message to the transport without processing /// </summary> /// <param name="message"></param> /// <param name="sendingActivation"></param> public void TransportMessage(Message message, IGrainContext sendingActivation = null) { MarkSameCallChainMessageAsInterleaving(sendingActivation, message); if (logger.IsEnabled(LogLevel.Trace)) logger.Trace(ErrorCode.Dispatcher_Send_AddressedMessage, "Addressed message {0}", message); Transport.SendMessage(message); }
public void DoCallback(Message response) { if (alreadyFired) return; lock (this) { if (alreadyFired) return; if (response.Result == Message.ResponseTypes.Rejection && response.RejectionType == Message.RejectionTypes.Transient) { if (resendFunc(Message)) { return; } } alreadyFired = true; DisposeTimer(); if (StatisticsCollector.CollectApplicationRequestsStats) { timeSinceIssued.Stop(); } if (unregister != null) { unregister(); } } if (StatisticsCollector.CollectApplicationRequestsStats) { ApplicationRequestsStatisticsGroup.OnAppRequestsEnd(timeSinceIssued.Elapsed); } // do callback outside the CallbackData lock. Just not a good practice to hold a lock for this unrelated operation. callback(response, context); }
public void SendMessage(Message msg) { GatewayConnection gatewayConnection = null; bool startRequired = false; // If there's a specific gateway specified, use it if (msg.TargetSilo != null) { Uri addr = msg.TargetSilo.ToGatewayUri(); lock (lockable) { if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive) { gatewayConnection = new GatewayConnection(addr, this); gatewayConnections[addr] = gatewayConnection; if (logger.IsVerbose) logger.Verbose("Creating gateway to {0} for pre-addressed message", addr); startRequired = true; } } } // For untargeted messages to system targets, and for unordered messages, pick a next connection in round robin fashion. else if (msg.TargetGrain.IsSystemTarget || msg.IsUnordered) { // Get the cached list of live gateways. // Pick a next gateway name in a round robin fashion. // See if we have a live connection to it. // If Yes, use it. // If not, create a new GatewayConnection and start it. // If start fails, we will mark this connection as dead and remove it from the GetCachedLiveGatewayNames. lock (lockable) { int msgNumber = numMessages; numMessages = unchecked(numMessages + 1); IList<Uri> gatewayNames = GatewayManager.GetLiveGateways(); int numGateways = gatewayNames.Count; if (numGateways == 0) { RejectMessage(msg, "No gateways available"); logger.Warn(ErrorCode.ProxyClient_CannotSend, "Unable to send message {0}; gateway manager state is {1}", msg, GatewayManager); return; } Uri addr = gatewayNames[msgNumber % numGateways]; if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive) { gatewayConnection = new GatewayConnection(addr, this); gatewayConnections[addr] = gatewayConnection; if (logger.IsVerbose) logger.Verbose(ErrorCode.ProxyClient_CreatedGatewayUnordered, "Creating gateway to {0} for unordered message to grain {1}", addr, msg.TargetGrain); startRequired = true; } // else - Fast path - we've got a live gatewayConnection to use } } // Otherwise, use the buckets to ensure ordering. else { var index = msg.TargetGrain.GetHashCode_Modulo((uint)grainBuckets.Length); lock (lockable) { // Repeated from above, at the declaration of the grainBuckets array: // Requests are bucketed by GrainID, so that all requests to a grain get routed through the same bucket. // Each bucket holds a (possibly null) weak reference to a GatewayConnection object. That connection instance is used // if the WeakReference is non-null, is alive, and points to a live gateway connection. If any of these conditions is // false, then a new gateway is selected using the gateway manager, and a new connection established if necessary. var weakRef = grainBuckets[index]; if ((weakRef != null) && weakRef.IsAlive) { gatewayConnection = weakRef.Target as GatewayConnection; } if ((gatewayConnection == null) || !gatewayConnection.IsLive) { var addr = GatewayManager.GetLiveGateway(); if (addr == null) { RejectMessage(msg, "No gateways available"); logger.Warn(ErrorCode.ProxyClient_CannotSend_NoGateway, "Unable to send message {0}; gateway manager state is {1}", msg, GatewayManager); return; } if (logger.IsVerbose2) logger.Verbose2(ErrorCode.ProxyClient_NewBucketIndex, "Starting new bucket index {0} for ordered messages to grain {1}", index, msg.TargetGrain); if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive) { gatewayConnection = new GatewayConnection(addr, this); gatewayConnections[addr] = gatewayConnection; if (logger.IsVerbose) logger.Verbose(ErrorCode.ProxyClient_CreatedGatewayToGrain, "Creating gateway to {0} for message to grain {1}, bucket {2}, grain id hash code {3}X", addr, msg.TargetGrain, index, msg.TargetGrain.GetHashCode().ToString("x")); startRequired = true; } grainBuckets[index] = new WeakReference(gatewayConnection); } } } if (startRequired) { gatewayConnection.Start(); if (!gatewayConnection.IsLive) { // if failed to start Gateway connection (failed to connect), try sending this msg to another Gateway. RejectOrResend(msg); return; } } try { gatewayConnection.QueueRequest(msg); if (logger.IsVerbose2) logger.Verbose2(ErrorCode.ProxyClient_QueueRequest, "Sending message {0} via gateway {1}", msg, gatewayConnection.Address); } catch (InvalidOperationException) { // This exception can be thrown if the gateway connection we selected was closed since we checked (i.e., we lost the race) // If this happens, we reject if the message is targeted to a specific silo, or try again if not RejectOrResend(msg); } }
// Forwarding is used by the receiver, usually when it cannot process the message and forwards it to another silo to perform the processing // (got here due to duplicate activation, outdated cache, silo is shutting down/overloaded, ...). private static bool MayForward(Message message, SiloMessagingOptions messagingOptions) { return message.ForwardCount < messagingOptions.MaxForwardCount // allow one more forward hop for multi-cluster case + (message.IsReturnedFromRemoteCluster ? 1 : 0); }
/// <summary> /// https://github.com/dotnet/orleans/issues/3184 /// Checks whether reentrancy is allowed for calls to grains that are already part of the call chain. /// Covers following case: grain A calls grain B, and while executing the invoked method B calls back to A. /// Design: Senders collection `RunningRequestsSenders` contains sending grains references /// during duration of request processing. If target of outgoing request is found in that collection - /// such request will be marked as interleaving in order to prevent deadlocks. /// </summary> private void MarkSameCallChainMessageAsInterleaving(IGrainContext sendingActivation, Message outgoing) { if (!schedulingOptions.AllowCallChainReentrancy) { return; } if ((sendingActivation as ActivationData)?.RunningRequestsSenders.Contains(outgoing.TargetActivation) == true) { outgoing.IsAlwaysInterleave = true; } }
internal async Task Invoke(IAddressable target, IInvokable invokable, Message message) { try { // Don't process messages that have already timed out if (message.IsExpired) { message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke); return; } RequestContext.Import(message.RequestContextData); if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget) { UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message)); // RequestContext is automatically saved in the msg upon send and propagated to the next hop // in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message); } object resultObject; try { var request = (InvokeMethodRequest) message.BodyObject; if (request.Arguments != null) { CancellationSourcesExtension.RegisterCancellationTokens(target, request, logger, this); } var invoker = invokable.GetInvoker(typeManager, request.InterfaceId, message.GenericGrainType); if (invoker is IGrainExtensionMethodInvoker && !(target is IGrainExtension)) { // We are trying the invoke a grain extension method on a grain // -- most likely reason is that the dynamic extension is not installed for this grain // So throw a specific exception here rather than a general InvalidCastException var error = String.Format( "Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}", target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName); var exc = new GrainExtensionNotInstalledException(error); string extraDebugInfo = null; #if DEBUG extraDebugInfo = Utils.GetStackTrace(); #endif logger.Warn(ErrorCode.Stream_ExtensionNotInstalled, string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc); throw exc; } resultObject = await InvokeWithInterceptors(target, request, invoker); } catch (Exception exc1) { if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay) { invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException, "Exception during Grain method call of message: " + message, exc1); } if (message.Direction != Message.Directions.OneWay) { SafeSendExceptionResponse(message, exc1); } return; } if (message.Direction == Message.Directions.OneWay) return; SafeSendResponse(message, resultObject); } catch (Exception exc2) { logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2); if (message.Direction != Message.Directions.OneWay) SafeSendExceptionResponse(message, exc2); } }
// this is a compatibility method for portions of the code base that don't use // async/await yet, which is almost everything. there's no liability to discarding the // Task returned by AsyncSendMessage() internal void SendMessage(Message message, IGrainContext sendingActivation = null) { AsyncSendMessage(message, sendingActivation).Ignore(); }
/// <summary> /// Reroute a message coming in through a gateway /// </summary> /// <param name="message"></param> internal void RerouteMessage(Message message) { ResendMessageImpl(message); }
private void SendRequestMessage( GrainReference target, Message message, TaskCompletionSource<object> context, Action<Message, TaskCompletionSource<object>> callback, string debugContext, InvokeMethodOptions options, string genericArguments = null) { // fill in sender if (message.SendingSilo == null) message.SendingSilo = MySilo; if (!String.IsNullOrEmpty(genericArguments)) message.GenericGrainType = genericArguments; SchedulingContext schedulingContext = RuntimeContext.Current != null ? RuntimeContext.Current.ActivationContext as SchedulingContext : null; ActivationData sendingActivation = null; if (schedulingContext == null) { throw new InvalidOperationException( String.Format("Trying to send a message {0} on a silo not from within grain and not from within system target (RuntimeContext is not set to SchedulingContext) " + "RuntimeContext.Current={1} TaskScheduler.Current={2}", message, RuntimeContext.Current == null ? "null" : RuntimeContext.Current.ToString(), TaskScheduler.Current)); } switch (schedulingContext.ContextType) { case SchedulingContextType.SystemThread: throw new ArgumentException( String.Format("Trying to send a message {0} on a silo not from within grain and not from within system target (RuntimeContext is of SchedulingContextType.SystemThread type)", message), "context"); case SchedulingContextType.Activation: message.SendingActivation = schedulingContext.Activation.ActivationId; message.SendingGrain = schedulingContext.Activation.Grain; sendingActivation = schedulingContext.Activation; break; case SchedulingContextType.SystemTarget: message.SendingActivation = schedulingContext.SystemTarget.ActivationId; message.SendingGrain = ((ISystemTargetBase)schedulingContext.SystemTarget).GrainId; break; } // fill in destination var targetGrainId = target.GrainId; message.TargetGrain = targetGrainId; if (targetGrainId.IsSystemTarget) { SiloAddress targetSilo = (target.SystemTargetSilo ?? MySilo); message.TargetSilo = targetSilo; message.TargetActivation = ActivationId.GetSystemActivation(targetGrainId, targetSilo); message.Category = targetGrainId.Equals(Constants.MembershipOracleId) ? Message.Categories.Ping : Message.Categories.System; } if (target.IsObserverReference) { message.TargetObserverId = target.ObserverId; } if (debugContext != null) message.DebugContext = debugContext; var oneWay = (options & InvokeMethodOptions.OneWay) != 0; if (context == null && !oneWay) logger.Warn(ErrorCode.IGC_SendRequest_NullContext, "Null context {0}: {1}", message, Utils.GetStackTrace()); if (message.IsExpirableMessage(Config.Globals)) message.Expiration = DateTime.UtcNow + ResponseTimeout + Constants.MAXIMUM_CLOCK_SKEW; if (!oneWay) { var callbackData = new CallbackData( callback, tryResendMessage, context, message, unregisterCallback, Config.Globals); callbacks.TryAdd(message.Id, callbackData); callbackData.StartTimer(ResponseTimeout); } if (targetGrainId.IsSystemTarget) { // Messages to system targets bypass the task system and get sent "in-line" dispatcher.TransportMessage(message); } else { dispatcher.SendMessage(message, sendingActivation); } }
private async Task AddressMessageAsync(Message message, PlacementTarget target, PlacementStrategy strategy, ActivationAddress targetAddress) { var placementResult = await placementDirectorsManager.SelectOrAddActivation(target, (IPlacementRuntime)this.catalog, strategy); SetMessageTargetPlacement(message, placementResult, targetAddress); }
private bool TryResendMessage(Message message) { if (!message.MayResend(config)) { return false; } if (logger.IsVerbose) logger.Verbose("Resend {0}", message); message.ResendCount = message.ResendCount + 1; message.TargetHistory = message.GetTargetHistory(); if (!message.TargetGrain.IsSystemTarget) { message.TargetActivation = null; message.TargetSilo = null; message.ClearTargetAddress(); } transport.SendMessage(message); return true; }
private void RejectMessage(Message msg, string reasonFormat, params object[] reasonParams) { if (!Running) return; var reason = String.Format(reasonFormat, reasonParams); if (msg.Direction != Message.Directions.Request) { if (logger.IsVerbose) logger.Verbose(ErrorCode.ProxyClient_DroppingMsg, "Dropping message: {0}. Reason = {1}", msg, reason); } else { if (logger.IsVerbose) logger.Verbose(ErrorCode.ProxyClient_RejectingMsg, "Rejecting message: {0}. Reason = {1}", msg, reason); MessagingStatisticsGroup.OnRejectedMessage(msg); Message error = msg.CreateRejectionResponse(Message.RejectionTypes.Unrecoverable, reason); QueueIncomingMessage(error); } }
public bool TryDecodeMessage(out Message msg) { msg = null; // Is there enough read into the buffer to continue (at least read the lengths?) if (receiveOffset - decodeOffset < CalculateKnownMessageSize()) return false; // parse lengths if needed if (headerLength == 0 || bodyLength == 0) { // get length segments List<ArraySegment<byte>> lenghts = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, decodeOffset, Message.LENGTH_HEADER_SIZE); // copy length segment to buffer int lengthBufferoffset = 0; foreach (ArraySegment<byte> seg in lenghts) { Buffer.BlockCopy(seg.Array, seg.Offset, lengthBuffer, lengthBufferoffset, seg.Count); lengthBufferoffset += seg.Count; } // read lengths headerLength = BitConverter.ToInt32(lengthBuffer, 0); bodyLength = BitConverter.ToInt32(lengthBuffer, 4); } // If message is too big for current buffer size, grow while (decodeOffset + CalculateKnownMessageSize() > currentBufferSize) { GrowBuffer(); } // Is there enough read into the buffer to read full message if (receiveOffset - decodeOffset < CalculateKnownMessageSize()) return false; // decode header int headerOffset = decodeOffset + Message.LENGTH_HEADER_SIZE; List<ArraySegment<byte>> header = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, headerOffset, headerLength); // decode body int bodyOffset = headerOffset + headerLength; List<ArraySegment<byte>> body = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, bodyOffset, bodyLength); // need to maintain ownership of buffer, so if we are supporting forwarding we need to duplicate the body buffer. if (supportForwarding) { body = DuplicateBuffer(body); } // build message msg = new Message(header, body, !supportForwarding); MessagingStatisticsGroup.OnMessageReceive(msg, headerLength, bodyLength); if (headerLength + bodyLength > Message.LargeMessageSizeThreshold) { Log.Info(ErrorCode.Messaging_LargeMsg_Incoming, "Receiving large message Size={0} HeaderLength={1} BodyLength={2}. Msg={3}", headerLength + bodyLength, headerLength, bodyLength, msg.ToString()); if (Log.IsVerbose3) Log.Verbose3("Received large message {0}", msg.ToLongString()); } // update parse receiveOffset and clear lengths decodeOffset = bodyOffset + bodyLength; headerLength = 0; bodyLength = 0; AdjustBuffer(); return true; }
private void InvokeLocalObjectAsync(LocalObjectData objectData, Message message) { var obj = (IAddressable)objectData.LocalObject.Target; if (obj == null) { //// Remove from the dictionary record for the garbage collected object? But now we won't be able to detect invalid dispatch IDs anymore. logger.Warn(ErrorCode.Runtime_Error_100162, String.Format("Object associated with Observer ID {0} has been garbage collected. Deleting object reference and unregistering it. Message = {1}", objectData.ObserverId, message)); LocalObjectData ignore; // Try to remove. If it's not there, we don't care. localObjects.TryRemove(objectData.ObserverId, out ignore); return; } bool start; lock (objectData.Messages) { objectData.Messages.Enqueue(message); start = !objectData.Running; objectData.Running = true; } if (logger.IsVerbose) logger.Verbose("InvokeLocalObjectAsync {0} start {1}", message, start); if (start) { // we use Task.Run() to ensure that the message pump operates asynchronously // with respect to the current thread. see // http://channel9.msdn.com/Events/TechEd/Europe/2013/DEV-B317#fbid=aIWUq0ssW74 // at position 54:45. // // according to the information posted at: // http://stackoverflow.com/questions/12245935/is-task-factory-startnew-guaranteed-to-use-another-thread-than-the-calling-thr // this idiom is dependent upon the a TaskScheduler not implementing the // override QueueTask as task inlining (as opposed to queueing). this seems // implausible to the author, since none of the .NET schedulers do this and // it is considered bad form (the OrleansTaskScheduler does not do this). // // if, for some reason this doesn't hold true, we can guarantee what we // want by passing a placeholder continuation token into Task.StartNew() // instead. i.e.: // // return Task.StartNew(() => ..., new CancellationToken()); Func<Task> asyncFunc = async () => await this.LocalObjectMessagePumpAsync(objectData); Task.Run(asyncFunc).Ignore(); } }
private void OnFail(Message msg, Message error, string resendLogMessageFormat, bool isOnTimeout = false) { lock (this) { if (alreadyFired) return; if (config.ResendOnTimeout && resendFunc(msg)) { if (logger.IsVerbose) logger.Verbose(resendLogMessageFormat, msg.ResendCount, msg); return; } alreadyFired = true; DisposeTimer(); if (StatisticsCollector.CollectApplicationRequestsStats) { timeSinceIssued.Stop(); } if (unregister != null) { unregister(); } } if (StatisticsCollector.CollectApplicationRequestsStats) { ApplicationRequestsStatisticsGroup.OnAppRequestsEnd(timeSinceIssued.Elapsed); if (isOnTimeout) { ApplicationRequestsStatisticsGroup.OnAppRequestsTimedOut(); } } callback(error, context); }
SendResponseAsync( Message message, object resultObject) { if (ExpireMessageIfExpired(message, MessagingStatisticsGroup.Phase.Respond)) return TaskDone.Done; object deepCopy = null; try { // we're expected to notify the caller if the deep copy failed. deepCopy = SerializationManager.DeepCopy(resultObject); } catch (Exception exc2) { SendResponse(message, Response.ExceptionResponse(exc2)); logger.Warn( ErrorCode.ProxyClient_OGC_SendResponseFailed, "Exception trying to send a response.", exc2); return TaskDone.Done; } // the deep-copy succeeded. SendResponse(message, new Response(deepCopy)); return TaskDone.Done; }
private void RejectOrResend(Message msg) { if (msg.TargetSilo != null) { RejectMessage(msg, String.Format("Target silo {0} is unavailable", msg.TargetSilo)); } else { SendMessage(msg); } }
private void SendResponse(Message request, Response response) { var message = request.CreateResponseMessage(); message.BodyObject = response; transport.SendMessage(message); }
internal void QueueIncomingMessage(Message msg) { #if TRACK_DETAILED_STATS if (StatisticsCollector.CollectQueueStats) { queueTracking.OnEnQueueRequest(1, PendingInboundMessages.Count, msg); } #endif PendingInboundMessages.Add(msg); }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { this.messagingTrace.OnDispatcherReceiveMessage(message); // Don't process messages that have already timed out if (message.IsExpired) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message); this.messagingTrace.OnDropExpiredMessage(message, MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget()) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType, message.RequestContextData, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { if (target.State == ActivationState.Valid) { this.activationCollector.TryRescheduleCollection(target); } // Silo is always capable to accept a new request. It's up to the activation to handle its internal state. // If activation is shutting down, it will queue and later forward this request. ReceiveRequest(message, target); } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = $"Error creating activation for {message.NewGrainType}. Message {message}"; logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } if (nea.IsStatelessWorker) { if (logger.IsEnabled(LogLevel.Debug)) logger.Debug(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, $"Intermediate StatelessWorker NonExistentActivation for message {message}, Exception {ex}"); } else { logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, $"Intermediate NonExistentActivation for message {message}, with Exception {ex}"); } ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context var origin = message.SendingSilo; scheduler.QueueAction( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async () => { try { if (this.logger.IsEnabled(LogLevel.Trace)) logger.Trace("UnregisterAfterNonexistingActivation addr={ActivationAddress} origin={SiloAddress}", nonExistentActivation, origin); await this.grainLocator.Unregister(nonExistentActivation, UnregistrationCause.NonexistentActivation); } catch (Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, $"Failed to un-register NonExistentActivation {nonExistentActivation}", exc); } }, catalog); ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn( ErrorCode.Dispatcher_NoTargetActivation, nonExistentActivation.Silo.IsClient ? "No target client {0} for response message: {1}. It's likely that the client recently disconnected." : "No target activation {0} for response message: {1}", nonExistentActivation, message); this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }
private void DispatchToLocalObject(Message message) { LocalObjectData objectData; GuidId observerId = message.TargetObserverId; if (observerId == null) { logger.Error( ErrorCode.ProxyClient_OGC_TargetNotFound_2, String.Format("Did not find TargetObserverId header in the message = {0}. A request message to a client is expected to have an observerId.", message)); return; } if (localObjects.TryGetValue(observerId, out objectData)) this.InvokeLocalObjectAsync(objectData, message); else { logger.Error( ErrorCode.ProxyClient_OGC_TargetNotFound, String.Format( "Unexpected target grain in request: {0}. Message={1}", message.TargetGrain, message)); } }
private static void OnMessageSend_Impl(SiloAddress targetSilo, Message.Directions direction, int numTotalBytes, int headerBytes, int numMsgsInBatch) { MessagesSentTotal.IncrementBy(numMsgsInBatch); MessagesSentPerDirection[(int)direction].IncrementBy(numMsgsInBatch); TotalBytesSent.IncrementBy(numTotalBytes); HeaderBytesSent.IncrementBy(headerBytes); sentMsgSizeHistogram.AddData(numTotalBytes); FindCounter(perSiloSendCounters, new StatisticName(StatisticNames.MESSAGING_SENT_MESSAGES_PER_SILO, (targetSilo != null ? targetSilo.ToString() : "Null")), CounterStorage.LogOnly).IncrementBy(numMsgsInBatch); }
private static bool ExpireMessageIfExpired(Message message, MessagingStatisticsGroup.Phase phase) { if (message.IsExpired) { message.DropExpiredMessage(phase); return true; } return false; }
internal static void OnMessageReceive(Message msg, int headerBytes, int bodyBytes) { MessagesReceived.Increment(); MessagesReceivedPerDirection[(int)msg.Direction].Increment(); totalBytesReceived.IncrementBy(headerBytes + bodyBytes); headerBytesReceived.IncrementBy(headerBytes); receiveMsgSizeHistogram.AddData(headerBytes + bodyBytes); SiloAddress addr = msg.SendingSilo; FindCounter(perSiloReceiveCounters, new StatisticName(StatisticNames.MESSAGING_RECEIVED_MESSAGES_PER_SILO, (addr != null ? addr.ToString() : "Null")), CounterStorage.LogOnly).Increment(); }
private void ReportException(Message message, Exception exception) { var request = (InvokeMethodRequest)message.BodyObject; switch (message.Direction) { default: throw new InvalidOperationException(); case Message.Directions.OneWay: { logger.Error( ErrorCode.ProxyClient_OGC_UnhandledExceptionInOneWayInvoke, String.Format( "Exception during invocation of notification method {0}, interface {1}. Ignoring exception because this is a one way request.", request.MethodId, request.InterfaceId), exception); break; } case Message.Directions.Request: { Exception deepCopy = null; try { // we're expected to notify the caller if the deep copy failed. deepCopy = (Exception)SerializationManager.DeepCopy(exception); } catch (Exception ex2) { SendResponse(message, Response.ExceptionResponse(ex2)); logger.Warn( ErrorCode.ProxyClient_OGC_SendExceptionResponseFailed, "Exception trying to send an exception response", ex2); return; } // the deep-copy succeeded. var response = Response.ExceptionResponse(deepCopy); SendResponse(message, response); break; } } }
internal static void OnRejectedMessage(Message msg) { if (msg == null || !msg.ContainsHeader(Message.Header.DIRECTION)) return; int direction = (int)msg.Direction; if (RejectedMessages[direction] == null) { RejectedMessages[direction] = CounterStatistic.FindOrCreate( new StatisticName(StatisticNames.MESSAGING_REJECTED_PER_DIRECTION, Enum.GetName(typeof(Message.Directions), direction))); } RejectedMessages[direction].Increment(); }
private void SendRequestMessage(GrainReference target, Message message, TaskCompletionSource<object> context, Action<Message, TaskCompletionSource<object>> callback, string debugContext = null, InvokeMethodOptions options = InvokeMethodOptions.None, string genericArguments = null) { var targetGrainId = target.GrainId; var oneWay = (options & InvokeMethodOptions.OneWay) != 0; message.SendingGrain = CurrentActivationAddress.Grain; message.SendingActivation = CurrentActivationAddress.Activation; message.TargetGrain = targetGrainId; if (!String.IsNullOrEmpty(genericArguments)) message.GenericGrainType = genericArguments; if (targetGrainId.IsSystemTarget) { // If the silo isn't be supplied, it will be filled in by the sender to be the gateway silo message.TargetSilo = target.SystemTargetSilo; if (target.SystemTargetSilo != null) { message.TargetActivation = ActivationId.GetSystemActivation(targetGrainId, target.SystemTargetSilo); } } // Client sending messages to another client (observer). Yes, we support that. if (target.IsObserverReference) { message.TargetObserverId = target.ObserverId; } if (debugContext != null) { message.DebugContext = debugContext; } if (message.IsExpirableMessage(config)) { // don't set expiration for system target messages. message.Expiration = DateTime.UtcNow + responseTimeout + Constants.MAXIMUM_CLOCK_SKEW; } if (!oneWay) { var callbackData = new CallbackData(callback, TryResendMessage, context, message, () => UnRegisterCallback(message.Id), config); callbacks.TryAdd(message.Id, callbackData); callbackData.StartTimer(responseTimeout); } if (logger.IsVerbose2) logger.Verbose2("Send {0}", message); transport.SendMessage(message); }
internal static void OnMessageReRoute(Message msg) { ReroutedMessages[(int)msg.Direction].Increment(); }
public void ReceiveResponse(Message response) { if (logger.IsVerbose2) logger.Verbose2("Received {0}", response); // ignore duplicate requests if (response.Result == Message.ResponseTypes.Rejection && response.RejectionType == Message.RejectionTypes.DuplicateRequest) return; CallbackData callbackData; var found = callbacks.TryGetValue(response.Id, out callbackData); if (found) { // We need to import the RequestContext here as well. // Unfortunately, it is not enough, since CallContext.LogicalGetData will not flow "up" from task completion source into the resolved task. // RequestContext.Import(response.RequestContextData); callbackData.DoCallback(response); } else { logger.Warn(ErrorCode.Runtime_Error_100011, "No callback for response message: " + response); } }
public void HandleNewRequest(Message request) { running = request; InsideRuntimeClient.Current.Invoke(this, this, request).Ignore(); }