internal RequestInvocationHistory(Message message)
{
GrainId = message.TargetGrain;
ActivationId = message.TargetActivation;
InterfaceId = message.InterfaceId;
MethodId = message.MethodId;
}
internal static void OnDispatcherMessageReceive(Message msg)
{
ISchedulingContext context = RuntimeContext.Current != null ? RuntimeContext.Current.ActivationContext : null;
dispatcherMessagesProcessingReceivedPerDirection[(int)msg.Direction].Increment();
dispatcherMessagesReceivedTotal.Increment();
if (context == null)
{
dispatcherReceivedByContext[0].Increment();
}
else if (context.ContextType == SchedulingContextType.Activation)
{
dispatcherReceivedByContext[1].Increment();
}
}
public Message Message { get; set; } // might hold metadata used by response pipeline
public CallbackData(Action<Message, TaskCompletionSource<object>> callback, Func<Message, bool> resendFunc, TaskCompletionSource<object> ctx, Message msg, Action unregisterDelegate, Action<CallbackData> onTimeout = null)
{
// We are never called without a callback func, but best to double check.
if (callback == null) throw new ArgumentNullException("callback");
// We are never called without a resend func, but best to double check.
if (resendFunc == null) throw new ArgumentNullException("resendFunc");
this.callback = callback;
this.resendFunc = resendFunc;
this.context = ctx;
this.Message = msg;
this.unregister = unregisterDelegate;
this.alreadyFired = false;
this.onTimeout = onTimeout;
}
public Task Pong(IDestination @from, Message message)
{
pongs++;
if (pings < repeats)
{
actor.Ping(this, msg);
pings++;
}
else if (pongs >= repeats)
{
subscribers.Notify(x => x.Done(pings, pongs));
}
return TaskDone.Done;
}
internal static Message CreateMessage(InvokeMethodRequest request, InvokeMethodOptions options)
{
var message = new Message(
Message.Categories.Application,
(options & InvokeMethodOptions.OneWay) != 0 ? Message.Directions.OneWay : Message.Directions.Request)
{
Id = CorrelationId.GetNext(),
InterfaceId = request.InterfaceId,
MethodId = request.MethodId,
IsReadOnly = (options & InvokeMethodOptions.ReadOnly) != 0,
IsUnordered = (options & InvokeMethodOptions.Unordered) != 0,
BodyObject = request
};
if ((options & InvokeMethodOptions.AlwaysInterleave) != 0)
message.IsAlwaysInterleave = true;
RequestContext.ExportToMessage(message);
return message;
}
public Message Message { get; set; } // might hold metadata used by response pipeline
public CallbackData(
Action<Message, TaskCompletionSource<object>> callback,
Func<Message, bool> resendFunc,
TaskCompletionSource<object> ctx,
Message msg,
Action<Message> unregisterDelegate,
IMessagingConfiguration config)
{
// We are never called without a callback func, but best to double check.
if (callback == null) throw new ArgumentNullException(nameof(callback));
// We are never called without a resend func, but best to double check.
if (resendFunc == null) throw new ArgumentNullException(nameof(resendFunc));
this.callback = callback;
this.resendFunc = resendFunc;
context = ctx;
Message = msg;
unregister = unregisterDelegate;
alreadyFired = false;
this.config = config;
}
public void HandleResponse(Message response)
{
running = response;
InsideRuntimeClient.Current.ReceiveResponse(response);
}
private bool TryResendMessage(Message message)
{
if (!message.MayResend(config))
{
return false;
}
if (logger.IsVerbose) logger.Verbose("Resend {0}", message);
message.ResendCount = message.ResendCount + 1;
message.SetMetadata(Message.Metadata.TARGET_HISTORY, message.GetTargetHistory());
if (!message.TargetGrain.IsSystemTarget)
{
message.RemoveHeader(Message.Header.TARGET_ACTIVATION);
message.RemoveHeader(Message.Header.TARGET_SILO);
}
transport.SendMessage(message);
return true;
}
/// <summary>
/// Determine if the activation is able to currently accept the given message
/// - always accept responses
/// For other messages, require that:
/// - activation is properly initialized
/// - the message would not cause a reentrancy conflict
/// </summary>
/// <param name="targetActivation"></param>
/// <param name="incoming"></param>
/// <returns></returns>
private bool ActivationMayAcceptRequest(ActivationData targetActivation, Message incoming)
{
if (targetActivation.State != ActivationState.Valid) return false;
if (!targetActivation.IsCurrentlyExecuting) return true;
return CanInterleave(targetActivation, incoming);
}
public Message WaitMessage(Message.Categories type, CancellationToken ct)
{
try
{
if (ct.IsCancellationRequested)
{
return null;
}
// Don't pass CancellationToken to Take. It causes too much spinning.
Message msg = PendingInboundMessages.Take();
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectQueueStats)
{
queueTracking.OnDeQueueRequest(msg);
}
#endif
return msg;
}
#if !NETSTANDARD
catch (ThreadAbortException exc)
{
// Silo may be shutting-down, so downgrade to verbose log
logger.Verbose(ErrorCode.ProxyClient_ThreadAbort, "Received thread abort exception -- exiting. {0}", exc);
Thread.ResetAbort();
return null;
}
#endif
catch (OperationCanceledException exc)
{
logger.Verbose(ErrorCode.ProxyClient_OperationCancelled, "Received operation cancelled exception -- exiting. {0}", exc);
return null;
}
catch (ObjectDisposedException exc)
{
logger.Verbose(ErrorCode.ProxyClient_OperationCancelled, "Received Object Disposed exception -- exiting. {0}", exc);
return null;
}
catch (InvalidOperationException exc)
{
logger.Verbose(ErrorCode.ProxyClient_OperationCancelled, "Received Invalid Operation exception -- exiting. {0}", exc);
return null;
}
catch (Exception ex)
{
logger.Error(ErrorCode.ProxyClient_ReceiveError, "Unexpected error getting an inbound message", ex);
return null;
}
}
/// <summary>
/// Directly send a message to the transport without processing
/// </summary>
/// <param name="message"></param>
/// <param name="sendingActivation"></param>
public void TransportMessage(Message message, IGrainContext sendingActivation = null)
{
MarkSameCallChainMessageAsInterleaving(sendingActivation, message);
if (logger.IsEnabled(LogLevel.Trace)) logger.Trace(ErrorCode.Dispatcher_Send_AddressedMessage, "Addressed message {0}", message);
Transport.SendMessage(message);
}
public void DoCallback(Message response)
{
if (alreadyFired)
return;
lock (this)
{
if (alreadyFired)
return;
if (response.Result == Message.ResponseTypes.Rejection && response.RejectionType == Message.RejectionTypes.Transient)
{
if (resendFunc(Message))
{
return;
}
}
alreadyFired = true;
DisposeTimer();
if (StatisticsCollector.CollectApplicationRequestsStats)
{
timeSinceIssued.Stop();
}
if (unregister != null)
{
unregister();
}
}
if (StatisticsCollector.CollectApplicationRequestsStats)
{
ApplicationRequestsStatisticsGroup.OnAppRequestsEnd(timeSinceIssued.Elapsed);
}
// do callback outside the CallbackData lock. Just not a good practice to hold a lock for this unrelated operation.
callback(response, context);
}
public void SendMessage(Message msg)
{
GatewayConnection gatewayConnection = null;
bool startRequired = false;
// If there's a specific gateway specified, use it
if (msg.TargetSilo != null)
{
Uri addr = msg.TargetSilo.ToGatewayUri();
lock (lockable)
{
if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive)
{
gatewayConnection = new GatewayConnection(addr, this);
gatewayConnections[addr] = gatewayConnection;
if (logger.IsVerbose) logger.Verbose("Creating gateway to {0} for pre-addressed message", addr);
startRequired = true;
}
}
}
// For untargeted messages to system targets, and for unordered messages, pick a next connection in round robin fashion.
else if (msg.TargetGrain.IsSystemTarget || msg.IsUnordered)
{
// Get the cached list of live gateways.
// Pick a next gateway name in a round robin fashion.
// See if we have a live connection to it.
// If Yes, use it.
// If not, create a new GatewayConnection and start it.
// If start fails, we will mark this connection as dead and remove it from the GetCachedLiveGatewayNames.
lock (lockable)
{
int msgNumber = numMessages;
numMessages = unchecked(numMessages + 1);
IList<Uri> gatewayNames = GatewayManager.GetLiveGateways();
int numGateways = gatewayNames.Count;
if (numGateways == 0)
{
RejectMessage(msg, "No gateways available");
logger.Warn(ErrorCode.ProxyClient_CannotSend, "Unable to send message {0}; gateway manager state is {1}", msg, GatewayManager);
return;
}
Uri addr = gatewayNames[msgNumber % numGateways];
if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive)
{
gatewayConnection = new GatewayConnection(addr, this);
gatewayConnections[addr] = gatewayConnection;
if (logger.IsVerbose) logger.Verbose(ErrorCode.ProxyClient_CreatedGatewayUnordered, "Creating gateway to {0} for unordered message to grain {1}", addr, msg.TargetGrain);
startRequired = true;
}
// else - Fast path - we've got a live gatewayConnection to use
}
}
// Otherwise, use the buckets to ensure ordering.
else
{
var index = msg.TargetGrain.GetHashCode_Modulo((uint)grainBuckets.Length);
lock (lockable)
{
// Repeated from above, at the declaration of the grainBuckets array:
// Requests are bucketed by GrainID, so that all requests to a grain get routed through the same bucket.
// Each bucket holds a (possibly null) weak reference to a GatewayConnection object. That connection instance is used
// if the WeakReference is non-null, is alive, and points to a live gateway connection. If any of these conditions is
// false, then a new gateway is selected using the gateway manager, and a new connection established if necessary.
var weakRef = grainBuckets[index];
if ((weakRef != null) && weakRef.IsAlive)
{
gatewayConnection = weakRef.Target as GatewayConnection;
}
if ((gatewayConnection == null) || !gatewayConnection.IsLive)
{
var addr = GatewayManager.GetLiveGateway();
if (addr == null)
{
RejectMessage(msg, "No gateways available");
logger.Warn(ErrorCode.ProxyClient_CannotSend_NoGateway, "Unable to send message {0}; gateway manager state is {1}", msg, GatewayManager);
return;
}
if (logger.IsVerbose2) logger.Verbose2(ErrorCode.ProxyClient_NewBucketIndex, "Starting new bucket index {0} for ordered messages to grain {1}", index, msg.TargetGrain);
if (!gatewayConnections.TryGetValue(addr, out gatewayConnection) || !gatewayConnection.IsLive)
{
gatewayConnection = new GatewayConnection(addr, this);
gatewayConnections[addr] = gatewayConnection;
if (logger.IsVerbose) logger.Verbose(ErrorCode.ProxyClient_CreatedGatewayToGrain, "Creating gateway to {0} for message to grain {1}, bucket {2}, grain id hash code {3}X", addr, msg.TargetGrain, index,
msg.TargetGrain.GetHashCode().ToString("x"));
startRequired = true;
}
grainBuckets[index] = new WeakReference(gatewayConnection);
}
}
}
if (startRequired)
{
gatewayConnection.Start();
if (!gatewayConnection.IsLive)
{
// if failed to start Gateway connection (failed to connect), try sending this msg to another Gateway.
RejectOrResend(msg);
return;
}
}
try
{
gatewayConnection.QueueRequest(msg);
if (logger.IsVerbose2) logger.Verbose2(ErrorCode.ProxyClient_QueueRequest, "Sending message {0} via gateway {1}", msg, gatewayConnection.Address);
}
catch (InvalidOperationException)
{
// This exception can be thrown if the gateway connection we selected was closed since we checked (i.e., we lost the race)
// If this happens, we reject if the message is targeted to a specific silo, or try again if not
RejectOrResend(msg);
}
}
// Forwarding is used by the receiver, usually when it cannot process the message and forwards it to another silo to perform the processing
// (got here due to duplicate activation, outdated cache, silo is shutting down/overloaded, ...).
private static bool MayForward(Message message, SiloMessagingOptions messagingOptions)
{
return message.ForwardCount < messagingOptions.MaxForwardCount
// allow one more forward hop for multi-cluster case
+ (message.IsReturnedFromRemoteCluster ? 1 : 0);
}
/// <summary>
/// https://github.com/dotnet/orleans/issues/3184
/// Checks whether reentrancy is allowed for calls to grains that are already part of the call chain.
/// Covers following case: grain A calls grain B, and while executing the invoked method B calls back to A.
/// Design: Senders collection `RunningRequestsSenders` contains sending grains references
/// during duration of request processing. If target of outgoing request is found in that collection -
/// such request will be marked as interleaving in order to prevent deadlocks.
/// </summary>
private void MarkSameCallChainMessageAsInterleaving(IGrainContext sendingActivation, Message outgoing)
{
if (!schedulingOptions.AllowCallChainReentrancy)
{
return;
}
if ((sendingActivation as ActivationData)?.RunningRequestsSenders.Contains(outgoing.TargetActivation) == true)
{
outgoing.IsAlwaysInterleave = true;
}
}
internal async Task Invoke(IAddressable target, IInvokable invokable, Message message)
{
try
{
// Don't process messages that have already timed out
if (message.IsExpired)
{
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke);
return;
}
RequestContext.Import(message.RequestContextData);
if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget)
{
UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message));
// RequestContext is automatically saved in the msg upon send and propagated to the next hop
// in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message);
}
object resultObject;
try
{
var request = (InvokeMethodRequest) message.BodyObject;
if (request.Arguments != null)
{
CancellationSourcesExtension.RegisterCancellationTokens(target, request, logger, this);
}
var invoker = invokable.GetInvoker(typeManager, request.InterfaceId, message.GenericGrainType);
if (invoker is IGrainExtensionMethodInvoker
&& !(target is IGrainExtension))
{
// We are trying the invoke a grain extension method on a grain
// -- most likely reason is that the dynamic extension is not installed for this grain
// So throw a specific exception here rather than a general InvalidCastException
var error = String.Format(
"Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}",
target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName);
var exc = new GrainExtensionNotInstalledException(error);
string extraDebugInfo = null;
#if DEBUG
extraDebugInfo = Utils.GetStackTrace();
#endif
logger.Warn(ErrorCode.Stream_ExtensionNotInstalled,
string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc);
throw exc;
}
resultObject = await InvokeWithInterceptors(target, request, invoker);
}
catch (Exception exc1)
{
if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay)
{
invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message, exc1);
}
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc1);
}
return;
}
if (message.Direction == Message.Directions.OneWay) return;
SafeSendResponse(message, resultObject);
}
catch (Exception exc2)
{
logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2);
if (message.Direction != Message.Directions.OneWay)
SafeSendExceptionResponse(message, exc2);
}
}
// this is a compatibility method for portions of the code base that don't use
// async/await yet, which is almost everything. there's no liability to discarding the
// Task returned by AsyncSendMessage()
internal void SendMessage(Message message, IGrainContext sendingActivation = null)
{
AsyncSendMessage(message, sendingActivation).Ignore();
}
/// <summary>
/// Reroute a message coming in through a gateway
/// </summary>
/// <param name="message"></param>
internal void RerouteMessage(Message message)
{
ResendMessageImpl(message);
}
private void SendRequestMessage(
GrainReference target,
Message message,
TaskCompletionSource<object> context,
Action<Message, TaskCompletionSource<object>> callback,
string debugContext,
InvokeMethodOptions options,
string genericArguments = null)
{
// fill in sender
if (message.SendingSilo == null)
message.SendingSilo = MySilo;
if (!String.IsNullOrEmpty(genericArguments))
message.GenericGrainType = genericArguments;
SchedulingContext schedulingContext = RuntimeContext.Current != null ?
RuntimeContext.Current.ActivationContext as SchedulingContext : null;
ActivationData sendingActivation = null;
if (schedulingContext == null)
{
throw new InvalidOperationException(
String.Format("Trying to send a message {0} on a silo not from within grain and not from within system target (RuntimeContext is not set to SchedulingContext) "
+ "RuntimeContext.Current={1} TaskScheduler.Current={2}",
message,
RuntimeContext.Current == null ? "null" : RuntimeContext.Current.ToString(),
TaskScheduler.Current));
}
switch (schedulingContext.ContextType)
{
case SchedulingContextType.SystemThread:
throw new ArgumentException(
String.Format("Trying to send a message {0} on a silo not from within grain and not from within system target (RuntimeContext is of SchedulingContextType.SystemThread type)", message), "context");
case SchedulingContextType.Activation:
message.SendingActivation = schedulingContext.Activation.ActivationId;
message.SendingGrain = schedulingContext.Activation.Grain;
sendingActivation = schedulingContext.Activation;
break;
case SchedulingContextType.SystemTarget:
message.SendingActivation = schedulingContext.SystemTarget.ActivationId;
message.SendingGrain = ((ISystemTargetBase)schedulingContext.SystemTarget).GrainId;
break;
}
// fill in destination
var targetGrainId = target.GrainId;
message.TargetGrain = targetGrainId;
if (targetGrainId.IsSystemTarget)
{
SiloAddress targetSilo = (target.SystemTargetSilo ?? MySilo);
message.TargetSilo = targetSilo;
message.TargetActivation = ActivationId.GetSystemActivation(targetGrainId, targetSilo);
message.Category = targetGrainId.Equals(Constants.MembershipOracleId) ?
Message.Categories.Ping : Message.Categories.System;
}
if (target.IsObserverReference)
{
message.TargetObserverId = target.ObserverId;
}
if (debugContext != null)
message.DebugContext = debugContext;
var oneWay = (options & InvokeMethodOptions.OneWay) != 0;
if (context == null && !oneWay)
logger.Warn(ErrorCode.IGC_SendRequest_NullContext, "Null context {0}: {1}", message, Utils.GetStackTrace());
if (message.IsExpirableMessage(Config.Globals))
message.Expiration = DateTime.UtcNow + ResponseTimeout + Constants.MAXIMUM_CLOCK_SKEW;
if (!oneWay)
{
var callbackData = new CallbackData(
callback,
tryResendMessage,
context,
message,
unregisterCallback,
Config.Globals);
callbacks.TryAdd(message.Id, callbackData);
callbackData.StartTimer(ResponseTimeout);
}
if (targetGrainId.IsSystemTarget)
{
// Messages to system targets bypass the task system and get sent "in-line"
dispatcher.TransportMessage(message);
}
else
{
dispatcher.SendMessage(message, sendingActivation);
}
}
private async Task AddressMessageAsync(Message message, PlacementTarget target, PlacementStrategy strategy, ActivationAddress targetAddress)
{
var placementResult = await placementDirectorsManager.SelectOrAddActivation(target, (IPlacementRuntime)this.catalog, strategy);
SetMessageTargetPlacement(message, placementResult, targetAddress);
}
private bool TryResendMessage(Message message)
{
if (!message.MayResend(config))
{
return false;
}
if (logger.IsVerbose) logger.Verbose("Resend {0}", message);
message.ResendCount = message.ResendCount + 1;
message.TargetHistory = message.GetTargetHistory();
if (!message.TargetGrain.IsSystemTarget)
{
message.TargetActivation = null;
message.TargetSilo = null;
message.ClearTargetAddress();
}
transport.SendMessage(message);
return true;
}
private void RejectMessage(Message msg, string reasonFormat, params object[] reasonParams)
{
if (!Running) return;
var reason = String.Format(reasonFormat, reasonParams);
if (msg.Direction != Message.Directions.Request)
{
if (logger.IsVerbose) logger.Verbose(ErrorCode.ProxyClient_DroppingMsg, "Dropping message: {0}. Reason = {1}", msg, reason);
}
else
{
if (logger.IsVerbose) logger.Verbose(ErrorCode.ProxyClient_RejectingMsg, "Rejecting message: {0}. Reason = {1}", msg, reason);
MessagingStatisticsGroup.OnRejectedMessage(msg);
Message error = msg.CreateRejectionResponse(Message.RejectionTypes.Unrecoverable, reason);
QueueIncomingMessage(error);
}
}
public bool TryDecodeMessage(out Message msg)
{
msg = null;
// Is there enough read into the buffer to continue (at least read the lengths?)
if (receiveOffset - decodeOffset < CalculateKnownMessageSize())
return false;
// parse lengths if needed
if (headerLength == 0 || bodyLength == 0)
{
// get length segments
List<ArraySegment<byte>> lenghts = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, decodeOffset, Message.LENGTH_HEADER_SIZE);
// copy length segment to buffer
int lengthBufferoffset = 0;
foreach (ArraySegment<byte> seg in lenghts)
{
Buffer.BlockCopy(seg.Array, seg.Offset, lengthBuffer, lengthBufferoffset, seg.Count);
lengthBufferoffset += seg.Count;
}
// read lengths
headerLength = BitConverter.ToInt32(lengthBuffer, 0);
bodyLength = BitConverter.ToInt32(lengthBuffer, 4);
}
// If message is too big for current buffer size, grow
while (decodeOffset + CalculateKnownMessageSize() > currentBufferSize)
{
GrowBuffer();
}
// Is there enough read into the buffer to read full message
if (receiveOffset - decodeOffset < CalculateKnownMessageSize())
return false;
// decode header
int headerOffset = decodeOffset + Message.LENGTH_HEADER_SIZE;
List<ArraySegment<byte>> header = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, headerOffset, headerLength);
// decode body
int bodyOffset = headerOffset + headerLength;
List<ArraySegment<byte>> body = ByteArrayBuilder.BuildSegmentListWithLengthLimit(readBuffer, bodyOffset, bodyLength);
// need to maintain ownership of buffer, so if we are supporting forwarding we need to duplicate the body buffer.
if (supportForwarding)
{
body = DuplicateBuffer(body);
}
// build message
msg = new Message(header, body, !supportForwarding);
MessagingStatisticsGroup.OnMessageReceive(msg, headerLength, bodyLength);
if (headerLength + bodyLength > Message.LargeMessageSizeThreshold)
{
Log.Info(ErrorCode.Messaging_LargeMsg_Incoming, "Receiving large message Size={0} HeaderLength={1} BodyLength={2}. Msg={3}",
headerLength + bodyLength, headerLength, bodyLength, msg.ToString());
if (Log.IsVerbose3) Log.Verbose3("Received large message {0}", msg.ToLongString());
}
// update parse receiveOffset and clear lengths
decodeOffset = bodyOffset + bodyLength;
headerLength = 0;
bodyLength = 0;
AdjustBuffer();
return true;
}
private void InvokeLocalObjectAsync(LocalObjectData objectData, Message message)
{
var obj = (IAddressable)objectData.LocalObject.Target;
if (obj == null)
{
//// Remove from the dictionary record for the garbage collected object? But now we won't be able to detect invalid dispatch IDs anymore.
logger.Warn(ErrorCode.Runtime_Error_100162,
String.Format("Object associated with Observer ID {0} has been garbage collected. Deleting object reference and unregistering it. Message = {1}", objectData.ObserverId, message));
LocalObjectData ignore;
// Try to remove. If it's not there, we don't care.
localObjects.TryRemove(objectData.ObserverId, out ignore);
return;
}
bool start;
lock (objectData.Messages)
{
objectData.Messages.Enqueue(message);
start = !objectData.Running;
objectData.Running = true;
}
if (logger.IsVerbose) logger.Verbose("InvokeLocalObjectAsync {0} start {1}", message, start);
if (start)
{
// we use Task.Run() to ensure that the message pump operates asynchronously
// with respect to the current thread. see
// http://channel9.msdn.com/Events/TechEd/Europe/2013/DEV-B317#fbid=aIWUq0ssW74
// at position 54:45.
//
// according to the information posted at:
// http://stackoverflow.com/questions/12245935/is-task-factory-startnew-guaranteed-to-use-another-thread-than-the-calling-thr
// this idiom is dependent upon the a TaskScheduler not implementing the
// override QueueTask as task inlining (as opposed to queueing). this seems
// implausible to the author, since none of the .NET schedulers do this and
// it is considered bad form (the OrleansTaskScheduler does not do this).
//
// if, for some reason this doesn't hold true, we can guarantee what we
// want by passing a placeholder continuation token into Task.StartNew()
// instead. i.e.:
//
// return Task.StartNew(() => ..., new CancellationToken());
Func<Task> asyncFunc =
async () =>
await this.LocalObjectMessagePumpAsync(objectData);
Task.Run(asyncFunc).Ignore();
}
}
private void OnFail(Message msg, Message error, string resendLogMessageFormat, bool isOnTimeout = false)
{
lock (this)
{
if (alreadyFired)
return;
if (config.ResendOnTimeout && resendFunc(msg))
{
if (logger.IsVerbose) logger.Verbose(resendLogMessageFormat, msg.ResendCount, msg);
return;
}
alreadyFired = true;
DisposeTimer();
if (StatisticsCollector.CollectApplicationRequestsStats)
{
timeSinceIssued.Stop();
}
if (unregister != null)
{
unregister();
}
}
if (StatisticsCollector.CollectApplicationRequestsStats)
{
ApplicationRequestsStatisticsGroup.OnAppRequestsEnd(timeSinceIssued.Elapsed);
if (isOnTimeout)
{
ApplicationRequestsStatisticsGroup.OnAppRequestsTimedOut();
}
}
callback(error, context);
}
SendResponseAsync(
Message message,
object resultObject)
{
if (ExpireMessageIfExpired(message, MessagingStatisticsGroup.Phase.Respond))
return TaskDone.Done;
object deepCopy = null;
try
{
// we're expected to notify the caller if the deep copy failed.
deepCopy = SerializationManager.DeepCopy(resultObject);
}
catch (Exception exc2)
{
SendResponse(message, Response.ExceptionResponse(exc2));
logger.Warn(
ErrorCode.ProxyClient_OGC_SendResponseFailed,
"Exception trying to send a response.", exc2);
return TaskDone.Done;
}
// the deep-copy succeeded.
SendResponse(message, new Response(deepCopy));
return TaskDone.Done;
}
private void RejectOrResend(Message msg)
{
if (msg.TargetSilo != null)
{
RejectMessage(msg, String.Format("Target silo {0} is unavailable", msg.TargetSilo));
}
else
{
SendMessage(msg);
}
}
private void SendResponse(Message request, Response response)
{
var message = request.CreateResponseMessage();
message.BodyObject = response;
transport.SendMessage(message);
}
internal void QueueIncomingMessage(Message msg)
{
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectQueueStats)
{
queueTracking.OnEnQueueRequest(1, PendingInboundMessages.Count, msg);
}
#endif
PendingInboundMessages.Add(msg);
}
/// <summary>
/// Receive a new message:
/// - validate order constraints, queue (or possibly redirect) if out of order
/// - validate transactions constraints
/// - invoke handler if ready, otherwise enqueue for later invocation
/// </summary>
/// <param name="message"></param>
public void ReceiveMessage(Message message)
{
this.messagingTrace.OnDispatcherReceiveMessage(message);
// Don't process messages that have already timed out
if (message.IsExpired)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message);
this.messagingTrace.OnDropExpiredMessage(message, MessagingStatisticsGroup.Phase.Dispatch);
return;
}
// check if its targeted at a new activation
if (message.TargetGrain.IsSystemTarget())
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message);
throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
}
try
{
Task ignore;
ActivationData target = catalog.GetOrCreateActivation(
message.TargetAddress,
message.IsNewPlacement,
message.NewGrainType,
String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType,
message.RequestContextData,
out ignore);
if (ignore != null)
{
ignore.Ignore();
}
if (message.Direction == Message.Directions.Response)
{
ReceiveResponse(message, target);
}
else // Request or OneWay
{
if (target.State == ActivationState.Valid)
{
this.activationCollector.TryRescheduleCollection(target);
}
// Silo is always capable to accept a new request. It's up to the activation to handle its internal state.
// If activation is shutting down, it will queue and later forward this request.
ReceiveRequest(message, target);
}
}
catch (Exception ex)
{
try
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message);
var nea = ex as Catalog.NonExistentActivationException;
if (nea == null)
{
var str = $"Error creating activation for {message.NewGrainType}. Message {message}";
logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
throw new OrleansException(str, ex);
}
if (nea.IsStatelessWorker)
{
if (logger.IsEnabled(LogLevel.Debug)) logger.Debug(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
$"Intermediate StatelessWorker NonExistentActivation for message {message}, Exception {ex}");
}
else
{
logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
$"Intermediate NonExistentActivation for message {message}, with Exception {ex}");
}
ActivationAddress nonExistentActivation = nea.NonExistentActivation;
if (message.Direction != Message.Directions.Response)
{
// Un-register the target activation so we don't keep getting spurious messages.
// The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
// this request snuck ahead of another request, with new placement requested, for the same activation.
// If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
// we want to make sure that we don't un-register the activation we just created.
// We would add a counter here, except that there's already a counter for this in the Catalog.
// Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
var origin = message.SendingSilo;
scheduler.QueueAction(
// don't use message.TargetAddress, cause it may have been removed from the headers by this time!
async () =>
{
try
{
if (this.logger.IsEnabled(LogLevel.Trace))
logger.Trace("UnregisterAfterNonexistingActivation addr={ActivationAddress} origin={SiloAddress}", nonExistentActivation, origin);
await this.grainLocator.Unregister(nonExistentActivation, UnregistrationCause.NonexistentActivation);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
$"Failed to un-register NonExistentActivation {nonExistentActivation}", exc);
}
},
catalog);
ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
}
else
{
logger.Warn(
ErrorCode.Dispatcher_NoTargetActivation,
nonExistentActivation.Silo.IsClient
? "No target client {0} for response message: {1}. It's likely that the client recently disconnected."
: "No target activation {0} for response message: {1}",
nonExistentActivation,
message);
this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
}
}
catch (Exception exc)
{
// Unable to create activation for this request - reject message
RejectMessage(message, Message.RejectionTypes.Transient, exc);
}
}
}
private void DispatchToLocalObject(Message message)
{
LocalObjectData objectData;
GuidId observerId = message.TargetObserverId;
if (observerId == null)
{
logger.Error(
ErrorCode.ProxyClient_OGC_TargetNotFound_2,
String.Format("Did not find TargetObserverId header in the message = {0}. A request message to a client is expected to have an observerId.", message));
return;
}
if (localObjects.TryGetValue(observerId, out objectData))
this.InvokeLocalObjectAsync(objectData, message);
else
{
logger.Error(
ErrorCode.ProxyClient_OGC_TargetNotFound,
String.Format(
"Unexpected target grain in request: {0}. Message={1}",
message.TargetGrain,
message));
}
}
private static void OnMessageSend_Impl(SiloAddress targetSilo, Message.Directions direction, int numTotalBytes, int headerBytes, int numMsgsInBatch)
{
MessagesSentTotal.IncrementBy(numMsgsInBatch);
MessagesSentPerDirection[(int)direction].IncrementBy(numMsgsInBatch);
TotalBytesSent.IncrementBy(numTotalBytes);
HeaderBytesSent.IncrementBy(headerBytes);
sentMsgSizeHistogram.AddData(numTotalBytes);
FindCounter(perSiloSendCounters, new StatisticName(StatisticNames.MESSAGING_SENT_MESSAGES_PER_SILO, (targetSilo != null ? targetSilo.ToString() : "Null")), CounterStorage.LogOnly).IncrementBy(numMsgsInBatch);
}
private static bool ExpireMessageIfExpired(Message message, MessagingStatisticsGroup.Phase phase)
{
if (message.IsExpired)
{
message.DropExpiredMessage(phase);
return true;
}
return false;
}
internal static void OnMessageReceive(Message msg, int headerBytes, int bodyBytes)
{
MessagesReceived.Increment();
MessagesReceivedPerDirection[(int)msg.Direction].Increment();
totalBytesReceived.IncrementBy(headerBytes + bodyBytes);
headerBytesReceived.IncrementBy(headerBytes);
receiveMsgSizeHistogram.AddData(headerBytes + bodyBytes);
SiloAddress addr = msg.SendingSilo;
FindCounter(perSiloReceiveCounters, new StatisticName(StatisticNames.MESSAGING_RECEIVED_MESSAGES_PER_SILO, (addr != null ? addr.ToString() : "Null")), CounterStorage.LogOnly).Increment();
}
private void ReportException(Message message, Exception exception)
{
var request = (InvokeMethodRequest)message.BodyObject;
switch (message.Direction)
{
default:
throw new InvalidOperationException();
case Message.Directions.OneWay:
{
logger.Error(
ErrorCode.ProxyClient_OGC_UnhandledExceptionInOneWayInvoke,
String.Format(
"Exception during invocation of notification method {0}, interface {1}. Ignoring exception because this is a one way request.",
request.MethodId,
request.InterfaceId),
exception);
break;
}
case Message.Directions.Request:
{
Exception deepCopy = null;
try
{
// we're expected to notify the caller if the deep copy failed.
deepCopy = (Exception)SerializationManager.DeepCopy(exception);
}
catch (Exception ex2)
{
SendResponse(message, Response.ExceptionResponse(ex2));
logger.Warn(
ErrorCode.ProxyClient_OGC_SendExceptionResponseFailed,
"Exception trying to send an exception response", ex2);
return;
}
// the deep-copy succeeded.
var response = Response.ExceptionResponse(deepCopy);
SendResponse(message, response);
break;
}
}
}
internal static void OnRejectedMessage(Message msg)
{
if (msg == null || !msg.ContainsHeader(Message.Header.DIRECTION)) return;
int direction = (int)msg.Direction;
if (RejectedMessages[direction] == null)
{
RejectedMessages[direction] = CounterStatistic.FindOrCreate(
new StatisticName(StatisticNames.MESSAGING_REJECTED_PER_DIRECTION, Enum.GetName(typeof(Message.Directions), direction)));
}
RejectedMessages[direction].Increment();
}
private void SendRequestMessage(GrainReference target, Message message, TaskCompletionSource<object> context, Action<Message, TaskCompletionSource<object>> callback, string debugContext = null, InvokeMethodOptions options = InvokeMethodOptions.None, string genericArguments = null)
{
var targetGrainId = target.GrainId;
var oneWay = (options & InvokeMethodOptions.OneWay) != 0;
message.SendingGrain = CurrentActivationAddress.Grain;
message.SendingActivation = CurrentActivationAddress.Activation;
message.TargetGrain = targetGrainId;
if (!String.IsNullOrEmpty(genericArguments))
message.GenericGrainType = genericArguments;
if (targetGrainId.IsSystemTarget)
{
// If the silo isn't be supplied, it will be filled in by the sender to be the gateway silo
message.TargetSilo = target.SystemTargetSilo;
if (target.SystemTargetSilo != null)
{
message.TargetActivation = ActivationId.GetSystemActivation(targetGrainId, target.SystemTargetSilo);
}
}
// Client sending messages to another client (observer). Yes, we support that.
if (target.IsObserverReference)
{
message.TargetObserverId = target.ObserverId;
}
if (debugContext != null)
{
message.DebugContext = debugContext;
}
if (message.IsExpirableMessage(config))
{
// don't set expiration for system target messages.
message.Expiration = DateTime.UtcNow + responseTimeout + Constants.MAXIMUM_CLOCK_SKEW;
}
if (!oneWay)
{
var callbackData = new CallbackData(callback, TryResendMessage, context, message, () => UnRegisterCallback(message.Id), config);
callbacks.TryAdd(message.Id, callbackData);
callbackData.StartTimer(responseTimeout);
}
if (logger.IsVerbose2) logger.Verbose2("Send {0}", message);
transport.SendMessage(message);
}
internal static void OnMessageReRoute(Message msg)
{
ReroutedMessages[(int)msg.Direction].Increment();
}
public void ReceiveResponse(Message response)
{
if (logger.IsVerbose2) logger.Verbose2("Received {0}", response);
// ignore duplicate requests
if (response.Result == Message.ResponseTypes.Rejection && response.RejectionType == Message.RejectionTypes.DuplicateRequest)
return;
CallbackData callbackData;
var found = callbacks.TryGetValue(response.Id, out callbackData);
if (found)
{
// We need to import the RequestContext here as well.
// Unfortunately, it is not enough, since CallContext.LogicalGetData will not flow "up" from task completion source into the resolved task.
// RequestContext.Import(response.RequestContextData);
callbackData.DoCallback(response);
}
else
{
logger.Warn(ErrorCode.Runtime_Error_100011, "No callback for response message: " + response);
}
}
public void HandleNewRequest(Message request)
{
running = request;
InsideRuntimeClient.Current.Invoke(this, this, request).Ignore();
}
