public void DoCallback(Message response)
{
if (this.IsCompleted)
{
return;
}
EventSourceUtils.EmitEvent(this.Message, OrleansCallBackDataEvent.DoCallbackAction);
if (Interlocked.CompareExchange(ref this.completed, 1, 0) == 0)
{
var requestStatistics = this.shared.RequestStatistics;
if (response.Result == Message.ResponseTypes.Rejection && response.RejectionType == Message.RejectionTypes.Transient)
{
if (this.shared.ShouldResend(this.Message))
{
return;
}
}
if (requestStatistics.CollectApplicationRequestsStats)
{
this.stopwatch.Stop();
}
if (requestStatistics.CollectApplicationRequestsStats)
{
requestStatistics.OnAppRequestsEnd(this.stopwatch.Elapsed);
}
// do callback outside the CallbackData lock. Just not a good practice to hold a lock for this unrelated operation.
this.shared.ResponseCallback(response, this.context);
}
}
public void SendResponse(Message request, Response response)
{
EventSourceUtils.EmitEvent(request, OrleansInsideRuntimeClientEvent.SendResponseAction);
// Don't process messages that have already timed out
if (request.IsExpired)
{
request.DropExpiredMessage(MessagingStatisticsGroup.Phase.Respond);
return;
}
this.Dispatcher.SendResponse(request, response);
}
public void SendRequest(
GrainReference target,
InvokeMethodRequest request,
TaskCompletionSource <object> context,
string debugContext,
InvokeMethodOptions options,
string genericArguments = null)
{
var message = this.messageFactory.CreateMessage(request, options);
EventSourceUtils.EmitEvent(message, OrleansInsideRuntimeClientEvent.SendRequestAction);
SendRequestMessage(target, message, context, debugContext, options, genericArguments);
}
public void OnTargetSiloFail()
{
if (this.IsCompleted)
{
return;
}
EventSourceUtils.EmitEvent(this.Message, OrleansCallBackDataEvent.OnTargetSiloFailAction);
var msg = this.Message;
var messageHistory = msg.GetTargetHistory();
string errorMsg =
$"The target silo became unavailable for message: {msg}. Target History is: {messageHistory}. See {Constants.TroubleshootingHelpLink} for troubleshooting help.";
this.shared.Logger.Warn(ErrorCode.Runtime_Error_100157, "{0} About to break its promise.", errorMsg);
var error = Message.CreatePromptExceptionResponse(msg, new SiloUnavailableException(errorMsg));
OnFail(msg, error, "On silo fail - Resend {0} for {1}");
}
public void OnTimeout(TimeSpan timeout)
{
if (this.IsCompleted)
{
return;
}
EventSourceUtils.EmitEvent(this.Message, OrleansCallBackDataEvent.OnTimeoutAction);
var msg = this.Message; // Local working copy
string messageHistory = msg.GetTargetHistory();
string errorMsg = $"Response did not arrive on time in {timeout} for message: {msg}. Target History is: {messageHistory}.";
this.shared.Logger.Warn(ErrorCode.Runtime_Error_100157, "{0} About to break its promise.", errorMsg);
var error = Message.CreatePromptExceptionResponse(msg, new TimeoutException(errorMsg));
OnFail(msg, error, "OnTimeout - Resend {0} for {1}", true);
}
/// <summary>
/// Receive a new message:
/// - validate order constraints, queue (or possibly redirect) if out of order
/// - validate transactions constraints
/// - invoke handler if ready, otherwise enqueue for later invocation
/// </summary>
/// <param name="message"></param>
public void ReceiveMessage(Message message)
{
EventSourceUtils.EmitEvent(message, OrleansDispatcherEvent.ReceiveMessageAction);
MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message);
// Don't process messages that have already timed out
if (message.IsExpired)
{
logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired");
message.DropExpiredMessage(this.logger, MessagingStatisticsGroup.Phase.Dispatch);
return;
}
// check if its targeted at a new activation
if (message.TargetGrain.IsSystemTarget)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target.");
throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
}
try
{
Task ignore;
ActivationData target = catalog.GetOrCreateActivation(
message.TargetAddress,
message.IsNewPlacement,
message.NewGrainType,
String.IsNullOrEmpty(message.GenericGrainType) ? null : message.GenericGrainType,
message.RequestContextData,
out ignore);
if (ignore != null)
{
ignore.Ignore();
}
if (message.Direction == Message.Directions.Response)
{
ReceiveResponse(message, target);
}
else // Request or OneWay
{
if (target.State == ActivationState.Valid)
{
this.activationCollector.TryRescheduleCollection(target);
}
// Silo is always capable to accept a new request. It's up to the activation to handle its internal state.
// If activation is shutting down, it will queue and later forward this request.
ReceiveRequest(message, target);
}
}
catch (Exception ex)
{
try
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation");
var nea = ex as Catalog.NonExistentActivationException;
if (nea == null)
{
var str = $"Error creating activation for {message.NewGrainType}. Message {message}";
logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
throw new OrleansException(str, ex);
}
if (nea.IsStatelessWorker)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
$"Intermediate StatelessWorker NonExistentActivation for message {message}, Exception {ex}");
}
}
else
{
logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
$"Intermediate NonExistentActivation for message {message}, with Exception {ex}");
}
ActivationAddress nonExistentActivation = nea.NonExistentActivation;
if (message.Direction != Message.Directions.Response)
{
// Un-register the target activation so we don't keep getting spurious messages.
// The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
// this request snuck ahead of another request, with new placement requested, for the same activation.
// If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
// we want to make sure that we don't un-register the activation we just created.
// We would add a counter here, except that there's already a counter for this in the Catalog.
// Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
var origin = message.SendingSilo;
scheduler.QueueWorkItem(new ClosureWorkItem(
// don't use message.TargetAddress, cause it may have been removed from the headers by this time!
async() =>
{
try
{
await this.localGrainDirectory.UnregisterAfterNonexistingActivation(
nonExistentActivation, origin);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
$"Failed to un-register NonExistentActivation {nonExistentActivation}", exc);
}
},
"LocalGrainDirectory.UnregisterAfterNonexistingActivation"),
catalog.SchedulingContext);
ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
}
else
{
logger.Warn(
ErrorCode.Dispatcher_NoTargetActivation,
nonExistentActivation.Silo.IsClient
? "No target client {0} for response message: {1}. It's likely that the client recently disconnected."
: "No target activation {0} for response message: {1}",
nonExistentActivation,
message);
this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
}
}
catch (Exception exc)
{
// Unable to create activation for this request - reject message
RejectMessage(message, Message.RejectionTypes.Transient, exc);
}
}
}
public void ReceiveResponse(Message message)
{
EventSourceUtils.EmitEvent(message, OrleansInsideRuntimeClientEvent.ReceiveResponseAction);
if (message.Result == Message.ResponseTypes.Rejection)
{
if (!message.TargetSilo.Matches(this.MySilo))
{
// gatewayed message - gateway back to sender
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace(ErrorCode.Dispatcher_NoCallbackForRejectionResp, "No callback for rejection response message: {0}", message);
}
this.Dispatcher.Transport.SendMessage(message);
return;
}
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug(ErrorCode.Dispatcher_HandleMsg, "HandleMessage {0}", message);
}
switch (message.RejectionType)
{
case Message.RejectionTypes.DuplicateRequest:
// try to remove from callbackData, just in case it is still there.
break;
case Message.RejectionTypes.Overloaded:
break;
case Message.RejectionTypes.Unrecoverable:
// fall through & reroute
case Message.RejectionTypes.Transient:
if (message.CacheInvalidationHeader == null)
{
// Remove from local directory cache. Note that SendingGrain is the original target, since message is the rejection response.
// If CacheMgmtHeader is present, we already did this. Otherwise, we left this code for backward compatability.
// It should be retired as we move to use CacheMgmtHeader in all relevant places.
this.Directory.InvalidateCacheEntry(message.SendingAddress);
}
break;
case Message.RejectionTypes.CacheInvalidation when message.HasCacheInvalidationHeader:
// The message targeted an invalid (eg, defunct) activation and this response serves only to invalidate this silo's activation cache.
return;
default:
logger.Error(ErrorCode.Dispatcher_InvalidEnum_RejectionType,
"Missing enum in switch: " + message.RejectionType);
break;
}
}
CallbackData callbackData;
bool found = callbacks.TryRemove(message.Id, out callbackData);
if (found)
{
if (message.TransactionInfo != null)
{
// NOTE: Not clear if thread-safe, revise
callbackData.TransactionInfo.Join(message.TransactionInfo);
}
// IMPORTANT: we do not schedule the response callback via the scheduler, since the only thing it does
// is to resolve/break the resolver. The continuations/waits that are based on this resolution will be scheduled as work items.
callbackData.DoCallback(message);
}
else
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug(ErrorCode.Dispatcher_NoCallbackForResp,
"No callback for response message: " + message);
}
}
}