private void ResponseCallback(Message message, TaskCompletionSource <object> context)
{
Response response;
if (message.Result != Message.ResponseTypes.Rejection)
{
try
{
response = (Response)message.GetDeserializedBody(this.serializationManager);
}
catch (Exception exc)
{
// catch the Deserialize exception and break the promise with it.
response = Response.ExceptionResponse(exc);
}
}
else
{
Exception rejection;
switch (message.RejectionType)
{
case Message.RejectionTypes.GatewayTooBusy:
rejection = new GatewayTooBusyException();
break;
case Message.RejectionTypes.DuplicateRequest:
return; // Ignore duplicates
default:
rejection = message.GetDeserializedBody(this.serializationManager) as Exception;
if (rejection == null)
{
if (string.IsNullOrEmpty(message.RejectionInfo))
{
message.RejectionInfo = "Unable to send request - no rejection info available";
}
rejection = new OrleansMessageRejectionException(message.RejectionInfo);
}
break;
}
response = Response.ExceptionResponse(rejection);
}
if (!response.ExceptionFlag)
{
context.TrySetResult(response.Data);
}
else
{
context.TrySetException(response.Exception);
}
}
/// <summary>
/// Resolve target address for a message
/// - use transaction info
/// - check ordering info in message and sending activation
/// - use sender's placement strategy
/// </summary>
/// <param name="message"></param>
/// <returns>Resolve when message is addressed (modifies message fields)</returns>
private Task AddressMessage(Message message)
{
var targetAddress = message.TargetAddress;
if (targetAddress.IsComplete)
{
return(Task.CompletedTask);
}
// placement strategy is determined by searching for a specification. first, we check for a strategy associated with the grain reference,
// second, we check for a strategy associated with the target's interface. third, we check for a strategy associated with the activation sending the
// message.
var strategy = targetAddress.Grain.IsGrain ? catalog.GetGrainPlacementStrategy(targetAddress.Grain) : null;
var request = message.IsUsingInterfaceVersions
? message.GetDeserializedBody(this.serializationManager) as InvokeMethodRequest
: null;
var target = new PlacementTarget(
message.TargetGrain,
message.RequestContextData,
request?.InterfaceId ?? 0,
request?.InterfaceVersion ?? 0);
PlacementResult placementResult;
if (placementDirectorsManager.TrySelectActivationSynchronously(
message.SendingAddress, target, this.catalog, strategy, out placementResult) && placementResult != null)
{
SetMessageTargetPlacement(message, placementResult, targetAddress);
return(Task.CompletedTask);
}
return(AddressMessageAsync(message, target, strategy, targetAddress));
}
/// <summary>
/// Handle an incoming message and queue/invoke appropriate handler
/// </summary>
/// <param name="message"></param>
/// <param name="targetActivation"></param>
public void HandleIncomingRequest(Message message, ActivationData targetActivation)
{
lock (targetActivation)
{
if (targetActivation.Grain.IsGrain && message.IsUsingInterfaceVersions)
{
var request = ((InvokeMethodRequest)message.GetDeserializedBody(this.serializationManager));
var compatibilityDirector = compatibilityDirectorManager.GetDirector(request.InterfaceId);
var currentVersion = catalog.GrainTypeManager.GetLocalSupportedVersion(request.InterfaceId);
if (!compatibilityDirector.IsCompatible(request.InterfaceVersion, currentVersion))
{
catalog.DeactivateActivationOnIdle(targetActivation);
}
}
if (targetActivation.State == ActivationState.Invalid || targetActivation.State == ActivationState.Deactivating)
{
ProcessRequestToInvalidActivation(message, targetActivation.Address, targetActivation.ForwardingAddress, "HandleIncomingRequest");
return;
}
// Now we can actually scheduler processing of this request
targetActivation.RecordRunning(message);
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedOk(message);
scheduler.QueueWorkItem(new InvokeWorkItem(targetActivation, message, this, this.invokeWorkItemLogger), targetActivation.SchedulingContext);
}
}
/// <summary>
/// Resolve target address for a message
/// - use transaction info
/// - check ordering info in message and sending activation
/// - use sender's placement strategy
/// </summary>
/// <param name="message"></param>
/// <returns>Resolve when message is addressed (modifies message fields)</returns>
private async Task AddressMessage(Message message)
{
var targetAddress = message.TargetAddress;
if (targetAddress.IsComplete)
{
return;
}
// placement strategy is determined by searching for a specification. first, we check for a strategy associated with the grain reference,
// second, we check for a strategy associated with the target's interface. third, we check for a strategy associated with the activation sending the
// message.
var strategy = targetAddress.Grain.IsGrain ? catalog.GetGrainPlacementStrategy(targetAddress.Grain) : null;
var request = message.IsUsingInterfaceVersions
? message.GetDeserializedBody(this.serializationManager) as InvokeMethodRequest
: null;
var target = new PlacementTarget(message.TargetGrain, request?.InterfaceId ?? 0, request?.InterfaceVersion ?? 0);
var placementResult = await this.placementDirectorsManager.SelectOrAddActivation(
message.SendingAddress, target, this.catalog, strategy);
if (placementResult.IsNewPlacement && targetAddress.Grain.IsClient)
{
logger.Error(ErrorCode.Dispatcher_AddressMsg_UnregisteredClient, String.Format("AddressMessage could not find target for client pseudo-grain {0}", message));
throw new KeyNotFoundException(String.Format("Attempting to send a message {0} to an unregistered client pseudo-grain {1}", message, targetAddress.Grain));
}
message.SetTargetPlacement(placementResult);
if (placementResult.IsNewPlacement)
{
CounterStatistic.FindOrCreate(StatisticNames.DISPATCHER_NEW_PLACEMENT).Increment();
}
if (logger.IsVerbose2)
{
logger.Verbose2(ErrorCode.Dispatcher_AddressMsg_SelectTarget, "AddressMessage Placement SelectTarget {0}", message);
}
}
public async Task Invoke(IAddressable target, IInvokable invokable, Message message)
{
try
{
// Don't process messages that have already timed out
if (message.IsExpired)
{
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke);
return;
}
RequestContext.Import(message.RequestContextData);
if (Config.Globals.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget)
{
UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message));
// RequestContext is automatically saved in the msg upon send and propagated to the next hop
// in RuntimeClient.CreateMessage -> RequestContext.ExportToMessage(message);
}
object resultObject;
try
{
var request = (InvokeMethodRequest)message.GetDeserializedBody(this.SerializationManager);
if (request.Arguments != null)
{
CancellationSourcesExtension.RegisterCancellationTokens(target, request, logger, this);
}
var invoker = invokable.GetInvoker(typeManager, request.InterfaceId, message.GenericGrainType);
if (invoker is IGrainExtensionMethodInvoker &&
!(target is IGrainExtension))
{
// We are trying the invoke a grain extension method on a grain
// -- most likely reason is that the dynamic extension is not installed for this grain
// So throw a specific exception here rather than a general InvalidCastException
var error = String.Format(
"Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}",
target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName);
var exc = new GrainExtensionNotInstalledException(error);
string extraDebugInfo = null;
#if DEBUG
extraDebugInfo = Utils.GetStackTrace();
#endif
logger.Warn(ErrorCode.Stream_ExtensionNotInstalled,
string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc);
throw exc;
}
#pragma warning disable 618
var invokeInterceptor = this.CurrentStreamProviderRuntime?.GetInvokeInterceptor();
#pragma warning restore 618
var requestInvoker = new GrainMethodInvoker(target, request, invoker, siloInterceptors, interfaceToImplementationMapping, invokeInterceptor);
await requestInvoker.Invoke();
resultObject = requestInvoker.Result;
}
catch (Exception exc1)
{
if (invokeExceptionLogger.IsVerbose || message.Direction == Message.Directions.OneWay)
{
invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message, exc1);
}
if (exc1 is InconsistentStateException && target is Grain)
{
var activation = ((Grain)target).Data;
invokeExceptionLogger.Info($"Deactivating {activation} due to inconsistent state.");
this.DeactivateOnIdle(activation.ActivationId);
}
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc1);
}
return;
}
if (message.Direction == Message.Directions.OneWay)
{
return;
}
SafeSendResponse(message, resultObject);
}
catch (Exception exc2)
{
logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2);
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc2);
}
}
}
public async Task Invoke(IAddressable target, IInvokable invokable, Message message)
{
try
{
// Don't process messages that have already timed out
if (message.IsExpired)
{
message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Invoke);
return;
}
RequestContextExtensions.Import(message.RequestContextData);
if (schedulingOptions.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget)
{
UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation, message.DebugContext));
// RequestContext is automatically saved in the msg upon send and propagated to the next hop
// in RuntimeClient.CreateMessage -> RequestContextExtensions.ExportToMessage(message);
}
bool startNewTransaction = false;
ITransactionInfo transactionInfo = message.TransactionInfo;
if (message.IsTransactionRequired && transactionInfo == null)
{
// TODO: this should be a configurable parameter
var transactionTimeout = Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(10);
// Start a new transaction
transactionInfo = await this.transactionAgent.Value.StartTransaction(message.IsReadOnly, transactionTimeout);
startNewTransaction = true;
}
if (transactionInfo != null)
{
TransactionContext.SetTransactionInfo(transactionInfo);
}
object resultObject;
try
{
var request = (InvokeMethodRequest)message.GetDeserializedBody(this.serializationManager);
if (request.Arguments != null)
{
CancellationSourcesExtension.RegisterCancellationTokens(target, request, this.loggerFactory, logger, this, this.cancellationTokenRuntime);
}
var invoker = invokable.GetInvoker(typeManager, request.InterfaceId, message.GenericGrainType);
if (invoker is IGrainExtensionMethodInvoker &&
!(target is IGrainExtension))
{
// We are trying the invoke a grain extension method on a grain
// -- most likely reason is that the dynamic extension is not installed for this grain
// So throw a specific exception here rather than a general InvalidCastException
var error = String.Format(
"Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}",
target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName);
var exc = new GrainExtensionNotInstalledException(error);
string extraDebugInfo = null;
#if DEBUG
extraDebugInfo = Utils.GetStackTrace();
#endif
logger.Warn(ErrorCode.Stream_ExtensionNotInstalled,
string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc);
throw exc;
}
var requestInvoker = new GrainMethodInvoker(target, request, invoker, GrainCallFilters, interfaceToImplementationMapping);
await requestInvoker.Invoke();
resultObject = requestInvoker.Result;
}
catch (Exception exc1)
{
if (invokeExceptionLogger.IsEnabled(LogLevel.Debug) || message.Direction == Message.Directions.OneWay)
{
invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message, exc1);
}
transactionInfo = TransactionContext.GetTransactionInfo();
if (transactionInfo != null)
{
// Must abort the transaction on exceptions
transactionInfo.IsAborted = true;
if (startNewTransaction)
{
var abortException = (exc1 as OrleansTransactionAbortedException) ??
new OrleansTransactionAbortedException(transactionInfo.TransactionId.ToString(), exc1);
this.transactionAgent.Value.Abort(transactionInfo, abortException);
exc1 = abortException;
}
}
// If a grain allowed an inconsistent state exception to escape and the exception originated from
// this activation, then deactivate it.
var ise = exc1 as InconsistentStateException;
if (ise != null && ise.IsSourceActivation)
{
// Mark the exception so that it doesn't deactivate any other activations.
ise.IsSourceActivation = false;
var activation = (target as Grain)?.Data;
if (activation != null)
{
invokeExceptionLogger.Info($"Deactivating {activation} due to inconsistent state.");
this.DeactivateOnIdle(activation.ActivationId);
}
}
if (message.Direction != Message.Directions.OneWay)
{
TransactionContext.Clear();
SafeSendExceptionResponse(message, exc1);
}
return;
}
transactionInfo = TransactionContext.GetTransactionInfo();
if (transactionInfo != null && !transactionInfo.ReconcilePending(out var numberOrphans))
{
var abortException = new OrleansOrphanCallException(transactionInfo.TransactionId.ToString(), numberOrphans);
// Can't exit before the transaction completes.
TransactionContext.GetTransactionInfo().IsAborted = true;
if (startNewTransaction)
{
this.transactionAgent.Value.Abort(TransactionContext.GetTransactionInfo(), abortException);
}
if (message.Direction != Message.Directions.OneWay)
{
TransactionContext.Clear();
SafeSendExceptionResponse(message, abortException);
}
return;
}
if (startNewTransaction)
{
// This request started the transaction, so we try to commit before returning.
await this.transactionAgent.Value.Commit(transactionInfo);
TransactionContext.Clear();
}
if (message.Direction == Message.Directions.OneWay)
{
return;
}
SafeSendResponse(message, resultObject);
}
catch (Exception exc2)
{
logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2);
try
{
if (exc2 is OrleansTransactionInDoubtException)
{
this.logger.LogError(exc2, "Transaction failed due to in doubt transaction");
}
else if (TransactionContext.GetTransactionInfo() != null)
{
// Must abort the transaction on exceptions
TransactionContext.GetTransactionInfo().IsAborted = true;
var abortException = (exc2 as OrleansTransactionAbortedException) ??
new OrleansTransactionAbortedException(TransactionContext.GetTransactionInfo().TransactionId.ToString(), exc2);
this.transactionAgent.Value.Abort(TransactionContext.GetTransactionInfo(), abortException);
}
}
finally
{
TransactionContext.Clear();
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc2);
}
}
}
finally
{
TransactionContext.Clear();
}
}