public void ReceiveResponse(Message message) { if (message.Result == Message.ResponseTypes.Rejection) { if (!message.TargetSilo.Matches(this.CurrentSilo)) { // gatewayed message - gateway back to sender if (logger.IsVerbose2) { logger.Verbose2(ErrorCode.Dispatcher_NoCallbackForRejectionResp, "No callback for rejection response message: {0}", message); } dispatcher.Transport.SendMessage(message); return; } if (logger.IsVerbose) { logger.Verbose(ErrorCode.Dispatcher_HandleMsg, "HandleMessage {0}", message); } switch (message.RejectionType) { case Message.RejectionTypes.DuplicateRequest: // try to remove from callbackData, just in case it is still there. break; case Message.RejectionTypes.Overloaded: break; case Message.RejectionTypes.Unrecoverable: // fall through & reroute case Message.RejectionTypes.Transient: if (!message.ContainsHeader(Message.Header.CACHE_INVALIDATION_HEADER)) { // Remove from local directory cache. Note that SendingGrain is the original target, since message is the rejection response. // If CacheMgmtHeader is present, we already did this. Otherwise, we left this code for backward compatability. // It should be retired as we move to use CacheMgmtHeader in all relevant places. directory.InvalidateCacheEntry(message.SendingAddress); } break; default: logger.Error(ErrorCode.Dispatcher_InvalidEnum_RejectionType, "Missing enum in switch: " + message.RejectionType); break; } } CallbackData callbackData; bool found = callbacks.TryGetValue(message.Id, out callbackData); if (found) { // IMPORTANT: we do not schedule the response callback via the scheduler, since the only thing it does // is to resolve/break the resolver. The continuations/waits that are based on this resolution will be scheduled as work items. callbackData.DoCallback(message); } else { if (logger.IsVerbose) { logger.Verbose(ErrorCode.Dispatcher_NoCallbackForResp, "No callback for response message: " + message); } } }
/// <summary> /// Receive a new message: /// - validate order constraints, queue (or possibly redirect) if out of order /// - validate transactions constraints /// - invoke handler if ready, otherwise enqueue for later invocation /// </summary> /// <param name="message"></param> public void ReceiveMessage(Message message) { MessagingProcessingStatisticsGroup.OnDispatcherMessageReceive(message); // Don't process messages that have already timed out if (message.IsExpired) { logger.Warn(ErrorCode.Dispatcher_DroppingExpiredMessage, "Dropping an expired message: {0}", message); MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Expired"); message.DropExpiredMessage(MessagingStatisticsGroup.Phase.Dispatch); return; } // check if its targeted at a new activation if (message.TargetGrain.IsSystemTarget) { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ReceiveMessage on system target."); throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message); } if (errorInjection && ShouldInjectError(message)) { if (logger.IsVerbose) { logger.Verbose(ErrorCode.Dispatcher_InjectingRejection, "Injecting a rejection"); } MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "ErrorInjection"); RejectMessage(message, Message.RejectionTypes.Unrecoverable, null, "Injected rejection"); return; } try { Task ignore; ActivationData target = catalog.GetOrCreateActivation( message.TargetAddress, message.IsNewPlacement, message.NewGrainType, message.GenericGrainType, message.RequestContextData, out ignore); if (ignore != null) { ignore.Ignore(); } if (message.Direction == Message.Directions.Response) { ReceiveResponse(message, target); } else // Request or OneWay { // Silo is always capable to accept a new request. It's up to the activation to handle its internal state. // If activation is shutting down, it will queue and later forward this request. ReceiveRequest(message, target); } } catch (Exception ex) { try { MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message, "Non-existent activation"); var nea = ex as Catalog.NonExistentActivationException; if (nea == null) { var str = String.Format("Error creating activation for {0}. Message {1}", message.NewGrainType, message); logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex); throw new OrleansException(str, ex); } if (nea.IsStatelessWorker) { if (logger.IsVerbose) { logger.Verbose(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, String.Format("Intermediate StatelessWorker NonExistentActivation for message {0}", message), ex); } } else { logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation, String.Format("Intermediate NonExistentActivation for message {0}", message), ex); } ActivationAddress nonExistentActivation = nea.NonExistentActivation; if (message.Direction != Message.Directions.Response) { // Un-register the target activation so we don't keep getting spurious messages. // The time delay (one minute, as of this writing) is to handle the unlikely but possible race where // this request snuck ahead of another request, with new placement requested, for the same activation. // If the activation registration request from the new placement somehow sneaks ahead of this un-registration, // we want to make sure that we don't un-register the activation we just created. // We would add a counter here, except that there's already a counter for this in the Catalog. // Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context if (config.Globals.DirectoryLazyDeregistrationDelay > TimeSpan.Zero) { Scheduler.QueueWorkItem(new ClosureWorkItem( // don't use message.TargetAddress, cause it may have been removed from the headers by this time! async() => { try { await Silo.CurrentSilo.LocalGrainDirectory.UnregisterConditionallyAsync( nonExistentActivation); } catch (Exception exc) { logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct, String.Format("Failed to un-register NonExistentActivation {0}", nonExistentActivation), exc); } }, () => "LocalGrainDirectory.UnregisterConditionallyAsync"), catalog.SchedulingContext); } ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation"); } else { logger.Warn(ErrorCode.Dispatcher_NoTargetActivation, "No target activation {0} for response message: {1}", nonExistentActivation, message); Silo.CurrentSilo.LocalGrainDirectory.InvalidateCacheEntry(nonExistentActivation); } } catch (Exception exc) { // Unable to create activation for this request - reject message RejectMessage(message, Message.RejectionTypes.Transient, exc); } } }