public MessageCenter(
ILocalSiloDetails siloDetails,
IOptions <SiloMessagingOptions> messagingOptions,
IOptions <NetworkingOptions> networkingOptions,
MessageFactory messageFactory,
Factory <MessageCenter, Gateway> gatewayFactory,
ILoggerFactory loggerFactory,
IOptions <StatisticsOptions> statisticsOptions,
ConnectionManager senderManager)
{
this.messagingOptions = messagingOptions.Value;
this.loggerFactory = loggerFactory;
this.senderManager = senderManager;
this.log = loggerFactory.CreateLogger <MessageCenter>();
this.messageFactory = messageFactory;
this.MyAddress = siloDetails.SiloAddress;
this.Initialize(networkingOptions, statisticsOptions);
if (siloDetails.GatewayAddress != null)
{
Gateway = gatewayFactory(this);
}
messageHandlers = new Action <Message> [Enum.GetValues(typeof(Message.Categories)).Length];
}
public InsideRuntimeClient(
ILocalSiloDetails siloDetails,
ClusterConfiguration config,
GrainTypeManager typeManager,
TypeMetadataCache typeMetadataCache,
OrleansTaskScheduler scheduler,
IServiceProvider serviceProvider,
SerializationManager serializationManager,
MessageFactory messageFactory,
IEnumerable <IGrainCallFilter> registeredInterceptors,
Factory <ITransactionAgent> transactionAgent,
ILoggerFactory loggerFactory,
IOptions <SiloMessagingOptions> messagingOptions)
{
this.ServiceProvider = serviceProvider;
this.SerializationManager = serializationManager;
MySilo = siloDetails.SiloAddress;
disposables = new List <IDisposable>();
callbacks = new ConcurrentDictionary <CorrelationId, CallbackData>();
Config = config;
config.OnConfigChange("Globals/Message", () => ResponseTimeout = Config.Globals.ResponseTimeout);
this.typeManager = typeManager;
this.messageFactory = messageFactory;
this.transactionAgent = new Lazy <ITransactionAgent>(() => transactionAgent());
this.Scheduler = scheduler;
this.ConcreteGrainFactory = new GrainFactory(this, typeMetadataCache);
tryResendMessage = msg => this.Dispatcher.TryResendMessage(msg);
unregisterCallback = msg => UnRegisterCallback(msg.Id);
this.siloInterceptors = new List <IGrainCallFilter>(registeredInterceptors);
this.logger = loggerFactory.CreateLogger <InsideRuntimeClient>();
this.invokeExceptionLogger = loggerFactory.CreateLogger($"{typeof(Grain).FullName}.InvokeException");
this.loggerFactory = loggerFactory;
this.messagingOptions = messagingOptions.Value;
this.callbackDataLogger = new LoggerWrapper <CallbackData>(loggerFactory);
this.timerLogger = loggerFactory.CreateLogger <SafeTimer>();
}
internal Dispatcher(
OrleansTaskScheduler scheduler,
MessageCenter messageCenter,
Catalog catalog,
IOptionsMonitor <SiloMessagingOptions> messagingOptions,
PlacementService placementService,
ILocalGrainDirectory localGrainDirectory,
MessageFactory messageFactory,
ILoggerFactory loggerFactory,
ActivationDirectory activationDirectory,
RuntimeMessagingTrace messagingTrace)
{
this.scheduler = scheduler;
this.catalog = catalog;
this.messageCenter = messageCenter;
this.messagingOptions = messagingOptions.CurrentValue;
this.placementService = placementService;
this.localGrainDirectory = localGrainDirectory;
this.messageFactory = messageFactory;
this.activationDirectory = activationDirectory;
this.messagingTrace = messagingTrace;
this.logger = loggerFactory.CreateLogger <Dispatcher>();
messageCenter.SetDispatcher(this);
}
public InsideRuntimeClient(
ILocalSiloDetails siloDetails,
GrainTypeManager typeManager,
TypeMetadataCache typeMetadataCache,
OrleansTaskScheduler scheduler,
IServiceProvider serviceProvider,
MessageFactory messageFactory,
Factory <ITransactionAgent> transactionAgent,
ILoggerFactory loggerFactory,
IOptions <SiloMessagingOptions> messagingOptions,
IGrainCancellationTokenRuntime cancellationTokenRuntime,
IOptions <SchedulingOptions> schedulerOptions,
ApplicationRequestsStatisticsGroup appRequestStatistics)
{
this.ServiceProvider = serviceProvider;
this.MySilo = siloDetails.SiloAddress;
this.disposables = new List <IDisposable>();
this.callbacks = new ConcurrentDictionary <CorrelationId, CallbackData>();
this.ResponseTimeout = messagingOptions.Value.ResponseTimeout;
this.typeManager = typeManager;
this.messageFactory = messageFactory;
this.transactionAgent = new Lazy <ITransactionAgent>(() => transactionAgent());
this.Scheduler = scheduler;
this.ConcreteGrainFactory = new GrainFactory(this, typeMetadataCache);
this.tryResendMessage = msg => this.Dispatcher.TryResendMessage(msg);
this.unregisterCallback = msg => this.UnRegisterCallback(msg.Id);
this.logger = loggerFactory.CreateLogger <InsideRuntimeClient>();
this.invokeExceptionLogger = loggerFactory.CreateLogger($"{typeof(Grain).FullName}.InvokeException");
this.loggerFactory = loggerFactory;
this.messagingOptions = messagingOptions.Value;
this.callbackDataLogger = loggerFactory.CreateLogger <CallbackData>();
this.timerLogger = loggerFactory.CreateLogger <SafeTimer>();
this.cancellationTokenRuntime = cancellationTokenRuntime;
this.appRequestStatistics = appRequestStatistics;
this.schedulingOptions = schedulerOptions.Value;
}
public MessageCenter(
ILocalSiloDetails siloDetails,
IOptions <SiloMessagingOptions> messagingOptions,
MessageFactory messageFactory,
Factory <MessageCenter, Gateway> gatewayFactory,
ILoggerFactory loggerFactory,
IOptions <StatisticsOptions> statisticsOptions,
ISiloStatusOracle siloStatusOracle,
ConnectionManager senderManager,
MessagingTrace messagingTrace)
{
this.messagingOptions = messagingOptions.Value;
this.loggerFactory = loggerFactory;
this.senderManager = senderManager;
this.messagingTrace = messagingTrace;
this.log = loggerFactory.CreateLogger <MessageCenter>();
this.messageFactory = messageFactory;
this.MyAddress = siloDetails.SiloAddress;
if (log.IsEnabled(LogLevel.Trace))
{
log.Trace("Starting initialization.");
}
OutboundQueue = new OutboundMessageQueue(this, this.loggerFactory.CreateLogger <OutboundMessageQueue>(), this.senderManager, siloStatusOracle, this.messagingTrace);
if (log.IsEnabled(LogLevel.Trace))
{
log.Trace("Completed initialization.");
}
if (siloDetails.GatewayAddress != null)
{
Gateway = gatewayFactory(this);
}
}
// Forwarding is used by the receiver, usually when it cannot process the message and forwards it to another silo to perform the processing
// (got here due to duplicate activation, outdated cache, silo is shutting down/overloaded, ...).
private static bool MayForward(Message message, SiloMessagingOptions messagingOptions)
{
return(message.ForwardCount < messagingOptions.MaxForwardCount
// allow one more forward hop for multi-cluster case
+ (message.IsReturnedFromRemoteCluster ? 1 : 0));
}
public void AnalyzeWorkload(DateTime now, IMessageCenter messageCenter, MessageFactory messageFactory, SiloMessagingOptions options)
{
var slowRunningRequestDuration = options.RequestProcessingWarningTime;
var longQueueTimeDuration = options.RequestQueueDelayWarningTime;
List <string> diagnostics = null;
lock (this)
{
if (State != ActivationState.Valid)
{
return;
}
if (this.Blocking is object)
{
var message = this.Blocking;
var timeSinceQueued = now - message.QueuedTime;
var executionTime = now - currentRequestStartTime;
if (executionTime >= slowRunningRequestDuration)
{
GetStatusList(ref diagnostics);
if (timeSinceQueued.HasValue)
{
diagnostics.Add($"Message {message} was enqueued {timeSinceQueued} ago and has now been executing for {executionTime}.");
}
else
{
diagnostics.Add($"Message {message} has been executing for {executionTime}.");
}
var response = messageFactory.CreateDiagnosticResponseMessage(message, isExecuting: true, isWaiting: false, diagnostics);
messageCenter.SendMessage(response);
}
}
foreach (var running in RunningRequests)
{
var message = running.Key;
var startTime = running.Value;
if (ReferenceEquals(message, this.Blocking))
{
continue;
}
// Check how long they've been executing.
var executionTime = now - startTime;
if (executionTime >= slowRunningRequestDuration)
{
// Interleaving message X has been executing for a long time
GetStatusList(ref diagnostics);
var messageDiagnostics = new List <string>(diagnostics)
{
$"Interleaving message {message} has been executing for {executionTime}."
};
var response = messageFactory.CreateDiagnosticResponseMessage(message, isExecuting: true, isWaiting: false, messageDiagnostics);
messageCenter.SendMessage(response);
}
}
if (waiting is object)
{
var queueLength = 1;
foreach (var message in waiting)
{
var waitTime = now - message.QueuedTime;
if (waitTime >= longQueueTimeDuration)
{
// Message X has been enqueued on the target grain for Y and is currently position QueueLength in queue for processing.
GetStatusList(ref diagnostics);
var messageDiagnostics = new List <string>(diagnostics)
{
$"Message {message} has been enqueued on the target grain for {waitTime} and is currently position {queueLength} in queue for processing."
};
var response = messageFactory.CreateDiagnosticResponseMessage(message, isExecuting: false, isWaiting: true, messageDiagnostics);
messageCenter.SendMessage(response);
}
queueLength++;
}
}
}
void GetStatusList(ref List <string> diagnostics)
{
if (diagnostics is object)
{
return;
}
diagnostics = new List <string>
{
this.ToDetailedString(),
$"TaskScheduler status: {this.WorkItemGroup.DumpStatus()}"
};
}
}
// Forwarding is used by the receiver, usually when it cannot process the message and forwards it to another silo to perform the processing
// (got here due to outdated cache, silo is shutting down/overloaded, ...).
private static bool MayForward(Message message, SiloMessagingOptions messagingOptions)
{
return(message.ForwardCount < messagingOptions.MaxForwardCount);
}
// Forwardings are used by the receiver, usualy when it cannot process the message and forwars it to another silo to perform the processing
// (got here due to outdated cache, silo is shutting down/overloaded, ...).
public bool MayForward(SiloMessagingOptions messagingOptions)
{
return(ForwardCount < messagingOptions.MaxForwardCount);
}