public IDictionary <GrainId, IGrainInfo> GetDirectoryForTypeNamesContaining(string expr)
{
var x = new Dictionary <GrainId, IGrainInfo>();
LocalGrainDirectory localGrainDirectory = this.host.Services.GetRequiredService <LocalGrainDirectory>();
var catalog = this.host.Services.GetRequiredService <Catalog>();
foreach (var kvp in localGrainDirectory.DirectoryPartition.GetItems())
{
if (kvp.Key.IsSystemTarget || kvp.Key.IsClient || !kvp.Key.IsGrain)
{
continue;// Skip system grains, system targets and clients
}
if (catalog.GetGrainTypeName(kvp.Key).Contains(expr))
{
x.Add(kvp.Key, kvp.Value);
}
}
return(x);
}
private async Task OnRuntimeServicesStart(CancellationToken ct)
{
//TODO: Setup all (or as many as possible) of the class started in this call to work directly with lifecyce
// The order of these 4 is pretty much arbitrary.
scheduler.Start();
messageCenter.Start();
incomingPingAgent.Start();
incomingSystemAgent.Start();
incomingAgent.Start();
LocalGrainDirectory.Start();
// Set up an execution context for this thread so that the target creation steps can use asynch values.
RuntimeContext.InitializeMainThread();
// Initialize the implicit stream subscribers table.
var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>();
var grainTypeManager = Services.GetRequiredService <GrainTypeManager>();
implicitStreamSubscriberTable.InitImplicitStreamSubscribers(grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray());
var siloProviderRuntime = Services.GetRequiredService <SiloProviderRuntime>();
runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime;
statisticsProviderManager = this.Services.GetRequiredService <StatisticsProviderManager>();
string statsProviderName = await statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
.WithTimeout(initTimeout);
if (statsProviderName != null)
{
LocalConfig.StatisticsProviderName = statsProviderName;
}
// can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
await siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WithTimeout(initTimeout);
await siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WithTimeout(initTimeout);
// This has to follow the above steps that start the runtime components
CreateSystemTargets();
await InjectDependencies();
// Validate the configuration.
GlobalConfig.Application.ValidateConfiguration(logger);
// Initialize storage providers once we have a basic silo runtime environment operating
storageProviderManager = this.Services.GetRequiredService <StorageProviderManager>();
await scheduler.QueueTask(
() => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WithTimeout(initTimeout);
ITransactionAgent transactionAgent = this.Services.GetRequiredService <ITransactionAgent>();
ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext;
await scheduler.QueueTask(transactionAgent.Start, transactionAgentContext)
.WithTimeout(initTimeout);
var versionStore = Services.GetService <IVersionStore>() as GrainVersionStore;
versionStore?.SetStorageManager(storageProviderManager);
logger.Debug("Storage provider manager created successfully.");
// Initialize log consistency providers once we have a basic silo runtime environment operating
logConsistencyProviderManager = this.Services.GetRequiredService <LogConsistencyProviderManager>();
await scheduler.QueueTask(
() => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WithTimeout(initTimeout);
logger.Debug("Log consistency provider manager created successfully.");
// Load and init stream providers before silo becomes active
var siloStreamProviderManager = (StreamProviderManager)this.Services.GetRequiredService <IStreamProviderManager>();
await scheduler.QueueTask(
() => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime),
providerManagerSystemTarget.SchedulingContext)
.WithTimeout(initTimeout);
runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager;
logger.Debug("Stream provider manager created successfully.");
// Load and init grain services before silo becomes active.
await CreateGrainServices(GlobalConfig.GrainServiceConfigurations);
this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
this.providerManagerSystemTarget.SchedulingContext;
await scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
.WithTimeout(initTimeout);
logger.Debug("Local silo status oracle created successfully.");
await scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext)
.WithTimeout(initTimeout);
logger.Debug("Local silo status oracle became active successfully.");
await scheduler.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
.WithTimeout(this.initTimeout);
//if running in multi cluster scenario, start the MultiClusterNetwork Oracle
if (GlobalConfig.HasMultiClusterNetwork)
{
logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
GlobalConfig.ServiceId, GlobalConfig.ClusterId);
this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
this.providerManagerSystemTarget.SchedulingContext;
await scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
.WithTimeout(initTimeout);
logger.Debug("multicluster oracle created successfully.");
}
try
{
this.siloStatistics.Start(this.LocalConfig);
logger.Debug("Silo statistics manager started successfully.");
// Finally, initialize the deployment load collector, for grains with load-based placement
var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
await this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
.WithTimeout(this.initTimeout);
logger.Debug("Silo deployment load publisher started successfully.");
// Start background timer tick to watch for platform execution stalls, such as when GC kicks in
this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants, this.executorService, this.loggerFactory);
this.platformWatchdog.Start();
if (this.logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Silo platform watchdog started successfully.");
}
if (this.reminderService != null)
{
// so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ??
this.providerManagerSystemTarget.SchedulingContext;
await this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext)
.WithTimeout(this.initTimeout);
this.logger.Debug("Reminder service started successfully.");
}
this.bootstrapProviderManager = this.Services.GetRequiredService <BootstrapProviderManager>();
await this.scheduler.QueueTask(
() => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations),
this.providerManagerSystemTarget.SchedulingContext)
.WithTimeout(this.initTimeout);
this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics
logger.Debug("App bootstrap calls done successfully.");
// Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
// also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
await this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext)
.WithTimeout(this.initTimeout);
logger.Debug("Stream providers started successfully.");
// Now that we're active, we can start the gateway
var mc = this.messageCenter as MessageCenter;
mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>());
logger.Debug("Message gateway service started successfully.");
this.SystemStatus = SystemStatus.Running;
}
catch (Exception exc)
{
this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
throw;
}
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Silo.Start complete: System status = {0}", this.SystemStatus);
}
}
internal Silo(SiloInitializationParameters initializationParams, IServiceProvider services)
{
string name = initializationParams.Name;
ClusterConfiguration config = initializationParams.ClusterConfig;
this.initializationParams = initializationParams;
this.SystemStatus = SystemStatus.Creating;
AsynchAgent.IsStarting = true;
var startTime = DateTime.UtcNow;
services?.GetService <TelemetryManager>()?.AddFromConfiguration(services, LocalConfig.TelemetryConfiguration);
StatisticsCollector.Initialize(LocalConfig);
initTimeout = GlobalConfig.MaxJoinAttemptTime;
if (Debugger.IsAttached)
{
initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), GlobalConfig.MaxJoinAttemptTime);
stopTimeout = initTimeout;
}
var localEndpoint = this.initializationParams.SiloAddress.Endpoint;
// Configure DI using Startup type
if (services == null)
{
var serviceCollection = new ServiceCollection();
serviceCollection.AddSingleton <Silo>(this);
serviceCollection.AddSingleton(initializationParams);
serviceCollection.AddLegacyClusterConfigurationSupport(config);
serviceCollection.Configure <SiloIdentityOptions>(options => options.SiloName = name);
var hostContext = new HostBuilderContext(new Dictionary <object, object>());
DefaultSiloServices.AddDefaultServices(hostContext, serviceCollection);
var applicationPartManager = hostContext.GetApplicationPartManager();
applicationPartManager.AddApplicationPartsFromAppDomain();
applicationPartManager.AddApplicationPartsFromBasePath();
services = StartupBuilder.ConfigureStartup(this.LocalConfig.StartupTypeName, serviceCollection);
services.GetService <TelemetryManager>()?.AddFromConfiguration(services, LocalConfig.TelemetryConfiguration);
}
services.GetService <SerializationManager>().RegisterSerializers(services.GetService <ApplicationPartManager>());
this.Services = services;
this.Services.InitializeSiloUnobservedExceptionsHandler();
//set PropagateActivityId flag from node cofnig
RequestContext.PropagateActivityId = this.initializationParams.NodeConfig.PropagateActivityId;
this.loggerFactory = this.Services.GetRequiredService <ILoggerFactory>();
logger = this.loggerFactory.CreateLogger <Silo>();
logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode));
if (!GCSettings.IsServerGC)
{
logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\">");
logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines).");
}
logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------",
this.initializationParams.Type, LocalConfig.DNSHostName, Environment.MachineName, localEndpoint, this.initializationParams.SiloAddress.Generation);
logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with the following configuration= " + Environment.NewLine + "{1}",
name, config.ToString(name));
var siloMessagingOptions = this.Services.GetRequiredService <IOptions <SiloMessagingOptions> >();
BufferPool.InitGlobalBufferPool(siloMessagingOptions);
try
{
grainFactory = Services.GetRequiredService <GrainFactory>();
}
catch (InvalidOperationException exc)
{
logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc);
throw;
}
// Performance metrics
siloStatistics = Services.GetRequiredService <SiloStatisticsManager>();
// The scheduler
scheduler = Services.GetRequiredService <OrleansTaskScheduler>();
healthCheckParticipants.Add(scheduler);
runtimeClient = Services.GetRequiredService <InsideRuntimeClient>();
// Initialize the message center
messageCenter = Services.GetRequiredService <MessageCenter>();
var dispatcher = this.Services.GetRequiredService <Dispatcher>();
messageCenter.RerouteHandler = dispatcher.RerouteMessage;
messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage;
// GrainRuntime can be created only here, after messageCenter was created.
grainRuntime = Services.GetRequiredService <IGrainRuntime>();
StreamProviderManager = Services.GetRequiredService <IStreamProviderManager>();
// Now the router/directory service
// This has to come after the message center //; note that it then gets injected back into the message center.;
localGrainDirectory = Services.GetRequiredService <LocalGrainDirectory>();
// Now the activation directory.
activationDirectory = Services.GetRequiredService <ActivationDirectory>();
// Now the consistent ring provider
RingProvider = Services.GetRequiredService <IConsistentRingProvider>();
catalog = Services.GetRequiredService <Catalog>();
siloStatistics.MetricsTable.Scheduler = scheduler;
siloStatistics.MetricsTable.ActivationDirectory = activationDirectory;
siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector;
siloStatistics.MetricsTable.MessageCenter = messageCenter;
executorService = Services.GetRequiredService <ExecutorService>();
// Now the incoming message agents
var messageFactory = this.Services.GetRequiredService <MessageFactory>();
incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory);
incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory);
incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory);
membershipOracle = Services.GetRequiredService <IMembershipOracle>();
if (!this.GlobalConfig.HasMultiClusterNetwork)
{
logger.Info("Skip multicluster oracle creation (no multicluster network configured)");
}
else
{
multiClusterOracle = Services.GetRequiredService <IMultiClusterOracle>();
}
this.SystemStatus = SystemStatus.Created;
AsynchAgent.IsStarting = false;
StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
() => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.
var fullSiloLifecycle = this.Services.GetRequiredService <SiloLifecycle>();
this.siloLifecycle = fullSiloLifecycle;
IEnumerable <ILifecycleParticipant <ISiloLifecycle> > lifecyleParticipants = this.Services.GetServices <ILifecycleParticipant <ISiloLifecycle> >();
foreach (ILifecycleParticipant <ISiloLifecycle> participant in lifecyleParticipants)
{
participant.Participate(fullSiloLifecycle);
}
this.Participate(fullSiloLifecycle);
logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
}
private void DoStart()
{
lock (lockable)
{
if (SystemStatus.Current != SystemStatus.Created)
{
throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Start state. This silo is in the {0} state.", SystemStatus.Current));
}
SystemStatus.Current = SystemStatus.Starting;
}
logger.Info(ErrorCode.SiloStarting, "Silo Start()");
// Hook up to receive notification of process exit / Ctrl-C events
AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
Console.CancelKeyPress += HandleProcessExit;
ConfigureThreadPoolAndServicePointSettings();
// This has to start first so that the directory system target factory gets loaded before we start the router.
typeManager.Start();
InsideRuntimeClient.Current.Start();
// The order of these 4 is pretty much arbitrary.
scheduler.Start();
messageCenter.Start();
incomingPingAgent.Start();
incomingSystemAgent.Start();
incomingAgent.Start();
LocalGrainDirectory.Start();
// Set up an execution context for this thread so that the target creation steps can use asynch values.
RuntimeContext.InitializeMainThread();
SiloProviderRuntime.Initialize(GlobalConfig);
statisticsProviderManager = new StatisticsProviderManager("Statistics", SiloProviderRuntime.Instance);
string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
.WaitForResultWithThrow(initTimeout);
if (statsProviderName != null)
{
LocalConfig.StatisticsProviderName = statsProviderName;
}
// can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
siloStatistics.SetSiloStatsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
siloStatistics.SetSiloMetricsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
membershipOracle = membershipFactory.CreateMembershipOracle(this).WaitForResultWithThrow(initTimeout);
// This has to follow the above steps that start the runtime components
CreateSystemTargets();
InjectDependencies();
// ensure this runs in the grain context, wait for it to complete
scheduler.QueueTask(CreateSystemGrains, catalog.SchedulingContext)
.WaitWithThrow(initTimeout);
// Initialize storage providers once we have a basic silo runtime environment operating
storageProviderManager = new StorageProviderManager();
scheduler.QueueTask(
() => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
catalog.SetStorageManager(storageProviderManager);
// Load and init stream providers before silo becomes active
var siloStreamProviderManager = new Orleans.Streams.StreamProviderManager();
scheduler.QueueTask(
() => siloStreamProviderManager.LoadStreamProviders(this.GlobalConfig.ProviderConfigurations, SiloProviderRuntime.Instance),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
InsideRuntimeClient.Current.CurrentStreamProviderManager = siloStreamProviderManager;
ISchedulingContext statusOracleContext = ((SystemTarget)LocalSiloStatusOracle).SchedulingContext;
bool waitForPrimaryToStart = globalConfig.PrimaryNodeIsRequired && siloType != SiloType.Primary;
scheduler.QueueTask(() => LocalSiloStatusOracle.Start(waitForPrimaryToStart), statusOracleContext)
.WaitWithThrow(initTimeout);
scheduler.QueueTask(LocalSiloStatusOracle.BecomeActive, statusOracleContext)
.WaitWithThrow(initTimeout);
try
{
siloStatistics.Start(LocalConfig);
// Finally, initialize the deployment load collector, for grains with load-based placement
scheduler.QueueTask(DeploymentLoadPublisher.Instance.Start, DeploymentLoadPublisher.Instance.SchedulingContext)
.WaitWithThrow(initTimeout);
// Start background timer tick to watch for platform execution stalls, such as when GC kicks in
platformWatchdog = new Watchdog(nodeConfig.StatisticsLogWriteInterval, healthCheckParticipants);
platformWatchdog.Start();
// so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
scheduler.QueueTask(reminderService.Start, ((SystemTarget)reminderService).SchedulingContext)
.WaitWithThrow(initTimeout);
// Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
var bootstrapProviderManager = new BootstrapProviderManager();
scheduler.QueueTask(
() => bootstrapProviderManager.LoadAppBootstrapProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
BootstrapProviders = bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics
// Now that we're active, we can start the gateway
var mc = messageCenter as MessageCenter;
if (mc != null)
{
mc.StartGateway();
}
SystemStatus.Current = SystemStatus.Running;
}
catch (Exception exc)
{
SafeExecute(() => logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", SiloAddress), exc));
FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
throw;
}
}
public Silo(ILocalSiloDetails siloDetails, IServiceProvider services)
{
string name = siloDetails.Name;
// Temporarily still require this. Hopefuly gone when 2.0 is released.
this.siloDetails = siloDetails;
this.SystemStatus = SystemStatus.Creating;
AsynchAgent.IsStarting = true; // todo. use ISiloLifecycle instead?
var startTime = DateTime.UtcNow;
IOptions <ClusterMembershipOptions> clusterMembershipOptions = services.GetRequiredService <IOptions <ClusterMembershipOptions> >();
initTimeout = clusterMembershipOptions.Value.MaxJoinAttemptTime;
if (Debugger.IsAttached)
{
initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), clusterMembershipOptions.Value.MaxJoinAttemptTime);
stopTimeout = initTimeout;
}
var localEndpoint = this.siloDetails.SiloAddress.Endpoint;
services.GetService <SerializationManager>().RegisterSerializers(services.GetService <IApplicationPartManager>());
this.Services = services;
this.Services.InitializeSiloUnobservedExceptionsHandler();
//set PropagateActivityId flag from node config
IOptions <SiloMessagingOptions> messagingOptions = services.GetRequiredService <IOptions <SiloMessagingOptions> >();
RequestContext.PropagateActivityId = messagingOptions.Value.PropagateActivityId;
this.loggerFactory = this.Services.GetRequiredService <ILoggerFactory>();
logger = this.loggerFactory.CreateLogger <Silo>();
logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode));
if (!GCSettings.IsServerGC)
{
logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\">");
logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines).");
}
logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing silo on host {0} MachineName {1} at {2}, gen {3} --------------",
this.siloDetails.DnsHostName, Environment.MachineName, localEndpoint, this.siloDetails.SiloAddress.Generation);
logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0}", name);
var siloMessagingOptions = this.Services.GetRequiredService <IOptions <SiloMessagingOptions> >();
BufferPool.InitGlobalBufferPool(siloMessagingOptions.Value);
try
{
grainFactory = Services.GetRequiredService <GrainFactory>();
}
catch (InvalidOperationException exc)
{
logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc);
throw;
}
// Performance metrics
siloStatistics = Services.GetRequiredService <SiloStatisticsManager>();
// The scheduler
scheduler = Services.GetRequiredService <OrleansTaskScheduler>();
healthCheckParticipants.Add(scheduler);
runtimeClient = Services.GetRequiredService <InsideRuntimeClient>();
// Initialize the message center
messageCenter = Services.GetRequiredService <MessageCenter>();
var dispatcher = this.Services.GetRequiredService <Dispatcher>();
messageCenter.RerouteHandler = dispatcher.RerouteMessage;
messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage;
// Now the router/directory service
// This has to come after the message center //; note that it then gets injected back into the message center.;
localGrainDirectory = Services.GetRequiredService <LocalGrainDirectory>();
// Now the activation directory.
activationDirectory = Services.GetRequiredService <ActivationDirectory>();
// Now the consistent ring provider
RingProvider = Services.GetRequiredService <IConsistentRingProvider>();
catalog = Services.GetRequiredService <Catalog>();
executorService = Services.GetRequiredService <ExecutorService>();
// Now the incoming message agents
var messageFactory = this.Services.GetRequiredService <MessageFactory>();
incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory);
incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory);
incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory);
membershipOracle = Services.GetRequiredService <IMembershipOracle>();
this.clusterOptions = Services.GetRequiredService <IOptions <ClusterOptions> >().Value;
var multiClusterOptions = Services.GetRequiredService <IOptions <MultiClusterOptions> >().Value;
if (!multiClusterOptions.HasMultiClusterNetwork)
{
logger.Info("Skip multicluster oracle creation (no multicluster network configured)");
}
else
{
multiClusterOracle = Services.GetRequiredService <IMultiClusterOracle>();
}
this.SystemStatus = SystemStatus.Created;
AsynchAgent.IsStarting = false;
StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
() => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.
this.siloLifecycle = this.Services.GetRequiredService <ISiloLifecycleSubject>();
// register all lifecycle participants
IEnumerable <ILifecycleParticipant <ISiloLifecycle> > lifecycleParticipants = this.Services.GetServices <ILifecycleParticipant <ISiloLifecycle> >();
foreach (ILifecycleParticipant <ISiloLifecycle> participant in lifecycleParticipants)
{
participant?.Participate(this.siloLifecycle);
}
// register all named lifecycle participants
IKeyedServiceCollection <string, ILifecycleParticipant <ISiloLifecycle> > namedLifecycleParticipantCollection = this.Services.GetService <IKeyedServiceCollection <string, ILifecycleParticipant <ISiloLifecycle> > >();
foreach (ILifecycleParticipant <ISiloLifecycle> participant in namedLifecycleParticipantCollection
?.GetServices(this.Services)
?.Select(s => s.GetService(this.Services)))
{
participant?.Participate(this.siloLifecycle);
}
// add self to lifecycle
this.Participate(this.siloLifecycle);
logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
}
internal Silo(string name, SiloType siloType, ClusterConfiguration config, ILocalDataStore keyStore)
{
SystemStatus.Current = SystemStatus.Creating;
CurrentSilo = this;
var startTime = DateTime.UtcNow;
this.siloType = siloType;
Name = name;
siloTerminatedEvent = new ManualResetEvent(false);
OrleansConfig = config;
globalConfig = config.Globals;
config.OnConfigChange("Defaults", () => nodeConfig = config.GetConfigurationForNode(name));
if (!TraceLogger.IsInitialized)
{
TraceLogger.Initialize(nodeConfig);
}
config.OnConfigChange("Defaults/Tracing", () => TraceLogger.Initialize(nodeConfig, true), false);
LimitManager.Initialize(nodeConfig);
ActivationData.Init(config);
StatisticsCollector.Initialize(nodeConfig);
SerializationManager.Initialize(globalConfig.UseStandardSerializer);
initTimeout = globalConfig.MaxJoinAttemptTime;
if (Debugger.IsAttached)
{
initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), globalConfig.MaxJoinAttemptTime);
stopTimeout = initTimeout;
}
IPEndPoint here = nodeConfig.Endpoint;
int generation = nodeConfig.Generation;
if (generation == 0)
{
generation = SiloAddress.AllocateNewGeneration();
nodeConfig.Generation = generation;
}
TraceLogger.MyIPEndPoint = here;
logger = TraceLogger.GetLogger("Silo", TraceLogger.LoggerType.Runtime);
logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on {1} at {2}, gen {3} --------------", siloType, nodeConfig.DNSHostName, here, generation);
logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with runtime Version='{1}' Config= \n{2}", name, RuntimeVersion.Current, config.ToString(name));
if (keyStore != null)
{
// Re-establish reference to shared local key store in this app domain
LocalDataStoreInstance.LocalDataStore = keyStore;
}
healthCheckParticipants = new List <IHealthCheckParticipant>();
BufferPool.InitGlobalBufferPool(globalConfig);
PlacementStrategy.Initialize(globalConfig);
UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnobservedExceptionHandler);
AppDomain.CurrentDomain.UnhandledException +=
(obj, ev) => DomainUnobservedExceptionHandler(obj, (Exception)ev.ExceptionObject);
typeManager = new GrainTypeManager(here.Address.Equals(IPAddress.Loopback));
// Performance metrics
siloStatistics = new SiloStatisticsManager(globalConfig, nodeConfig);
config.OnConfigChange("Defaults/LoadShedding", () => siloStatistics.MetricsTable.NodeConfig = nodeConfig, false);
// The scheduler
scheduler = new OrleansTaskScheduler(globalConfig, nodeConfig);
healthCheckParticipants.Add(scheduler);
// Initialize the message center
var mc = new MessageCenter(here, generation, globalConfig, siloStatistics.MetricsTable);
if (nodeConfig.IsGatewayNode)
{
mc.InstallGateway(nodeConfig.ProxyGatewayEndpoint);
}
messageCenter = mc;
// Now the router/directory service
// This has to come after the message center //; note that it then gets injected back into the message center.;
localGrainDirectory = new LocalGrainDirectory(this);
// Now the activation directory.
// This needs to know which router to use so that it can keep the global directory in synch with the local one.
activationDirectory = new ActivationDirectory();
// Now the consistent ring provider
RingProvider = GlobalConfig.UseVirtualBucketsConsistentRing ?
(IConsistentRingProvider) new VirtualBucketsRingProvider(SiloAddress, GlobalConfig.NumVirtualBucketsConsistentRing)
: new ConsistentRingProvider(SiloAddress);
Action <Dispatcher> setDispatcher;
catalog = new Catalog(Constants.CatalogId, SiloAddress, Name, LocalGrainDirectory, typeManager, scheduler, activationDirectory, config, out setDispatcher);
var dispatcher = new Dispatcher(scheduler, messageCenter, catalog, config);
setDispatcher(dispatcher);
RuntimeClient.Current = new InsideRuntimeClient(
dispatcher,
catalog,
LocalGrainDirectory,
SiloAddress,
config,
RingProvider,
typeManager);
messageCenter.RerouteHandler = InsideRuntimeClient.Current.RerouteMessage;
messageCenter.SniffIncomingMessage = InsideRuntimeClient.Current.SniffIncomingMessage;
messageCenter.ClientDropHandler = grainIds =>
{
catalog.DeleteGrainsLocal(grainIds).Ignore();
scheduler.RunOrQueueAction(() =>
{
// Consider: batch delete
foreach (var id in grainIds)
{
LocalGrainDirectory.DeleteGrain(id).Ignore();
}
}, catalog.SchedulingContext);
};
siloStatistics.MetricsTable.Scheduler = scheduler;
siloStatistics.MetricsTable.ActivationDirectory = activationDirectory;
siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector;
siloStatistics.MetricsTable.MessageCenter = messageCenter;
DeploymentLoadPublisher.CreateDeploymentLoadPublisher(this, globalConfig);
PlacementDirectorsManager.CreatePlacementDirectorsManager(globalConfig);
// Now the incoming message agents
incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, dispatcher);
incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, dispatcher);
incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, dispatcher);
membershipFactory = new MembershipFactory();
reminderFactory = new LocalReminderServiceFactory();
SystemStatus.Current = SystemStatus.Created;
StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
() => TraceLogger.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.
TestHookup = new TestHookups(this);
logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
}
internal Silo(string name, SiloType siloType, ClusterConfiguration config, ILocalDataStore keyStore)
{
SystemStatus.Current = SystemStatus.Creating;
CurrentSilo = this;
var startTime = DateTime.UtcNow;
this.siloType = siloType;
Name = name;
siloTerminatedEvent = new ManualResetEvent(false);
OrleansConfig = config;
globalConfig = config.Globals;
config.OnConfigChange("Defaults", () => nodeConfig = config.GetConfigurationForNode(name));
if (!TraceLogger.IsInitialized)
{
TraceLogger.Initialize(nodeConfig);
}
config.OnConfigChange("Defaults/Tracing", () => TraceLogger.Initialize(nodeConfig, true), false);
ActivationData.Init(config, nodeConfig);
StatisticsCollector.Initialize(nodeConfig);
SerializationManager.Initialize(globalConfig.UseStandardSerializer);
initTimeout = globalConfig.MaxJoinAttemptTime;
if (Debugger.IsAttached)
{
initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), globalConfig.MaxJoinAttemptTime);
stopTimeout = initTimeout;
}
IPEndPoint here = nodeConfig.Endpoint;
int generation = nodeConfig.Generation;
if (generation == 0)
{
generation = SiloAddress.AllocateNewGeneration();
nodeConfig.Generation = generation;
}
TraceLogger.MyIPEndPoint = here;
logger = TraceLogger.GetLogger("Silo", TraceLogger.LoggerType.Runtime);
logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode));
if (!GCSettings.IsServerGC || !GCSettings.LatencyMode.Equals(GCLatencyMode.Batch))
{
logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on or with GCLatencyMode.Batch enabled - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\"> and <configuration>-<runtime>-<gcConcurrent enabled=\"false\"/>");
}
logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------",
siloType, nodeConfig.DNSHostName, Environment.MachineName, here, generation);
logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with runtime Version='{1}' .NET version='{2}' Is .NET 4.5={3} OS version='{4}' Config= " + Environment.NewLine + "{5}",
name, RuntimeVersion.Current, Environment.Version, ConfigUtilities.IsNet45OrNewer(), Environment.OSVersion, config.ToString(name));
if (keyStore != null)
{
// Re-establish reference to shared local key store in this app domain
LocalDataStoreInstance.LocalDataStore = keyStore;
}
healthCheckParticipants = new List <IHealthCheckParticipant>();
allSiloProviders = new List <IProvider>();
BufferPool.InitGlobalBufferPool(globalConfig);
PlacementStrategy.Initialize(globalConfig);
UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnobservedExceptionHandler);
AppDomain.CurrentDomain.UnhandledException +=
(obj, ev) => DomainUnobservedExceptionHandler(obj, (Exception)ev.ExceptionObject);
grainFactory = new GrainFactory();
typeManager = new GrainTypeManager(here.Address.Equals(IPAddress.Loopback), grainFactory);
// Performance metrics
siloStatistics = new SiloStatisticsManager(globalConfig, nodeConfig);
config.OnConfigChange("Defaults/LoadShedding", () => siloStatistics.MetricsTable.NodeConfig = nodeConfig, false);
// The scheduler
scheduler = new OrleansTaskScheduler(globalConfig, nodeConfig);
healthCheckParticipants.Add(scheduler);
// Initialize the message center
var mc = new MessageCenter(here, generation, globalConfig, siloStatistics.MetricsTable);
if (nodeConfig.IsGatewayNode)
{
mc.InstallGateway(nodeConfig.ProxyGatewayEndpoint);
}
messageCenter = mc;
SiloIdentity = SiloAddress.ToLongString();
// GrainRuntime can be created only here, after messageCenter was created.
grainRuntime = new GrainRuntime(
globalConfig.ServiceId,
SiloIdentity,
grainFactory,
new TimerRegistry(),
new ReminderRegistry(),
new StreamProviderManager());
// Now the router/directory service
// This has to come after the message center //; note that it then gets injected back into the message center.;
localGrainDirectory = new LocalGrainDirectory(this);
// Now the activation directory.
// This needs to know which router to use so that it can keep the global directory in synch with the local one.
activationDirectory = new ActivationDirectory();
// Now the consistent ring provider
RingProvider = GlobalConfig.UseVirtualBucketsConsistentRing ?
(IConsistentRingProvider) new VirtualBucketsRingProvider(SiloAddress, GlobalConfig.NumVirtualBucketsConsistentRing)
: new ConsistentRingProvider(SiloAddress);
Action <Dispatcher> setDispatcher;
catalog = new Catalog(Constants.CatalogId, SiloAddress, Name, LocalGrainDirectory, typeManager, scheduler, activationDirectory, config, grainRuntime, out setDispatcher);
var dispatcher = new Dispatcher(scheduler, messageCenter, catalog, config);
setDispatcher(dispatcher);
RuntimeClient.Current = new InsideRuntimeClient(
dispatcher,
catalog,
LocalGrainDirectory,
SiloAddress,
config,
RingProvider,
typeManager,
grainFactory);
messageCenter.RerouteHandler = InsideRuntimeClient.Current.RerouteMessage;
messageCenter.SniffIncomingMessage = InsideRuntimeClient.Current.SniffIncomingMessage;
siloStatistics.MetricsTable.Scheduler = scheduler;
siloStatistics.MetricsTable.ActivationDirectory = activationDirectory;
siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector;
siloStatistics.MetricsTable.MessageCenter = messageCenter;
DeploymentLoadPublisher.CreateDeploymentLoadPublisher(this, globalConfig);
PlacementDirectorsManager.CreatePlacementDirectorsManager(globalConfig);
// Now the incoming message agents
incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, dispatcher);
incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, dispatcher);
incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, dispatcher);
membershipFactory = new MembershipFactory();
reminderFactory = new LocalReminderServiceFactory();
SystemStatus.Current = SystemStatus.Created;
StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
() => TraceLogger.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.
TestHookup = new TestHookups(this);
logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
}
public Silo(ILocalSiloDetails siloDetails, IServiceProvider services)
{
string name = siloDetails.Name;
// Temporarily still require this. Hopefuly gone when 2.0 is released.
this.siloDetails = siloDetails;
this.SystemStatus = SystemStatus.Creating;
var startTime = DateTime.UtcNow;
IOptions <ClusterMembershipOptions> clusterMembershipOptions = services.GetRequiredService <IOptions <ClusterMembershipOptions> >();
initTimeout = clusterMembershipOptions.Value.MaxJoinAttemptTime;
if (Debugger.IsAttached)
{
initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), clusterMembershipOptions.Value.MaxJoinAttemptTime);
stopTimeout = initTimeout;
}
var localEndpoint = this.siloDetails.SiloAddress.Endpoint;
this.Services = services;
//set PropagateActivityId flag from node config
IOptions <SiloMessagingOptions> messagingOptions = services.GetRequiredService <IOptions <SiloMessagingOptions> >();
RequestContext.PropagateActivityId = messagingOptions.Value.PropagateActivityId;
this.waitForMessageToBeQueuedForOutbound = messagingOptions.Value.WaitForMessageToBeQueuedForOutboundTime;
this.loggerFactory = this.Services.GetRequiredService <ILoggerFactory>();
logger = this.loggerFactory.CreateLogger <Silo>();
logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode));
if (!GCSettings.IsServerGC)
{
logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\">");
logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines).");
}
if (logger.IsEnabled(LogLevel.Debug))
{
var highestLogLevel = logger.IsEnabled(LogLevel.Trace) ? nameof(LogLevel.Trace) : nameof(LogLevel.Debug);
logger.LogWarning(
new EventId((int)ErrorCode.SiloGcWarning),
$"A verbose logging level ({highestLogLevel}) is configured. This will impact performance. The recommended log level is {nameof(LogLevel.Information)}.");
}
logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing silo on host {0} MachineName {1} at {2}, gen {3} --------------",
this.siloDetails.DnsHostName, Environment.MachineName, localEndpoint, this.siloDetails.SiloAddress.Generation);
logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0}", name);
try
{
grainFactory = Services.GetRequiredService <GrainFactory>();
}
catch (InvalidOperationException exc)
{
logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc);
throw;
}
// Performance metrics
siloStatistics = Services.GetRequiredService <SiloStatisticsManager>();
runtimeClient = Services.GetRequiredService <InsideRuntimeClient>();
// Initialize the message center
messageCenter = Services.GetRequiredService <MessageCenter>();
messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage;
// Now the router/directory service
// This has to come after the message center //; note that it then gets injected back into the message center.;
localGrainDirectory = Services.GetRequiredService <LocalGrainDirectory>();
// Now the consistent ring provider
RingProvider = Services.GetRequiredService <IConsistentRingProvider>();
catalog = Services.GetRequiredService <Catalog>();
siloStatusOracle = Services.GetRequiredService <ISiloStatusOracle>();
this.membershipService = Services.GetRequiredService <IMembershipService>();
this.SystemStatus = SystemStatus.Created;
StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
() => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.
this.siloLifecycle = this.Services.GetRequiredService <ISiloLifecycleSubject>();
// register all lifecycle participants
IEnumerable <ILifecycleParticipant <ISiloLifecycle> > lifecycleParticipants = this.Services.GetServices <ILifecycleParticipant <ISiloLifecycle> >();
foreach (ILifecycleParticipant <ISiloLifecycle> participant in lifecycleParticipants)
{
participant?.Participate(this.siloLifecycle);
}
// register all named lifecycle participants
IKeyedServiceCollection <string, ILifecycleParticipant <ISiloLifecycle> > namedLifecycleParticipantCollection = this.Services.GetService <IKeyedServiceCollection <string, ILifecycleParticipant <ISiloLifecycle> > >();
foreach (ILifecycleParticipant <ISiloLifecycle> participant in namedLifecycleParticipantCollection
?.GetServices(this.Services)
?.Select(s => s.GetService(this.Services)))
{
participant?.Participate(this.siloLifecycle);
}
// add self to lifecycle
this.Participate(this.siloLifecycle);
logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
}
private void DoStart()
{
lock (lockable)
{
if (!this.SystemStatus.Equals(SystemStatus.Created))
{
throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", this.SystemStatus));
}
this.SystemStatus = SystemStatus.Starting;
}
logger.Info(ErrorCode.SiloStarting, "Silo Start()");
// Hook up to receive notification of process exit / Ctrl-C events
AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
if (GlobalConfig.FastKillOnCancelKeyPress)
{
Console.CancelKeyPress += HandleProcessExit;
}
ConfigureThreadPoolAndServicePointSettings();
// This has to start first so that the directory system target factory gets loaded before we start the router.
grainTypeManager.Start();
runtimeClient.Start();
// The order of these 4 is pretty much arbitrary.
scheduler.Start();
messageCenter.Start();
incomingPingAgent.Start();
incomingSystemAgent.Start();
incomingAgent.Start();
LocalGrainDirectory.Start();
// Set up an execution context for this thread so that the target creation steps can use asynch values.
RuntimeContext.InitializeMainThread();
// Initialize the implicit stream subscribers table.
var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>();
implicitStreamSubscriberTable.InitImplicitStreamSubscribers(this.grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray());
var siloProviderRuntime = Services.GetRequiredService <SiloProviderRuntime>();
runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime;
statisticsProviderManager = this.Services.GetRequiredService <StatisticsProviderManager>();
string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
.WaitForResultWithThrow(initTimeout);
if (statsProviderName != null)
{
LocalConfig.StatisticsProviderName = statsProviderName;
}
// can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout);
siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout);
// This has to follow the above steps that start the runtime components
CreateSystemTargets();
InjectDependencies();
// Validate the configuration.
GlobalConfig.Application.ValidateConfiguration(logger);
// Initialize storage providers once we have a basic silo runtime environment operating
storageProviderManager = this.Services.GetRequiredService <StorageProviderManager>();
scheduler.QueueTask(
() => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
ITransactionAgent transactionAgent = this.Services.GetRequiredService <ITransactionAgent>();
ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext;
scheduler.QueueTask(transactionAgent.Start, transactionAgentContext)
.WaitWithThrow(initTimeout);
var versionStore = Services.GetService <IVersionStore>() as GrainVersionStore;
versionStore?.SetStorageManager(storageProviderManager);
if (logger.IsVerbose)
{
logger.Verbose("Storage provider manager created successfully.");
}
// Initialize log consistency providers once we have a basic silo runtime environment operating
logConsistencyProviderManager = this.Services.GetRequiredService <LogConsistencyProviderManager>();
scheduler.QueueTask(
() => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Log consistency provider manager created successfully.");
}
// Load and init stream providers before silo becomes active
var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager;
scheduler.QueueTask(
() => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager;
if (logger.IsVerbose)
{
logger.Verbose("Stream provider manager created successfully.");
}
// Load and init grain services before silo becomes active.
CreateGrainServices(GlobalConfig.GrainServiceConfigurations);
this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
this.providerManagerSystemTarget.SchedulingContext;
scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Local silo status oracle created successfully.");
}
scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Local silo status oracle became active successfully.");
}
scheduler.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
.WaitWithThrow(this.initTimeout);
//if running in multi cluster scenario, start the MultiClusterNetwork Oracle
if (GlobalConfig.HasMultiClusterNetwork)
{
logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
GlobalConfig.ServiceId, GlobalConfig.ClusterId);
this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
this.providerManagerSystemTarget.SchedulingContext;
scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("multicluster oracle created successfully.");
}
}
try
{
this.siloStatistics.Start(this.LocalConfig);
if (this.logger.IsVerbose)
{
this.logger.Verbose("Silo statistics manager started successfully.");
}
// Finally, initialize the deployment load collector, for grains with load-based placement
var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
.WaitWithThrow(this.initTimeout);
if (this.logger.IsVerbose)
{
this.logger.Verbose("Silo deployment load publisher started successfully.");
}
// Start background timer tick to watch for platform execution stalls, such as when GC kicks in
this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants);
this.platformWatchdog.Start();
if (this.logger.IsVerbose)
{
this.logger.Verbose("Silo platform watchdog started successfully.");
}
if (this.reminderService != null)
{
// so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ??
this.providerManagerSystemTarget.SchedulingContext;
this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext)
.WaitWithThrow(this.initTimeout);
if (this.logger.IsVerbose)
{
this.logger.Verbose("Reminder service started successfully.");
}
}
this.bootstrapProviderManager = this.Services.GetRequiredService <BootstrapProviderManager>();
this.scheduler.QueueTask(
() => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations),
this.providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(this.initTimeout);
this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics
if (this.logger.IsVerbose)
{
this.logger.Verbose("App bootstrap calls done successfully.");
}
// Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
// also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(this.initTimeout);
if (this.logger.IsVerbose)
{
this.logger.Verbose("Stream providers started successfully.");
}
// Now that we're active, we can start the gateway
var mc = this.messageCenter as MessageCenter;
mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>());
if (this.logger.IsVerbose)
{
this.logger.Verbose("Message gateway service started successfully.");
}
this.SystemStatus = SystemStatus.Running;
}
catch (Exception exc)
{
this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
this.FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
throw;
}
if (logger.IsVerbose)
{
logger.Verbose("Silo.Start complete: System status = {0}", this.SystemStatus);
}
}
internal Silo(SiloInitializationParameters initializationParams, IServiceProvider services)
{
string name = initializationParams.Name;
ClusterConfiguration config = initializationParams.ClusterConfig;
this.initializationParams = initializationParams;
this.SystemStatus = SystemStatus.Creating;
AsynchAgent.IsStarting = true;
var startTime = DateTime.UtcNow;
if (!LogManager.IsInitialized)
{
LogManager.Initialize(LocalConfig);
}
services?.GetService <TelemetryManager>()?.AddFromConfiguration(services, LocalConfig.TelemetryConfiguration);
config.OnConfigChange("Defaults/Tracing", () => LogManager.Initialize(LocalConfig, true), false);
StatisticsCollector.Initialize(LocalConfig);
initTimeout = GlobalConfig.MaxJoinAttemptTime;
if (Debugger.IsAttached)
{
initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), GlobalConfig.MaxJoinAttemptTime);
stopTimeout = initTimeout;
}
var localEndpoint = this.initializationParams.SiloAddress.Endpoint;
LogManager.MyIPEndPoint = localEndpoint;
logger = LogManager.GetLogger("Silo", LoggerType.Runtime);
logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode));
if (!GCSettings.IsServerGC)
{
logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\">");
logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines).");
}
logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------",
this.initializationParams.Type, LocalConfig.DNSHostName, Environment.MachineName, localEndpoint, this.initializationParams.SiloAddress.Generation);
logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with the following configuration= " + Environment.NewLine + "{1}",
name, config.ToString(name));
// Configure DI using Startup type
if (services == null)
{
var serviceCollection = new ServiceCollection();
serviceCollection.AddSingleton <Silo>(this);
serviceCollection.AddSingleton(initializationParams);
DefaultSiloServices.AddDefaultServices(serviceCollection);
services = StartupBuilder.ConfigureStartup(this.LocalConfig.StartupTypeName, serviceCollection);
services.GetService <TelemetryManager>()?.AddFromConfiguration(services, LocalConfig.TelemetryConfiguration);
}
this.Services = services;
this.assemblyProcessor = this.Services.GetRequiredService <AssemblyProcessor>();
this.assemblyProcessor.Initialize();
BufferPool.InitGlobalBufferPool(GlobalConfig);
if (!UnobservedExceptionsHandlerClass.TrySetUnobservedExceptionHandler(UnobservedExceptionHandler))
{
logger.Warn(ErrorCode.Runtime_Error_100153, "Unable to set unobserved exception handler because it was already set.");
}
AppDomain.CurrentDomain.UnhandledException += this.DomainUnobservedExceptionHandler;
try
{
grainFactory = Services.GetRequiredService <GrainFactory>();
}
catch (InvalidOperationException exc)
{
logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc);
throw;
}
grainTypeManager = Services.GetRequiredService <GrainTypeManager>();
// Performance metrics
siloStatistics = Services.GetRequiredService <SiloStatisticsManager>();
// The scheduler
scheduler = Services.GetRequiredService <OrleansTaskScheduler>();
healthCheckParticipants.Add(scheduler);
runtimeClient = Services.GetRequiredService <InsideRuntimeClient>();
// Initialize the message center
messageCenter = Services.GetRequiredService <MessageCenter>();
var dispatcher = this.Services.GetRequiredService <Dispatcher>();
messageCenter.RerouteHandler = dispatcher.RerouteMessage;
messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage;
// GrainRuntime can be created only here, after messageCenter was created.
grainRuntime = Services.GetRequiredService <IGrainRuntime>();
// Now the router/directory service
// This has to come after the message center //; note that it then gets injected back into the message center.;
localGrainDirectory = Services.GetRequiredService <LocalGrainDirectory>();
// Now the activation directory.
activationDirectory = Services.GetRequiredService <ActivationDirectory>();
// Now the consistent ring provider
RingProvider = Services.GetRequiredService <IConsistentRingProvider>();
catalog = Services.GetRequiredService <Catalog>();
siloStatistics.MetricsTable.Scheduler = scheduler;
siloStatistics.MetricsTable.ActivationDirectory = activationDirectory;
siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector;
siloStatistics.MetricsTable.MessageCenter = messageCenter;
// Now the incoming message agents
var messageFactory = this.Services.GetRequiredService <MessageFactory>();
incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory);
incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory);
incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory);
membershipOracle = Services.GetRequiredService <IMembershipOracle>();
if (!this.GlobalConfig.HasMultiClusterNetwork)
{
logger.Info("Skip multicluster oracle creation (no multicluster network configured)");
}
else
{
multiClusterOracle = Services.GetRequiredService <IMultiClusterOracle>();
}
this.SystemStatus = SystemStatus.Created;
AsynchAgent.IsStarting = false;
StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
() => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.
logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
}
private void DoStart()
{
lock (lockable)
{
if (!SystemStatus.Current.Equals(SystemStatus.Created))
{
throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", SystemStatus.Current));
}
SystemStatus.Current = SystemStatus.Starting;
}
logger.Info(ErrorCode.SiloStarting, "Silo Start()");
// Hook up to receive notification of process exit / Ctrl-C events
AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
Console.CancelKeyPress += HandleProcessExit;
ConfigureThreadPoolAndServicePointSettings();
// This has to start first so that the directory system target factory gets loaded before we start the router.
typeManager.Start();
InsideRuntimeClient.Current.Start();
// The order of these 4 is pretty much arbitrary.
scheduler.Start();
messageCenter.Start();
incomingPingAgent.Start();
incomingSystemAgent.Start();
incomingAgent.Start();
LocalGrainDirectory.Start();
// Set up an execution context for this thread so that the target creation steps can use asynch values.
RuntimeContext.InitializeMainThread();
SiloProviderRuntime.Initialize(GlobalConfig, SiloIdentity, grainFactory, Services);
InsideRuntimeClient.Current.CurrentStreamProviderRuntime = SiloProviderRuntime.Instance;
statisticsProviderManager = new StatisticsProviderManager("Statistics", SiloProviderRuntime.Instance);
string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
.WaitForResultWithThrow(initTimeout);
if (statsProviderName != null)
{
LocalConfig.StatisticsProviderName = statsProviderName;
}
allSiloProviders.AddRange(statisticsProviderManager.GetProviders());
// can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
siloStatistics.SetSiloStatsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
siloStatistics.SetSiloMetricsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
IMembershipTable membershipTable = membershipFactory.GetMembershipTable(GlobalConfig.LivenessType, GlobalConfig.MembershipTableAssembly);
membershipOracle = membershipFactory.CreateMembershipOracle(this, membershipTable);
multiClusterOracle = multiClusterFactory.CreateGossipOracle(this).WaitForResultWithThrow(initTimeout);
// This has to follow the above steps that start the runtime components
CreateSystemTargets();
InjectDependencies();
// Validate the configuration.
GlobalConfig.Application.ValidateConfiguration(logger);
// ensure this runs in the grain context, wait for it to complete
scheduler.QueueTask(CreateSystemGrains, catalog.SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("System grains created successfully.");
}
// Initialize storage providers once we have a basic silo runtime environment operating
storageProviderManager = new StorageProviderManager(grainFactory, Services);
scheduler.QueueTask(
() => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
catalog.SetStorageManager(storageProviderManager);
allSiloProviders.AddRange(storageProviderManager.GetProviders());
if (logger.IsVerbose)
{
logger.Verbose("Storage provider manager created successfully.");
}
// Load and init stream providers before silo becomes active
var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager;
scheduler.QueueTask(
() => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, SiloProviderRuntime.Instance),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
InsideRuntimeClient.Current.CurrentStreamProviderManager = siloStreamProviderManager;
allSiloProviders.AddRange(siloStreamProviderManager.GetProviders());
if (logger.IsVerbose)
{
logger.Verbose("Stream provider manager created successfully.");
}
ISchedulingContext statusOracleContext = ((SystemTarget)LocalSiloStatusOracle).SchedulingContext;
bool waitForPrimaryToStart = globalConfig.PrimaryNodeIsRequired && siloType != SiloType.Primary;
if (waitForPrimaryToStart) // only in MembershipTableGrain case.
{
scheduler.QueueTask(() => membershipFactory.WaitForTableToInit(membershipTable), statusOracleContext)
.WaitWithThrow(initTimeout);
}
scheduler.QueueTask(() => membershipTable.InitializeMembershipTable(GlobalConfig, true, LogManager.GetLogger(membershipTable.GetType().Name)), statusOracleContext)
.WaitWithThrow(initTimeout);
scheduler.QueueTask(() => LocalSiloStatusOracle.Start(), statusOracleContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Local silo status oracle created successfully.");
}
scheduler.QueueTask(LocalSiloStatusOracle.BecomeActive, statusOracleContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Local silo status oracle became active successfully.");
}
//if running in multi cluster scenario, start the MultiClusterNetwork Oracle
if (GlobalConfig.HasMultiClusterNetwork)
{
logger.Info("Creating multicluster oracle with my ServiceId={0} and ClusterId={1}.",
GlobalConfig.ServiceId, GlobalConfig.ClusterId);
ISchedulingContext clusterStatusContext = ((SystemTarget)multiClusterOracle).SchedulingContext;
scheduler.QueueTask(() => multiClusterOracle.Start(LocalSiloStatusOracle), clusterStatusContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("multicluster oracle created successfully.");
}
}
try
{
siloStatistics.Start(LocalConfig);
if (logger.IsVerbose)
{
logger.Verbose("Silo statistics manager started successfully.");
}
// Finally, initialize the deployment load collector, for grains with load-based placement
scheduler.QueueTask(DeploymentLoadPublisher.Instance.Start, DeploymentLoadPublisher.Instance.SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Silo deployment load publisher started successfully.");
}
// Start background timer tick to watch for platform execution stalls, such as when GC kicks in
platformWatchdog = new Watchdog(nodeConfig.StatisticsLogWriteInterval, healthCheckParticipants);
platformWatchdog.Start();
if (logger.IsVerbose)
{
logger.Verbose("Silo platform watchdog started successfully.");
}
if (reminderService != null)
{
// so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
scheduler.QueueTask(reminderService.Start, ((SystemTarget)reminderService).SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Reminder service started successfully.");
}
}
bootstrapProviderManager = new BootstrapProviderManager();
scheduler.QueueTask(
() => bootstrapProviderManager.LoadAppBootstrapProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
BootstrapProviders = bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics
allSiloProviders.AddRange(BootstrapProviders);
if (logger.IsVerbose)
{
logger.Verbose("App bootstrap calls done successfully.");
}
// Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
// also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Stream providers started successfully.");
}
// Now that we're active, we can start the gateway
var mc = messageCenter as MessageCenter;
if (mc != null)
{
mc.StartGateway(clientRegistrar);
}
if (logger.IsVerbose)
{
logger.Verbose("Message gateway service started successfully.");
}
SystemStatus.Current = SystemStatus.Running;
}
catch (Exception exc)
{
SafeExecute(() => logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", SiloAddress), exc));
FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
throw;
}
if (logger.IsVerbose)
{
logger.Verbose("Silo.Start complete: System status = {0}", SystemStatus.Current);
}
}
