public IDictionary <GrainId, IGrainInfo> GetDirectoryForTypeNamesContaining(string expr) { var x = new Dictionary <GrainId, IGrainInfo>(); LocalGrainDirectory localGrainDirectory = this.host.Services.GetRequiredService <LocalGrainDirectory>(); var catalog = this.host.Services.GetRequiredService <Catalog>(); foreach (var kvp in localGrainDirectory.DirectoryPartition.GetItems()) { if (kvp.Key.IsSystemTarget || kvp.Key.IsClient || !kvp.Key.IsGrain) { continue;// Skip system grains, system targets and clients } if (catalog.GetGrainTypeName(kvp.Key).Contains(expr)) { x.Add(kvp.Key, kvp.Value); } } return(x); }
private async Task OnRuntimeServicesStart(CancellationToken ct) { //TODO: Setup all (or as many as possible) of the class started in this call to work directly with lifecyce // The order of these 4 is pretty much arbitrary. scheduler.Start(); messageCenter.Start(); incomingPingAgent.Start(); incomingSystemAgent.Start(); incomingAgent.Start(); LocalGrainDirectory.Start(); // Set up an execution context for this thread so that the target creation steps can use asynch values. RuntimeContext.InitializeMainThread(); // Initialize the implicit stream subscribers table. var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>(); var grainTypeManager = Services.GetRequiredService <GrainTypeManager>(); implicitStreamSubscriberTable.InitImplicitStreamSubscribers(grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray()); var siloProviderRuntime = Services.GetRequiredService <SiloProviderRuntime>(); runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime; statisticsProviderManager = this.Services.GetRequiredService <StatisticsProviderManager>(); string statsProviderName = await statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations) .WithTimeout(initTimeout); if (statsProviderName != null) { LocalConfig.StatisticsProviderName = statsProviderName; } // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress). await siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WithTimeout(initTimeout); await siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WithTimeout(initTimeout); // This has to follow the above steps that start the runtime components CreateSystemTargets(); await InjectDependencies(); // Validate the configuration. GlobalConfig.Application.ValidateConfiguration(logger); // Initialize storage providers once we have a basic silo runtime environment operating storageProviderManager = this.Services.GetRequiredService <StorageProviderManager>(); await scheduler.QueueTask( () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations), providerManagerSystemTarget.SchedulingContext) .WithTimeout(initTimeout); ITransactionAgent transactionAgent = this.Services.GetRequiredService <ITransactionAgent>(); ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext; await scheduler.QueueTask(transactionAgent.Start, transactionAgentContext) .WithTimeout(initTimeout); var versionStore = Services.GetService <IVersionStore>() as GrainVersionStore; versionStore?.SetStorageManager(storageProviderManager); logger.Debug("Storage provider manager created successfully."); // Initialize log consistency providers once we have a basic silo runtime environment operating logConsistencyProviderManager = this.Services.GetRequiredService <LogConsistencyProviderManager>(); await scheduler.QueueTask( () => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations), providerManagerSystemTarget.SchedulingContext) .WithTimeout(initTimeout); logger.Debug("Log consistency provider manager created successfully."); // Load and init stream providers before silo becomes active var siloStreamProviderManager = (StreamProviderManager)this.Services.GetRequiredService <IStreamProviderManager>(); await scheduler.QueueTask( () => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime), providerManagerSystemTarget.SchedulingContext) .WithTimeout(initTimeout); runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager; logger.Debug("Stream provider manager created successfully."); // Load and init grain services before silo becomes active. await CreateGrainServices(GlobalConfig.GrainServiceConfigurations); this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ?? this.providerManagerSystemTarget.SchedulingContext; await scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext) .WithTimeout(initTimeout); logger.Debug("Local silo status oracle created successfully."); await scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext) .WithTimeout(initTimeout); logger.Debug("Local silo status oracle became active successfully."); await scheduler.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext) .WithTimeout(this.initTimeout); //if running in multi cluster scenario, start the MultiClusterNetwork Oracle if (GlobalConfig.HasMultiClusterNetwork) { logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.", GlobalConfig.ServiceId, GlobalConfig.ClusterId); this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ?? this.providerManagerSystemTarget.SchedulingContext; await scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext) .WithTimeout(initTimeout); logger.Debug("multicluster oracle created successfully."); } try { this.siloStatistics.Start(this.LocalConfig); logger.Debug("Silo statistics manager started successfully."); // Finally, initialize the deployment load collector, for grains with load-based placement var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>(); await this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext) .WithTimeout(this.initTimeout); logger.Debug("Silo deployment load publisher started successfully."); // Start background timer tick to watch for platform execution stalls, such as when GC kicks in this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants, this.executorService, this.loggerFactory); this.platformWatchdog.Start(); if (this.logger.IsEnabled(LogLevel.Debug)) { logger.Debug("Silo platform watchdog started successfully."); } if (this.reminderService != null) { // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ?? this.providerManagerSystemTarget.SchedulingContext; await this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext) .WithTimeout(this.initTimeout); this.logger.Debug("Reminder service started successfully."); } this.bootstrapProviderManager = this.Services.GetRequiredService <BootstrapProviderManager>(); await this.scheduler.QueueTask( () => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations), this.providerManagerSystemTarget.SchedulingContext) .WithTimeout(this.initTimeout); this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics logger.Debug("App bootstrap calls done successfully."); // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active). // also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive. await this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext) .WithTimeout(this.initTimeout); logger.Debug("Stream providers started successfully."); // Now that we're active, we can start the gateway var mc = this.messageCenter as MessageCenter; mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>()); logger.Debug("Message gateway service started successfully."); this.SystemStatus = SystemStatus.Running; } catch (Exception exc) { this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc)); throw; } if (logger.IsEnabled(LogLevel.Debug)) { logger.Debug("Silo.Start complete: System status = {0}", this.SystemStatus); } }
internal Silo(SiloInitializationParameters initializationParams, IServiceProvider services) { string name = initializationParams.Name; ClusterConfiguration config = initializationParams.ClusterConfig; this.initializationParams = initializationParams; this.SystemStatus = SystemStatus.Creating; AsynchAgent.IsStarting = true; var startTime = DateTime.UtcNow; services?.GetService <TelemetryManager>()?.AddFromConfiguration(services, LocalConfig.TelemetryConfiguration); StatisticsCollector.Initialize(LocalConfig); initTimeout = GlobalConfig.MaxJoinAttemptTime; if (Debugger.IsAttached) { initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), GlobalConfig.MaxJoinAttemptTime); stopTimeout = initTimeout; } var localEndpoint = this.initializationParams.SiloAddress.Endpoint; // Configure DI using Startup type if (services == null) { var serviceCollection = new ServiceCollection(); serviceCollection.AddSingleton <Silo>(this); serviceCollection.AddSingleton(initializationParams); serviceCollection.AddLegacyClusterConfigurationSupport(config); serviceCollection.Configure <SiloIdentityOptions>(options => options.SiloName = name); var hostContext = new HostBuilderContext(new Dictionary <object, object>()); DefaultSiloServices.AddDefaultServices(hostContext, serviceCollection); var applicationPartManager = hostContext.GetApplicationPartManager(); applicationPartManager.AddApplicationPartsFromAppDomain(); applicationPartManager.AddApplicationPartsFromBasePath(); services = StartupBuilder.ConfigureStartup(this.LocalConfig.StartupTypeName, serviceCollection); services.GetService <TelemetryManager>()?.AddFromConfiguration(services, LocalConfig.TelemetryConfiguration); } services.GetService <SerializationManager>().RegisterSerializers(services.GetService <ApplicationPartManager>()); this.Services = services; this.Services.InitializeSiloUnobservedExceptionsHandler(); //set PropagateActivityId flag from node cofnig RequestContext.PropagateActivityId = this.initializationParams.NodeConfig.PropagateActivityId; this.loggerFactory = this.Services.GetRequiredService <ILoggerFactory>(); logger = this.loggerFactory.CreateLogger <Silo>(); logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode)); if (!GCSettings.IsServerGC) { logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\">"); logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines)."); } logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------", this.initializationParams.Type, LocalConfig.DNSHostName, Environment.MachineName, localEndpoint, this.initializationParams.SiloAddress.Generation); logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with the following configuration= " + Environment.NewLine + "{1}", name, config.ToString(name)); var siloMessagingOptions = this.Services.GetRequiredService <IOptions <SiloMessagingOptions> >(); BufferPool.InitGlobalBufferPool(siloMessagingOptions); try { grainFactory = Services.GetRequiredService <GrainFactory>(); } catch (InvalidOperationException exc) { logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc); throw; } // Performance metrics siloStatistics = Services.GetRequiredService <SiloStatisticsManager>(); // The scheduler scheduler = Services.GetRequiredService <OrleansTaskScheduler>(); healthCheckParticipants.Add(scheduler); runtimeClient = Services.GetRequiredService <InsideRuntimeClient>(); // Initialize the message center messageCenter = Services.GetRequiredService <MessageCenter>(); var dispatcher = this.Services.GetRequiredService <Dispatcher>(); messageCenter.RerouteHandler = dispatcher.RerouteMessage; messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage; // GrainRuntime can be created only here, after messageCenter was created. grainRuntime = Services.GetRequiredService <IGrainRuntime>(); StreamProviderManager = Services.GetRequiredService <IStreamProviderManager>(); // Now the router/directory service // This has to come after the message center //; note that it then gets injected back into the message center.; localGrainDirectory = Services.GetRequiredService <LocalGrainDirectory>(); // Now the activation directory. activationDirectory = Services.GetRequiredService <ActivationDirectory>(); // Now the consistent ring provider RingProvider = Services.GetRequiredService <IConsistentRingProvider>(); catalog = Services.GetRequiredService <Catalog>(); siloStatistics.MetricsTable.Scheduler = scheduler; siloStatistics.MetricsTable.ActivationDirectory = activationDirectory; siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector; siloStatistics.MetricsTable.MessageCenter = messageCenter; executorService = Services.GetRequiredService <ExecutorService>(); // Now the incoming message agents var messageFactory = this.Services.GetRequiredService <MessageFactory>(); incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory); incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory); incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory); membershipOracle = Services.GetRequiredService <IMembershipOracle>(); if (!this.GlobalConfig.HasMultiClusterNetwork) { logger.Info("Skip multicluster oracle creation (no multicluster network configured)"); } else { multiClusterOracle = Services.GetRequiredService <IMultiClusterOracle>(); } this.SystemStatus = SystemStatus.Created; AsynchAgent.IsStarting = false; StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME, () => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs. var fullSiloLifecycle = this.Services.GetRequiredService <SiloLifecycle>(); this.siloLifecycle = fullSiloLifecycle; IEnumerable <ILifecycleParticipant <ISiloLifecycle> > lifecyleParticipants = this.Services.GetServices <ILifecycleParticipant <ISiloLifecycle> >(); foreach (ILifecycleParticipant <ISiloLifecycle> participant in lifecyleParticipants) { participant.Participate(fullSiloLifecycle); } this.Participate(fullSiloLifecycle); logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode()); }
private void DoStart() { lock (lockable) { if (SystemStatus.Current != SystemStatus.Created) { throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Start state. This silo is in the {0} state.", SystemStatus.Current)); } SystemStatus.Current = SystemStatus.Starting; } logger.Info(ErrorCode.SiloStarting, "Silo Start()"); // Hook up to receive notification of process exit / Ctrl-C events AppDomain.CurrentDomain.ProcessExit += HandleProcessExit; Console.CancelKeyPress += HandleProcessExit; ConfigureThreadPoolAndServicePointSettings(); // This has to start first so that the directory system target factory gets loaded before we start the router. typeManager.Start(); InsideRuntimeClient.Current.Start(); // The order of these 4 is pretty much arbitrary. scheduler.Start(); messageCenter.Start(); incomingPingAgent.Start(); incomingSystemAgent.Start(); incomingAgent.Start(); LocalGrainDirectory.Start(); // Set up an execution context for this thread so that the target creation steps can use asynch values. RuntimeContext.InitializeMainThread(); SiloProviderRuntime.Initialize(GlobalConfig); statisticsProviderManager = new StatisticsProviderManager("Statistics", SiloProviderRuntime.Instance); string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations) .WaitForResultWithThrow(initTimeout); if (statsProviderName != null) { LocalConfig.StatisticsProviderName = statsProviderName; } // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress). siloStatistics.SetSiloStatsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout); siloStatistics.SetSiloMetricsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout); membershipOracle = membershipFactory.CreateMembershipOracle(this).WaitForResultWithThrow(initTimeout); // This has to follow the above steps that start the runtime components CreateSystemTargets(); InjectDependencies(); // ensure this runs in the grain context, wait for it to complete scheduler.QueueTask(CreateSystemGrains, catalog.SchedulingContext) .WaitWithThrow(initTimeout); // Initialize storage providers once we have a basic silo runtime environment operating storageProviderManager = new StorageProviderManager(); scheduler.QueueTask( () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); catalog.SetStorageManager(storageProviderManager); // Load and init stream providers before silo becomes active var siloStreamProviderManager = new Orleans.Streams.StreamProviderManager(); scheduler.QueueTask( () => siloStreamProviderManager.LoadStreamProviders(this.GlobalConfig.ProviderConfigurations, SiloProviderRuntime.Instance), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); InsideRuntimeClient.Current.CurrentStreamProviderManager = siloStreamProviderManager; ISchedulingContext statusOracleContext = ((SystemTarget)LocalSiloStatusOracle).SchedulingContext; bool waitForPrimaryToStart = globalConfig.PrimaryNodeIsRequired && siloType != SiloType.Primary; scheduler.QueueTask(() => LocalSiloStatusOracle.Start(waitForPrimaryToStart), statusOracleContext) .WaitWithThrow(initTimeout); scheduler.QueueTask(LocalSiloStatusOracle.BecomeActive, statusOracleContext) .WaitWithThrow(initTimeout); try { siloStatistics.Start(LocalConfig); // Finally, initialize the deployment load collector, for grains with load-based placement scheduler.QueueTask(DeploymentLoadPublisher.Instance.Start, DeploymentLoadPublisher.Instance.SchedulingContext) .WaitWithThrow(initTimeout); // Start background timer tick to watch for platform execution stalls, such as when GC kicks in platformWatchdog = new Watchdog(nodeConfig.StatisticsLogWriteInterval, healthCheckParticipants); platformWatchdog.Start(); // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service scheduler.QueueTask(reminderService.Start, ((SystemTarget)reminderService).SchedulingContext) .WaitWithThrow(initTimeout); // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active). scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); var bootstrapProviderManager = new BootstrapProviderManager(); scheduler.QueueTask( () => bootstrapProviderManager.LoadAppBootstrapProviders(GlobalConfig.ProviderConfigurations), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); BootstrapProviders = bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics // Now that we're active, we can start the gateway var mc = messageCenter as MessageCenter; if (mc != null) { mc.StartGateway(); } SystemStatus.Current = SystemStatus.Running; } catch (Exception exc) { SafeExecute(() => logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", SiloAddress), exc)); FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale. throw; } }
public Silo(ILocalSiloDetails siloDetails, IServiceProvider services) { string name = siloDetails.Name; // Temporarily still require this. Hopefuly gone when 2.0 is released. this.siloDetails = siloDetails; this.SystemStatus = SystemStatus.Creating; AsynchAgent.IsStarting = true; // todo. use ISiloLifecycle instead? var startTime = DateTime.UtcNow; IOptions <ClusterMembershipOptions> clusterMembershipOptions = services.GetRequiredService <IOptions <ClusterMembershipOptions> >(); initTimeout = clusterMembershipOptions.Value.MaxJoinAttemptTime; if (Debugger.IsAttached) { initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), clusterMembershipOptions.Value.MaxJoinAttemptTime); stopTimeout = initTimeout; } var localEndpoint = this.siloDetails.SiloAddress.Endpoint; services.GetService <SerializationManager>().RegisterSerializers(services.GetService <IApplicationPartManager>()); this.Services = services; this.Services.InitializeSiloUnobservedExceptionsHandler(); //set PropagateActivityId flag from node config IOptions <SiloMessagingOptions> messagingOptions = services.GetRequiredService <IOptions <SiloMessagingOptions> >(); RequestContext.PropagateActivityId = messagingOptions.Value.PropagateActivityId; this.loggerFactory = this.Services.GetRequiredService <ILoggerFactory>(); logger = this.loggerFactory.CreateLogger <Silo>(); logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode)); if (!GCSettings.IsServerGC) { logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\">"); logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines)."); } logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing silo on host {0} MachineName {1} at {2}, gen {3} --------------", this.siloDetails.DnsHostName, Environment.MachineName, localEndpoint, this.siloDetails.SiloAddress.Generation); logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0}", name); var siloMessagingOptions = this.Services.GetRequiredService <IOptions <SiloMessagingOptions> >(); BufferPool.InitGlobalBufferPool(siloMessagingOptions.Value); try { grainFactory = Services.GetRequiredService <GrainFactory>(); } catch (InvalidOperationException exc) { logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc); throw; } // Performance metrics siloStatistics = Services.GetRequiredService <SiloStatisticsManager>(); // The scheduler scheduler = Services.GetRequiredService <OrleansTaskScheduler>(); healthCheckParticipants.Add(scheduler); runtimeClient = Services.GetRequiredService <InsideRuntimeClient>(); // Initialize the message center messageCenter = Services.GetRequiredService <MessageCenter>(); var dispatcher = this.Services.GetRequiredService <Dispatcher>(); messageCenter.RerouteHandler = dispatcher.RerouteMessage; messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage; // Now the router/directory service // This has to come after the message center //; note that it then gets injected back into the message center.; localGrainDirectory = Services.GetRequiredService <LocalGrainDirectory>(); // Now the activation directory. activationDirectory = Services.GetRequiredService <ActivationDirectory>(); // Now the consistent ring provider RingProvider = Services.GetRequiredService <IConsistentRingProvider>(); catalog = Services.GetRequiredService <Catalog>(); executorService = Services.GetRequiredService <ExecutorService>(); // Now the incoming message agents var messageFactory = this.Services.GetRequiredService <MessageFactory>(); incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory); incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory); incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory); membershipOracle = Services.GetRequiredService <IMembershipOracle>(); this.clusterOptions = Services.GetRequiredService <IOptions <ClusterOptions> >().Value; var multiClusterOptions = Services.GetRequiredService <IOptions <MultiClusterOptions> >().Value; if (!multiClusterOptions.HasMultiClusterNetwork) { logger.Info("Skip multicluster oracle creation (no multicluster network configured)"); } else { multiClusterOracle = Services.GetRequiredService <IMultiClusterOracle>(); } this.SystemStatus = SystemStatus.Created; AsynchAgent.IsStarting = false; StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME, () => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs. this.siloLifecycle = this.Services.GetRequiredService <ISiloLifecycleSubject>(); // register all lifecycle participants IEnumerable <ILifecycleParticipant <ISiloLifecycle> > lifecycleParticipants = this.Services.GetServices <ILifecycleParticipant <ISiloLifecycle> >(); foreach (ILifecycleParticipant <ISiloLifecycle> participant in lifecycleParticipants) { participant?.Participate(this.siloLifecycle); } // register all named lifecycle participants IKeyedServiceCollection <string, ILifecycleParticipant <ISiloLifecycle> > namedLifecycleParticipantCollection = this.Services.GetService <IKeyedServiceCollection <string, ILifecycleParticipant <ISiloLifecycle> > >(); foreach (ILifecycleParticipant <ISiloLifecycle> participant in namedLifecycleParticipantCollection ?.GetServices(this.Services) ?.Select(s => s.GetService(this.Services))) { participant?.Participate(this.siloLifecycle); } // add self to lifecycle this.Participate(this.siloLifecycle); logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode()); }
internal Silo(string name, SiloType siloType, ClusterConfiguration config, ILocalDataStore keyStore) { SystemStatus.Current = SystemStatus.Creating; CurrentSilo = this; var startTime = DateTime.UtcNow; this.siloType = siloType; Name = name; siloTerminatedEvent = new ManualResetEvent(false); OrleansConfig = config; globalConfig = config.Globals; config.OnConfigChange("Defaults", () => nodeConfig = config.GetConfigurationForNode(name)); if (!TraceLogger.IsInitialized) { TraceLogger.Initialize(nodeConfig); } config.OnConfigChange("Defaults/Tracing", () => TraceLogger.Initialize(nodeConfig, true), false); LimitManager.Initialize(nodeConfig); ActivationData.Init(config); StatisticsCollector.Initialize(nodeConfig); SerializationManager.Initialize(globalConfig.UseStandardSerializer); initTimeout = globalConfig.MaxJoinAttemptTime; if (Debugger.IsAttached) { initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), globalConfig.MaxJoinAttemptTime); stopTimeout = initTimeout; } IPEndPoint here = nodeConfig.Endpoint; int generation = nodeConfig.Generation; if (generation == 0) { generation = SiloAddress.AllocateNewGeneration(); nodeConfig.Generation = generation; } TraceLogger.MyIPEndPoint = here; logger = TraceLogger.GetLogger("Silo", TraceLogger.LoggerType.Runtime); logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on {1} at {2}, gen {3} --------------", siloType, nodeConfig.DNSHostName, here, generation); logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with runtime Version='{1}' Config= \n{2}", name, RuntimeVersion.Current, config.ToString(name)); if (keyStore != null) { // Re-establish reference to shared local key store in this app domain LocalDataStoreInstance.LocalDataStore = keyStore; } healthCheckParticipants = new List <IHealthCheckParticipant>(); BufferPool.InitGlobalBufferPool(globalConfig); PlacementStrategy.Initialize(globalConfig); UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnobservedExceptionHandler); AppDomain.CurrentDomain.UnhandledException += (obj, ev) => DomainUnobservedExceptionHandler(obj, (Exception)ev.ExceptionObject); typeManager = new GrainTypeManager(here.Address.Equals(IPAddress.Loopback)); // Performance metrics siloStatistics = new SiloStatisticsManager(globalConfig, nodeConfig); config.OnConfigChange("Defaults/LoadShedding", () => siloStatistics.MetricsTable.NodeConfig = nodeConfig, false); // The scheduler scheduler = new OrleansTaskScheduler(globalConfig, nodeConfig); healthCheckParticipants.Add(scheduler); // Initialize the message center var mc = new MessageCenter(here, generation, globalConfig, siloStatistics.MetricsTable); if (nodeConfig.IsGatewayNode) { mc.InstallGateway(nodeConfig.ProxyGatewayEndpoint); } messageCenter = mc; // Now the router/directory service // This has to come after the message center //; note that it then gets injected back into the message center.; localGrainDirectory = new LocalGrainDirectory(this); // Now the activation directory. // This needs to know which router to use so that it can keep the global directory in synch with the local one. activationDirectory = new ActivationDirectory(); // Now the consistent ring provider RingProvider = GlobalConfig.UseVirtualBucketsConsistentRing ? (IConsistentRingProvider) new VirtualBucketsRingProvider(SiloAddress, GlobalConfig.NumVirtualBucketsConsistentRing) : new ConsistentRingProvider(SiloAddress); Action <Dispatcher> setDispatcher; catalog = new Catalog(Constants.CatalogId, SiloAddress, Name, LocalGrainDirectory, typeManager, scheduler, activationDirectory, config, out setDispatcher); var dispatcher = new Dispatcher(scheduler, messageCenter, catalog, config); setDispatcher(dispatcher); RuntimeClient.Current = new InsideRuntimeClient( dispatcher, catalog, LocalGrainDirectory, SiloAddress, config, RingProvider, typeManager); messageCenter.RerouteHandler = InsideRuntimeClient.Current.RerouteMessage; messageCenter.SniffIncomingMessage = InsideRuntimeClient.Current.SniffIncomingMessage; messageCenter.ClientDropHandler = grainIds => { catalog.DeleteGrainsLocal(grainIds).Ignore(); scheduler.RunOrQueueAction(() => { // Consider: batch delete foreach (var id in grainIds) { LocalGrainDirectory.DeleteGrain(id).Ignore(); } }, catalog.SchedulingContext); }; siloStatistics.MetricsTable.Scheduler = scheduler; siloStatistics.MetricsTable.ActivationDirectory = activationDirectory; siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector; siloStatistics.MetricsTable.MessageCenter = messageCenter; DeploymentLoadPublisher.CreateDeploymentLoadPublisher(this, globalConfig); PlacementDirectorsManager.CreatePlacementDirectorsManager(globalConfig); // Now the incoming message agents incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, dispatcher); incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, dispatcher); incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, dispatcher); membershipFactory = new MembershipFactory(); reminderFactory = new LocalReminderServiceFactory(); SystemStatus.Current = SystemStatus.Created; StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME, () => TraceLogger.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs. TestHookup = new TestHookups(this); logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode()); }
internal Silo(string name, SiloType siloType, ClusterConfiguration config, ILocalDataStore keyStore) { SystemStatus.Current = SystemStatus.Creating; CurrentSilo = this; var startTime = DateTime.UtcNow; this.siloType = siloType; Name = name; siloTerminatedEvent = new ManualResetEvent(false); OrleansConfig = config; globalConfig = config.Globals; config.OnConfigChange("Defaults", () => nodeConfig = config.GetConfigurationForNode(name)); if (!TraceLogger.IsInitialized) { TraceLogger.Initialize(nodeConfig); } config.OnConfigChange("Defaults/Tracing", () => TraceLogger.Initialize(nodeConfig, true), false); ActivationData.Init(config, nodeConfig); StatisticsCollector.Initialize(nodeConfig); SerializationManager.Initialize(globalConfig.UseStandardSerializer); initTimeout = globalConfig.MaxJoinAttemptTime; if (Debugger.IsAttached) { initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), globalConfig.MaxJoinAttemptTime); stopTimeout = initTimeout; } IPEndPoint here = nodeConfig.Endpoint; int generation = nodeConfig.Generation; if (generation == 0) { generation = SiloAddress.AllocateNewGeneration(); nodeConfig.Generation = generation; } TraceLogger.MyIPEndPoint = here; logger = TraceLogger.GetLogger("Silo", TraceLogger.LoggerType.Runtime); logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode)); if (!GCSettings.IsServerGC || !GCSettings.LatencyMode.Equals(GCLatencyMode.Batch)) { logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on or with GCLatencyMode.Batch enabled - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\"> and <configuration>-<runtime>-<gcConcurrent enabled=\"false\"/>"); } logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------", siloType, nodeConfig.DNSHostName, Environment.MachineName, here, generation); logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with runtime Version='{1}' .NET version='{2}' Is .NET 4.5={3} OS version='{4}' Config= " + Environment.NewLine + "{5}", name, RuntimeVersion.Current, Environment.Version, ConfigUtilities.IsNet45OrNewer(), Environment.OSVersion, config.ToString(name)); if (keyStore != null) { // Re-establish reference to shared local key store in this app domain LocalDataStoreInstance.LocalDataStore = keyStore; } healthCheckParticipants = new List <IHealthCheckParticipant>(); allSiloProviders = new List <IProvider>(); BufferPool.InitGlobalBufferPool(globalConfig); PlacementStrategy.Initialize(globalConfig); UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnobservedExceptionHandler); AppDomain.CurrentDomain.UnhandledException += (obj, ev) => DomainUnobservedExceptionHandler(obj, (Exception)ev.ExceptionObject); grainFactory = new GrainFactory(); typeManager = new GrainTypeManager(here.Address.Equals(IPAddress.Loopback), grainFactory); // Performance metrics siloStatistics = new SiloStatisticsManager(globalConfig, nodeConfig); config.OnConfigChange("Defaults/LoadShedding", () => siloStatistics.MetricsTable.NodeConfig = nodeConfig, false); // The scheduler scheduler = new OrleansTaskScheduler(globalConfig, nodeConfig); healthCheckParticipants.Add(scheduler); // Initialize the message center var mc = new MessageCenter(here, generation, globalConfig, siloStatistics.MetricsTable); if (nodeConfig.IsGatewayNode) { mc.InstallGateway(nodeConfig.ProxyGatewayEndpoint); } messageCenter = mc; SiloIdentity = SiloAddress.ToLongString(); // GrainRuntime can be created only here, after messageCenter was created. grainRuntime = new GrainRuntime( globalConfig.ServiceId, SiloIdentity, grainFactory, new TimerRegistry(), new ReminderRegistry(), new StreamProviderManager()); // Now the router/directory service // This has to come after the message center //; note that it then gets injected back into the message center.; localGrainDirectory = new LocalGrainDirectory(this); // Now the activation directory. // This needs to know which router to use so that it can keep the global directory in synch with the local one. activationDirectory = new ActivationDirectory(); // Now the consistent ring provider RingProvider = GlobalConfig.UseVirtualBucketsConsistentRing ? (IConsistentRingProvider) new VirtualBucketsRingProvider(SiloAddress, GlobalConfig.NumVirtualBucketsConsistentRing) : new ConsistentRingProvider(SiloAddress); Action <Dispatcher> setDispatcher; catalog = new Catalog(Constants.CatalogId, SiloAddress, Name, LocalGrainDirectory, typeManager, scheduler, activationDirectory, config, grainRuntime, out setDispatcher); var dispatcher = new Dispatcher(scheduler, messageCenter, catalog, config); setDispatcher(dispatcher); RuntimeClient.Current = new InsideRuntimeClient( dispatcher, catalog, LocalGrainDirectory, SiloAddress, config, RingProvider, typeManager, grainFactory); messageCenter.RerouteHandler = InsideRuntimeClient.Current.RerouteMessage; messageCenter.SniffIncomingMessage = InsideRuntimeClient.Current.SniffIncomingMessage; siloStatistics.MetricsTable.Scheduler = scheduler; siloStatistics.MetricsTable.ActivationDirectory = activationDirectory; siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector; siloStatistics.MetricsTable.MessageCenter = messageCenter; DeploymentLoadPublisher.CreateDeploymentLoadPublisher(this, globalConfig); PlacementDirectorsManager.CreatePlacementDirectorsManager(globalConfig); // Now the incoming message agents incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, dispatcher); incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, dispatcher); incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, dispatcher); membershipFactory = new MembershipFactory(); reminderFactory = new LocalReminderServiceFactory(); SystemStatus.Current = SystemStatus.Created; StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME, () => TraceLogger.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs. TestHookup = new TestHookups(this); logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode()); }
public Silo(ILocalSiloDetails siloDetails, IServiceProvider services) { string name = siloDetails.Name; // Temporarily still require this. Hopefuly gone when 2.0 is released. this.siloDetails = siloDetails; this.SystemStatus = SystemStatus.Creating; var startTime = DateTime.UtcNow; IOptions <ClusterMembershipOptions> clusterMembershipOptions = services.GetRequiredService <IOptions <ClusterMembershipOptions> >(); initTimeout = clusterMembershipOptions.Value.MaxJoinAttemptTime; if (Debugger.IsAttached) { initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), clusterMembershipOptions.Value.MaxJoinAttemptTime); stopTimeout = initTimeout; } var localEndpoint = this.siloDetails.SiloAddress.Endpoint; this.Services = services; //set PropagateActivityId flag from node config IOptions <SiloMessagingOptions> messagingOptions = services.GetRequiredService <IOptions <SiloMessagingOptions> >(); RequestContext.PropagateActivityId = messagingOptions.Value.PropagateActivityId; this.waitForMessageToBeQueuedForOutbound = messagingOptions.Value.WaitForMessageToBeQueuedForOutboundTime; this.loggerFactory = this.Services.GetRequiredService <ILoggerFactory>(); logger = this.loggerFactory.CreateLogger <Silo>(); logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode)); if (!GCSettings.IsServerGC) { logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\">"); logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines)."); } if (logger.IsEnabled(LogLevel.Debug)) { var highestLogLevel = logger.IsEnabled(LogLevel.Trace) ? nameof(LogLevel.Trace) : nameof(LogLevel.Debug); logger.LogWarning( new EventId((int)ErrorCode.SiloGcWarning), $"A verbose logging level ({highestLogLevel}) is configured. This will impact performance. The recommended log level is {nameof(LogLevel.Information)}."); } logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing silo on host {0} MachineName {1} at {2}, gen {3} --------------", this.siloDetails.DnsHostName, Environment.MachineName, localEndpoint, this.siloDetails.SiloAddress.Generation); logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0}", name); try { grainFactory = Services.GetRequiredService <GrainFactory>(); } catch (InvalidOperationException exc) { logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc); throw; } // Performance metrics siloStatistics = Services.GetRequiredService <SiloStatisticsManager>(); runtimeClient = Services.GetRequiredService <InsideRuntimeClient>(); // Initialize the message center messageCenter = Services.GetRequiredService <MessageCenter>(); messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage; // Now the router/directory service // This has to come after the message center //; note that it then gets injected back into the message center.; localGrainDirectory = Services.GetRequiredService <LocalGrainDirectory>(); // Now the consistent ring provider RingProvider = Services.GetRequiredService <IConsistentRingProvider>(); catalog = Services.GetRequiredService <Catalog>(); siloStatusOracle = Services.GetRequiredService <ISiloStatusOracle>(); this.membershipService = Services.GetRequiredService <IMembershipService>(); this.SystemStatus = SystemStatus.Created; StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME, () => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs. this.siloLifecycle = this.Services.GetRequiredService <ISiloLifecycleSubject>(); // register all lifecycle participants IEnumerable <ILifecycleParticipant <ISiloLifecycle> > lifecycleParticipants = this.Services.GetServices <ILifecycleParticipant <ISiloLifecycle> >(); foreach (ILifecycleParticipant <ISiloLifecycle> participant in lifecycleParticipants) { participant?.Participate(this.siloLifecycle); } // register all named lifecycle participants IKeyedServiceCollection <string, ILifecycleParticipant <ISiloLifecycle> > namedLifecycleParticipantCollection = this.Services.GetService <IKeyedServiceCollection <string, ILifecycleParticipant <ISiloLifecycle> > >(); foreach (ILifecycleParticipant <ISiloLifecycle> participant in namedLifecycleParticipantCollection ?.GetServices(this.Services) ?.Select(s => s.GetService(this.Services))) { participant?.Participate(this.siloLifecycle); } // add self to lifecycle this.Participate(this.siloLifecycle); logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode()); }
private void DoStart() { lock (lockable) { if (!this.SystemStatus.Equals(SystemStatus.Created)) { throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", this.SystemStatus)); } this.SystemStatus = SystemStatus.Starting; } logger.Info(ErrorCode.SiloStarting, "Silo Start()"); // Hook up to receive notification of process exit / Ctrl-C events AppDomain.CurrentDomain.ProcessExit += HandleProcessExit; if (GlobalConfig.FastKillOnCancelKeyPress) { Console.CancelKeyPress += HandleProcessExit; } ConfigureThreadPoolAndServicePointSettings(); // This has to start first so that the directory system target factory gets loaded before we start the router. grainTypeManager.Start(); runtimeClient.Start(); // The order of these 4 is pretty much arbitrary. scheduler.Start(); messageCenter.Start(); incomingPingAgent.Start(); incomingSystemAgent.Start(); incomingAgent.Start(); LocalGrainDirectory.Start(); // Set up an execution context for this thread so that the target creation steps can use asynch values. RuntimeContext.InitializeMainThread(); // Initialize the implicit stream subscribers table. var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>(); implicitStreamSubscriberTable.InitImplicitStreamSubscribers(this.grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray()); var siloProviderRuntime = Services.GetRequiredService <SiloProviderRuntime>(); runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime; statisticsProviderManager = this.Services.GetRequiredService <StatisticsProviderManager>(); string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations) .WaitForResultWithThrow(initTimeout); if (statsProviderName != null) { LocalConfig.StatisticsProviderName = statsProviderName; } // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress). siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout); siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout); // This has to follow the above steps that start the runtime components CreateSystemTargets(); InjectDependencies(); // Validate the configuration. GlobalConfig.Application.ValidateConfiguration(logger); // Initialize storage providers once we have a basic silo runtime environment operating storageProviderManager = this.Services.GetRequiredService <StorageProviderManager>(); scheduler.QueueTask( () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); ITransactionAgent transactionAgent = this.Services.GetRequiredService <ITransactionAgent>(); ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext; scheduler.QueueTask(transactionAgent.Start, transactionAgentContext) .WaitWithThrow(initTimeout); var versionStore = Services.GetService <IVersionStore>() as GrainVersionStore; versionStore?.SetStorageManager(storageProviderManager); if (logger.IsVerbose) { logger.Verbose("Storage provider manager created successfully."); } // Initialize log consistency providers once we have a basic silo runtime environment operating logConsistencyProviderManager = this.Services.GetRequiredService <LogConsistencyProviderManager>(); scheduler.QueueTask( () => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); if (logger.IsVerbose) { logger.Verbose("Log consistency provider manager created successfully."); } // Load and init stream providers before silo becomes active var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager; scheduler.QueueTask( () => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager; if (logger.IsVerbose) { logger.Verbose("Stream provider manager created successfully."); } // Load and init grain services before silo becomes active. CreateGrainServices(GlobalConfig.GrainServiceConfigurations); this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ?? this.providerManagerSystemTarget.SchedulingContext; scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext) .WaitWithThrow(initTimeout); if (logger.IsVerbose) { logger.Verbose("Local silo status oracle created successfully."); } scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext) .WaitWithThrow(initTimeout); if (logger.IsVerbose) { logger.Verbose("Local silo status oracle became active successfully."); } scheduler.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext) .WaitWithThrow(this.initTimeout); //if running in multi cluster scenario, start the MultiClusterNetwork Oracle if (GlobalConfig.HasMultiClusterNetwork) { logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.", GlobalConfig.ServiceId, GlobalConfig.ClusterId); this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ?? this.providerManagerSystemTarget.SchedulingContext; scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext) .WaitWithThrow(initTimeout); if (logger.IsVerbose) { logger.Verbose("multicluster oracle created successfully."); } } try { this.siloStatistics.Start(this.LocalConfig); if (this.logger.IsVerbose) { this.logger.Verbose("Silo statistics manager started successfully."); } // Finally, initialize the deployment load collector, for grains with load-based placement var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>(); this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext) .WaitWithThrow(this.initTimeout); if (this.logger.IsVerbose) { this.logger.Verbose("Silo deployment load publisher started successfully."); } // Start background timer tick to watch for platform execution stalls, such as when GC kicks in this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants); this.platformWatchdog.Start(); if (this.logger.IsVerbose) { this.logger.Verbose("Silo platform watchdog started successfully."); } if (this.reminderService != null) { // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ?? this.providerManagerSystemTarget.SchedulingContext; this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext) .WaitWithThrow(this.initTimeout); if (this.logger.IsVerbose) { this.logger.Verbose("Reminder service started successfully."); } } this.bootstrapProviderManager = this.Services.GetRequiredService <BootstrapProviderManager>(); this.scheduler.QueueTask( () => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations), this.providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(this.initTimeout); this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics if (this.logger.IsVerbose) { this.logger.Verbose("App bootstrap calls done successfully."); } // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active). // also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive. this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(this.initTimeout); if (this.logger.IsVerbose) { this.logger.Verbose("Stream providers started successfully."); } // Now that we're active, we can start the gateway var mc = this.messageCenter as MessageCenter; mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>()); if (this.logger.IsVerbose) { this.logger.Verbose("Message gateway service started successfully."); } this.SystemStatus = SystemStatus.Running; } catch (Exception exc) { this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc)); this.FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale. throw; } if (logger.IsVerbose) { logger.Verbose("Silo.Start complete: System status = {0}", this.SystemStatus); } }
internal Silo(SiloInitializationParameters initializationParams, IServiceProvider services) { string name = initializationParams.Name; ClusterConfiguration config = initializationParams.ClusterConfig; this.initializationParams = initializationParams; this.SystemStatus = SystemStatus.Creating; AsynchAgent.IsStarting = true; var startTime = DateTime.UtcNow; if (!LogManager.IsInitialized) { LogManager.Initialize(LocalConfig); } services?.GetService <TelemetryManager>()?.AddFromConfiguration(services, LocalConfig.TelemetryConfiguration); config.OnConfigChange("Defaults/Tracing", () => LogManager.Initialize(LocalConfig, true), false); StatisticsCollector.Initialize(LocalConfig); initTimeout = GlobalConfig.MaxJoinAttemptTime; if (Debugger.IsAttached) { initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), GlobalConfig.MaxJoinAttemptTime); stopTimeout = initTimeout; } var localEndpoint = this.initializationParams.SiloAddress.Endpoint; LogManager.MyIPEndPoint = localEndpoint; logger = LogManager.GetLogger("Silo", LoggerType.Runtime); logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode)); if (!GCSettings.IsServerGC) { logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\">"); logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines)."); } logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------", this.initializationParams.Type, LocalConfig.DNSHostName, Environment.MachineName, localEndpoint, this.initializationParams.SiloAddress.Generation); logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with the following configuration= " + Environment.NewLine + "{1}", name, config.ToString(name)); // Configure DI using Startup type if (services == null) { var serviceCollection = new ServiceCollection(); serviceCollection.AddSingleton <Silo>(this); serviceCollection.AddSingleton(initializationParams); DefaultSiloServices.AddDefaultServices(serviceCollection); services = StartupBuilder.ConfigureStartup(this.LocalConfig.StartupTypeName, serviceCollection); services.GetService <TelemetryManager>()?.AddFromConfiguration(services, LocalConfig.TelemetryConfiguration); } this.Services = services; this.assemblyProcessor = this.Services.GetRequiredService <AssemblyProcessor>(); this.assemblyProcessor.Initialize(); BufferPool.InitGlobalBufferPool(GlobalConfig); if (!UnobservedExceptionsHandlerClass.TrySetUnobservedExceptionHandler(UnobservedExceptionHandler)) { logger.Warn(ErrorCode.Runtime_Error_100153, "Unable to set unobserved exception handler because it was already set."); } AppDomain.CurrentDomain.UnhandledException += this.DomainUnobservedExceptionHandler; try { grainFactory = Services.GetRequiredService <GrainFactory>(); } catch (InvalidOperationException exc) { logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc); throw; } grainTypeManager = Services.GetRequiredService <GrainTypeManager>(); // Performance metrics siloStatistics = Services.GetRequiredService <SiloStatisticsManager>(); // The scheduler scheduler = Services.GetRequiredService <OrleansTaskScheduler>(); healthCheckParticipants.Add(scheduler); runtimeClient = Services.GetRequiredService <InsideRuntimeClient>(); // Initialize the message center messageCenter = Services.GetRequiredService <MessageCenter>(); var dispatcher = this.Services.GetRequiredService <Dispatcher>(); messageCenter.RerouteHandler = dispatcher.RerouteMessage; messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage; // GrainRuntime can be created only here, after messageCenter was created. grainRuntime = Services.GetRequiredService <IGrainRuntime>(); // Now the router/directory service // This has to come after the message center //; note that it then gets injected back into the message center.; localGrainDirectory = Services.GetRequiredService <LocalGrainDirectory>(); // Now the activation directory. activationDirectory = Services.GetRequiredService <ActivationDirectory>(); // Now the consistent ring provider RingProvider = Services.GetRequiredService <IConsistentRingProvider>(); catalog = Services.GetRequiredService <Catalog>(); siloStatistics.MetricsTable.Scheduler = scheduler; siloStatistics.MetricsTable.ActivationDirectory = activationDirectory; siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector; siloStatistics.MetricsTable.MessageCenter = messageCenter; // Now the incoming message agents var messageFactory = this.Services.GetRequiredService <MessageFactory>(); incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory); incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory); incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory); membershipOracle = Services.GetRequiredService <IMembershipOracle>(); if (!this.GlobalConfig.HasMultiClusterNetwork) { logger.Info("Skip multicluster oracle creation (no multicluster network configured)"); } else { multiClusterOracle = Services.GetRequiredService <IMultiClusterOracle>(); } this.SystemStatus = SystemStatus.Created; AsynchAgent.IsStarting = false; StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME, () => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs. logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode()); }
private void DoStart() { lock (lockable) { if (!SystemStatus.Current.Equals(SystemStatus.Created)) { throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", SystemStatus.Current)); } SystemStatus.Current = SystemStatus.Starting; } logger.Info(ErrorCode.SiloStarting, "Silo Start()"); // Hook up to receive notification of process exit / Ctrl-C events AppDomain.CurrentDomain.ProcessExit += HandleProcessExit; Console.CancelKeyPress += HandleProcessExit; ConfigureThreadPoolAndServicePointSettings(); // This has to start first so that the directory system target factory gets loaded before we start the router. typeManager.Start(); InsideRuntimeClient.Current.Start(); // The order of these 4 is pretty much arbitrary. scheduler.Start(); messageCenter.Start(); incomingPingAgent.Start(); incomingSystemAgent.Start(); incomingAgent.Start(); LocalGrainDirectory.Start(); // Set up an execution context for this thread so that the target creation steps can use asynch values. RuntimeContext.InitializeMainThread(); SiloProviderRuntime.Initialize(GlobalConfig, SiloIdentity, grainFactory, Services); InsideRuntimeClient.Current.CurrentStreamProviderRuntime = SiloProviderRuntime.Instance; statisticsProviderManager = new StatisticsProviderManager("Statistics", SiloProviderRuntime.Instance); string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations) .WaitForResultWithThrow(initTimeout); if (statsProviderName != null) { LocalConfig.StatisticsProviderName = statsProviderName; } allSiloProviders.AddRange(statisticsProviderManager.GetProviders()); // can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress). siloStatistics.SetSiloStatsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout); siloStatistics.SetSiloMetricsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout); IMembershipTable membershipTable = membershipFactory.GetMembershipTable(GlobalConfig.LivenessType, GlobalConfig.MembershipTableAssembly); membershipOracle = membershipFactory.CreateMembershipOracle(this, membershipTable); multiClusterOracle = multiClusterFactory.CreateGossipOracle(this).WaitForResultWithThrow(initTimeout); // This has to follow the above steps that start the runtime components CreateSystemTargets(); InjectDependencies(); // Validate the configuration. GlobalConfig.Application.ValidateConfiguration(logger); // ensure this runs in the grain context, wait for it to complete scheduler.QueueTask(CreateSystemGrains, catalog.SchedulingContext) .WaitWithThrow(initTimeout); if (logger.IsVerbose) { logger.Verbose("System grains created successfully."); } // Initialize storage providers once we have a basic silo runtime environment operating storageProviderManager = new StorageProviderManager(grainFactory, Services); scheduler.QueueTask( () => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); catalog.SetStorageManager(storageProviderManager); allSiloProviders.AddRange(storageProviderManager.GetProviders()); if (logger.IsVerbose) { logger.Verbose("Storage provider manager created successfully."); } // Load and init stream providers before silo becomes active var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager; scheduler.QueueTask( () => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, SiloProviderRuntime.Instance), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); InsideRuntimeClient.Current.CurrentStreamProviderManager = siloStreamProviderManager; allSiloProviders.AddRange(siloStreamProviderManager.GetProviders()); if (logger.IsVerbose) { logger.Verbose("Stream provider manager created successfully."); } ISchedulingContext statusOracleContext = ((SystemTarget)LocalSiloStatusOracle).SchedulingContext; bool waitForPrimaryToStart = globalConfig.PrimaryNodeIsRequired && siloType != SiloType.Primary; if (waitForPrimaryToStart) // only in MembershipTableGrain case. { scheduler.QueueTask(() => membershipFactory.WaitForTableToInit(membershipTable), statusOracleContext) .WaitWithThrow(initTimeout); } scheduler.QueueTask(() => membershipTable.InitializeMembershipTable(GlobalConfig, true, LogManager.GetLogger(membershipTable.GetType().Name)), statusOracleContext) .WaitWithThrow(initTimeout); scheduler.QueueTask(() => LocalSiloStatusOracle.Start(), statusOracleContext) .WaitWithThrow(initTimeout); if (logger.IsVerbose) { logger.Verbose("Local silo status oracle created successfully."); } scheduler.QueueTask(LocalSiloStatusOracle.BecomeActive, statusOracleContext) .WaitWithThrow(initTimeout); if (logger.IsVerbose) { logger.Verbose("Local silo status oracle became active successfully."); } //if running in multi cluster scenario, start the MultiClusterNetwork Oracle if (GlobalConfig.HasMultiClusterNetwork) { logger.Info("Creating multicluster oracle with my ServiceId={0} and ClusterId={1}.", GlobalConfig.ServiceId, GlobalConfig.ClusterId); ISchedulingContext clusterStatusContext = ((SystemTarget)multiClusterOracle).SchedulingContext; scheduler.QueueTask(() => multiClusterOracle.Start(LocalSiloStatusOracle), clusterStatusContext) .WaitWithThrow(initTimeout); if (logger.IsVerbose) { logger.Verbose("multicluster oracle created successfully."); } } try { siloStatistics.Start(LocalConfig); if (logger.IsVerbose) { logger.Verbose("Silo statistics manager started successfully."); } // Finally, initialize the deployment load collector, for grains with load-based placement scheduler.QueueTask(DeploymentLoadPublisher.Instance.Start, DeploymentLoadPublisher.Instance.SchedulingContext) .WaitWithThrow(initTimeout); if (logger.IsVerbose) { logger.Verbose("Silo deployment load publisher started successfully."); } // Start background timer tick to watch for platform execution stalls, such as when GC kicks in platformWatchdog = new Watchdog(nodeConfig.StatisticsLogWriteInterval, healthCheckParticipants); platformWatchdog.Start(); if (logger.IsVerbose) { logger.Verbose("Silo platform watchdog started successfully."); } if (reminderService != null) { // so, we have the view of the membership in the consistentRingProvider. We can start the reminder service scheduler.QueueTask(reminderService.Start, ((SystemTarget)reminderService).SchedulingContext) .WaitWithThrow(initTimeout); if (logger.IsVerbose) { logger.Verbose("Reminder service started successfully."); } } bootstrapProviderManager = new BootstrapProviderManager(); scheduler.QueueTask( () => bootstrapProviderManager.LoadAppBootstrapProviders(GlobalConfig.ProviderConfigurations), providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); BootstrapProviders = bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics allSiloProviders.AddRange(BootstrapProviders); if (logger.IsVerbose) { logger.Verbose("App bootstrap calls done successfully."); } // Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active). // also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive. scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, providerManagerSystemTarget.SchedulingContext) .WaitWithThrow(initTimeout); if (logger.IsVerbose) { logger.Verbose("Stream providers started successfully."); } // Now that we're active, we can start the gateway var mc = messageCenter as MessageCenter; if (mc != null) { mc.StartGateway(clientRegistrar); } if (logger.IsVerbose) { logger.Verbose("Message gateway service started successfully."); } SystemStatus.Current = SystemStatus.Running; } catch (Exception exc) { SafeExecute(() => logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", SiloAddress), exc)); FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale. throw; } if (logger.IsVerbose) { logger.Verbose("Silo.Start complete: System status = {0}", SystemStatus.Current); } }