internal void ClientAdded(GrainId clientId)
{
// Use a ActivationId that is hashed from clientId, and not random ActivationId.
// That way, when we refresh it in the directiry, it's the same one.
var addr = GetClientActivationAddress(clientId);
scheduler.QueueTask(
() => ExecuteWithRetries(() => grainDirectory.RegisterAsync(addr, singleActivation: false), ErrorCode.ClientRegistrarFailedToRegister, String.Format("Directory.RegisterAsync {0} failed.", addr)),
this).Ignore();
}
private async Task OnRuntimeGrainServicesStart(CancellationToken ct)
{
var stopWatch = Stopwatch.StartNew();
// Load and init grain services before silo becomes active.
await StartAsyncTaskWithPerfAnalysis("Init grain services",
() => CreateGrainServices(), stopWatch);
var versionStore = Services.GetService <IVersionStore>();
await StartAsyncTaskWithPerfAnalysis("Init type manager", () => scheduler
.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager)
.WithTimeout(this.initTimeout, $"TypeManager Initializing failed due to timeout {initTimeout}"), stopWatch);
try
{
StatisticsOptions statisticsOptions = Services.GetRequiredService <IOptions <StatisticsOptions> >().Value;
StartTaskWithPerfAnalysis("Start silo statistics", () => this.siloStatistics.Start(statisticsOptions), stopWatch);
logger.Debug("Silo statistics manager started successfully.");
// Finally, initialize the deployment load collector, for grains with load-based placement
await StartAsyncTaskWithPerfAnalysis("Start deployment load collector", StartDeploymentLoadCollector, stopWatch);
async Task StartDeploymentLoadCollector()
{
var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
await this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher)
.WithTimeout(this.initTimeout, $"Starting DeploymentLoadPublisher failed due to timeout {initTimeout}");
logger.Debug("Silo deployment load publisher started successfully.");
}
// Start background timer tick to watch for platform execution stalls, such as when GC kicks in
var healthCheckParticipants = this.Services.GetService <IEnumerable <IHealthCheckParticipant> >().ToList();
this.platformWatchdog = new Watchdog(statisticsOptions.LogWriteInterval, healthCheckParticipants, this.loggerFactory.CreateLogger <Watchdog>());
this.platformWatchdog.Start();
if (this.logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Silo platform watchdog started successfully.");
}
}
catch (Exception exc)
{
this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
throw;
}
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Silo.Start complete: System status = {0}", this.SystemStatus);
}
}
private async Task <KeyValuePair <SiloAddress, GrainInterfaceMap> > GetTargetSiloGrainInterfaceMap(SiloAddress siloAddress)
{
try
{
var remoteTypeManager = this.grainFactory.GetSystemTarget <ISiloTypeManager>(Constants.TypeManagerId, siloAddress);
var siloTypeCodeMap = await scheduler.QueueTask(() => remoteTypeManager.GetSiloTypeCodeMap(), this);
return(new KeyValuePair <SiloAddress, GrainInterfaceMap>(siloAddress, siloTypeCodeMap));
}
catch (Exception ex)
{
// Will be retried on the next timer hit
logger.Error(ErrorCode.TypeManager_GetSiloGrainInterfaceMapError, $"Exception when trying to get GrainInterfaceMap for silos {siloAddress}", ex);
hasToRefreshClusterGrainInterfaceMap = true;
return(new KeyValuePair <SiloAddress, GrainInterfaceMap>(siloAddress, null));
}
}
public Task ReportMetrics(ISiloPerformanceMetrics metricsData)
{
logger.Info("{0} ReportMetrics called", GetType().Name);
taskScheduler.QueueTask(() => grain.ReportMetricsCalled(), schedulingContext).Ignore();
return(Task.CompletedTask);
}
private async Task OnRuntimeServicesStart(CancellationToken ct)
{
//TODO: Setup all (or as many as possible) of the class started in this call to work directly with lifecyce
// The order of these 4 is pretty much arbitrary.
scheduler.Start();
messageCenter.Start();
incomingPingAgent.Start();
incomingSystemAgent.Start();
incomingAgent.Start();
LocalGrainDirectory.Start();
// Set up an execution context for this thread so that the target creation steps can use asynch values.
RuntimeContext.InitializeMainThread();
// Initialize the implicit stream subscribers table.
var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>();
var grainTypeManager = Services.GetRequiredService <GrainTypeManager>();
implicitStreamSubscriberTable.InitImplicitStreamSubscribers(grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray());
var siloProviderRuntime = Services.GetRequiredService <SiloProviderRuntime>();
runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime;
statisticsProviderManager = this.Services.GetRequiredService <StatisticsProviderManager>();
string statsProviderName = await statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
.WithTimeout(initTimeout);
if (statsProviderName != null)
{
LocalConfig.StatisticsProviderName = statsProviderName;
}
// can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
await siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WithTimeout(initTimeout);
await siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WithTimeout(initTimeout);
// This has to follow the above steps that start the runtime components
CreateSystemTargets();
await InjectDependencies();
// Validate the configuration.
GlobalConfig.Application.ValidateConfiguration(logger);
// Initialize storage providers once we have a basic silo runtime environment operating
storageProviderManager = this.Services.GetRequiredService <StorageProviderManager>();
await scheduler.QueueTask(
() => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WithTimeout(initTimeout);
ITransactionAgent transactionAgent = this.Services.GetRequiredService <ITransactionAgent>();
ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext;
await scheduler.QueueTask(transactionAgent.Start, transactionAgentContext)
.WithTimeout(initTimeout);
var versionStore = Services.GetService <IVersionStore>() as GrainVersionStore;
versionStore?.SetStorageManager(storageProviderManager);
logger.Debug("Storage provider manager created successfully.");
// Initialize log consistency providers once we have a basic silo runtime environment operating
logConsistencyProviderManager = this.Services.GetRequiredService <LogConsistencyProviderManager>();
await scheduler.QueueTask(
() => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WithTimeout(initTimeout);
logger.Debug("Log consistency provider manager created successfully.");
// Load and init stream providers before silo becomes active
var siloStreamProviderManager = (StreamProviderManager)this.Services.GetRequiredService <IStreamProviderManager>();
await scheduler.QueueTask(
() => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime),
providerManagerSystemTarget.SchedulingContext)
.WithTimeout(initTimeout);
runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager;
logger.Debug("Stream provider manager created successfully.");
// Load and init grain services before silo becomes active.
await CreateGrainServices(GlobalConfig.GrainServiceConfigurations);
this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
this.providerManagerSystemTarget.SchedulingContext;
await scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
.WithTimeout(initTimeout);
logger.Debug("Local silo status oracle created successfully.");
await scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext)
.WithTimeout(initTimeout);
logger.Debug("Local silo status oracle became active successfully.");
await scheduler.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
.WithTimeout(this.initTimeout);
//if running in multi cluster scenario, start the MultiClusterNetwork Oracle
if (GlobalConfig.HasMultiClusterNetwork)
{
logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
GlobalConfig.ServiceId, GlobalConfig.ClusterId);
this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
this.providerManagerSystemTarget.SchedulingContext;
await scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
.WithTimeout(initTimeout);
logger.Debug("multicluster oracle created successfully.");
}
try
{
this.siloStatistics.Start(this.LocalConfig);
logger.Debug("Silo statistics manager started successfully.");
// Finally, initialize the deployment load collector, for grains with load-based placement
var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
await this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
.WithTimeout(this.initTimeout);
logger.Debug("Silo deployment load publisher started successfully.");
// Start background timer tick to watch for platform execution stalls, such as when GC kicks in
this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants, this.executorService, this.loggerFactory);
this.platformWatchdog.Start();
if (this.logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Silo platform watchdog started successfully.");
}
if (this.reminderService != null)
{
// so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ??
this.providerManagerSystemTarget.SchedulingContext;
await this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext)
.WithTimeout(this.initTimeout);
this.logger.Debug("Reminder service started successfully.");
}
this.bootstrapProviderManager = this.Services.GetRequiredService <BootstrapProviderManager>();
await this.scheduler.QueueTask(
() => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations),
this.providerManagerSystemTarget.SchedulingContext)
.WithTimeout(this.initTimeout);
this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics
logger.Debug("App bootstrap calls done successfully.");
// Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
// also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
await this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext)
.WithTimeout(this.initTimeout);
logger.Debug("Stream providers started successfully.");
// Now that we're active, we can start the gateway
var mc = this.messageCenter as MessageCenter;
mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>());
logger.Debug("Message gateway service started successfully.");
this.SystemStatus = SystemStatus.Running;
}
catch (Exception exc)
{
this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
throw;
}
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Silo.Start complete: System status = {0}", this.SystemStatus);
}
}
void IRingRangeListener.RangeChangeNotification(IRingRange oldRange, IRingRange newRange, bool increased)
{
scheduler.QueueTask(() => OnRangeChange(oldRange, newRange, increased), this).Ignore();
}
private void DoStart()
{
lock (lockable)
{
if (SystemStatus.Current != SystemStatus.Created)
{
throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Start state. This silo is in the {0} state.", SystemStatus.Current));
}
SystemStatus.Current = SystemStatus.Starting;
}
logger.Info(ErrorCode.SiloStarting, "Silo Start()");
// Hook up to receive notification of process exit / Ctrl-C events
AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
Console.CancelKeyPress += HandleProcessExit;
ConfigureThreadPoolAndServicePointSettings();
// This has to start first so that the directory system target factory gets loaded before we start the router.
typeManager.Start();
InsideRuntimeClient.Current.Start();
// The order of these 4 is pretty much arbitrary.
scheduler.Start();
messageCenter.Start();
incomingPingAgent.Start();
incomingSystemAgent.Start();
incomingAgent.Start();
LocalGrainDirectory.Start();
// Set up an execution context for this thread so that the target creation steps can use asynch values.
RuntimeContext.InitializeMainThread();
SiloProviderRuntime.Initialize(GlobalConfig);
statisticsProviderManager = new StatisticsProviderManager("Statistics", SiloProviderRuntime.Instance);
string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
.WaitForResultWithThrow(initTimeout);
if (statsProviderName != null)
{
LocalConfig.StatisticsProviderName = statsProviderName;
}
// can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
siloStatistics.SetSiloStatsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
siloStatistics.SetSiloMetricsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
membershipOracle = membershipFactory.CreateMembershipOracle(this).WaitForResultWithThrow(initTimeout);
// This has to follow the above steps that start the runtime components
CreateSystemTargets();
InjectDependencies();
// ensure this runs in the grain context, wait for it to complete
scheduler.QueueTask(CreateSystemGrains, catalog.SchedulingContext)
.WaitWithThrow(initTimeout);
// Initialize storage providers once we have a basic silo runtime environment operating
storageProviderManager = new StorageProviderManager();
scheduler.QueueTask(
() => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
catalog.SetStorageManager(storageProviderManager);
// Load and init stream providers before silo becomes active
var siloStreamProviderManager = new Orleans.Streams.StreamProviderManager();
scheduler.QueueTask(
() => siloStreamProviderManager.LoadStreamProviders(this.GlobalConfig.ProviderConfigurations, SiloProviderRuntime.Instance),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
InsideRuntimeClient.Current.CurrentStreamProviderManager = siloStreamProviderManager;
ISchedulingContext statusOracleContext = ((SystemTarget)LocalSiloStatusOracle).SchedulingContext;
bool waitForPrimaryToStart = globalConfig.PrimaryNodeIsRequired && siloType != SiloType.Primary;
scheduler.QueueTask(() => LocalSiloStatusOracle.Start(waitForPrimaryToStart), statusOracleContext)
.WaitWithThrow(initTimeout);
scheduler.QueueTask(LocalSiloStatusOracle.BecomeActive, statusOracleContext)
.WaitWithThrow(initTimeout);
try
{
siloStatistics.Start(LocalConfig);
// Finally, initialize the deployment load collector, for grains with load-based placement
scheduler.QueueTask(DeploymentLoadPublisher.Instance.Start, DeploymentLoadPublisher.Instance.SchedulingContext)
.WaitWithThrow(initTimeout);
// Start background timer tick to watch for platform execution stalls, such as when GC kicks in
platformWatchdog = new Watchdog(nodeConfig.StatisticsLogWriteInterval, healthCheckParticipants);
platformWatchdog.Start();
// so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
scheduler.QueueTask(reminderService.Start, ((SystemTarget)reminderService).SchedulingContext)
.WaitWithThrow(initTimeout);
// Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
var bootstrapProviderManager = new BootstrapProviderManager();
scheduler.QueueTask(
() => bootstrapProviderManager.LoadAppBootstrapProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
BootstrapProviders = bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics
// Now that we're active, we can start the gateway
var mc = messageCenter as MessageCenter;
if (mc != null)
{
mc.StartGateway();
}
SystemStatus.Current = SystemStatus.Running;
}
catch (Exception exc)
{
SafeExecute(() => logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", SiloAddress), exc));
FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
throw;
}
}
private async Task OnRuntimeGrainServicesStart(CancellationToken ct)
{
var stopWatch = Stopwatch.StartNew();
// Load and init grain services before silo becomes active.
await StartAsyncTaskWithPerfAnalysis("Init grain services",
() => CreateGrainServices(), stopWatch);
this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
this.fallbackScheduler.SchedulingContext;
await StartAsyncTaskWithPerfAnalysis("Starting local silo status oracle", StartMembershipOracle, stopWatch);
async Task StartMembershipOracle()
{
await scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
.WithTimeout(initTimeout, $"Starting MembershipOracle failed due to timeout {initTimeout}");
logger.Debug("Local silo status oracle created successfully.");
}
var versionStore = Services.GetService <IVersionStore>();
await StartAsyncTaskWithPerfAnalysis("Init type manager", () => scheduler
.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
.WithTimeout(this.initTimeout, $"TypeManager Initializing failed due to timeout {initTimeout}"), stopWatch);
//if running in multi cluster scenario, start the MultiClusterNetwork Oracle
if (this.multiClusterOracle != null)
{
await StartAsyncTaskWithPerfAnalysis("Start multicluster oracle", StartMultiClusterOracle, stopWatch);
async Task StartMultiClusterOracle()
{
logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
this.clusterOptions.ServiceId, this.clusterOptions.ClusterId);
this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
this.fallbackScheduler.SchedulingContext;
await scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
.WithTimeout(initTimeout, $"Starting MultiClusterOracle failed due to timeout {initTimeout}");
logger.Debug("multicluster oracle created successfully.");
}
}
try
{
StatisticsOptions statisticsOptions = Services.GetRequiredService <IOptions <StatisticsOptions> >().Value;
StartTaskWithPerfAnalysis("Start silo statistics", () => this.siloStatistics.Start(statisticsOptions), stopWatch);
logger.Debug("Silo statistics manager started successfully.");
// Finally, initialize the deployment load collector, for grains with load-based placement
await StartAsyncTaskWithPerfAnalysis("Start deployment load collector", StartDeploymentLoadCollector, stopWatch);
async Task StartDeploymentLoadCollector()
{
var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
await this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
.WithTimeout(this.initTimeout, $"Starting DeploymentLoadPublisher failed due to timeout {initTimeout}");
logger.Debug("Silo deployment load publisher started successfully.");
}
// Start background timer tick to watch for platform execution stalls, such as when GC kicks in
this.platformWatchdog = new Watchdog(statisticsOptions.LogWriteInterval, this.healthCheckParticipants, this.executorService, this.loggerFactory);
this.platformWatchdog.Start();
if (this.logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Silo platform watchdog started successfully.");
}
}
catch (Exception exc)
{
this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
throw;
}
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Silo.Start complete: System status = {0}", this.SystemStatus);
}
}
private async Task OnRuntimeGrainServicesStart(CancellationToken ct)
{
var stopWatch = Stopwatch.StartNew();
await StartAsyncTaskWithPerfAnalysis("Init transaction agent", InitTransactionAgent, stopWatch);
async Task InitTransactionAgent()
{
ITransactionAgent transactionAgent = this.Services.GetRequiredService <ITransactionAgent>();
ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext;
await scheduler.QueueTask(transactionAgent.Start, transactionAgentContext)
.WithTimeout(initTimeout);
}
// Load and init grain services before silo becomes active.
await StartAsyncTaskWithPerfAnalysis("Init grain services",
() => CreateGrainServices(GlobalConfig.GrainServiceConfigurations), stopWatch);
this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
this.fallbackScheduler.SchedulingContext;
await StartAsyncTaskWithPerfAnalysis("Start local silo status oracle", StartMembershipOracle, stopWatch);
async Task StartMembershipOracle()
{
await scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
.WithTimeout(initTimeout);
logger.Debug("Local silo status oracle created successfully.");
await scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext)
.WithTimeout(initTimeout);
logger.Debug("Local silo status oracle became active successfully.");
}
var versionStore = Services.GetService <IVersionStore>();
await StartAsyncTaskWithPerfAnalysis("Init type manager", () => scheduler
.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
.WithTimeout(this.initTimeout), stopWatch);
//if running in multi cluster scenario, start the MultiClusterNetwork Oracle
if (this.multiClusterOracle != null)
{
await StartAsyncTaskWithPerfAnalysis("Start multicluster oracle", StartMultiClusterOracle, stopWatch);
async Task StartMultiClusterOracle()
{
logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
this.siloOptions.ServiceId, this.siloOptions.ClusterId);
this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
this.fallbackScheduler.SchedulingContext;
await scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
.WithTimeout(initTimeout);
logger.Debug("multicluster oracle created successfully.");
}
}
try
{
StatisticsOptions statisticsOptions = Services.GetRequiredService <IOptions <StatisticsOptions> >().Value;
StartTaskWithPerfAnalysis("Start silo statistics", () => this.siloStatistics.Start(statisticsOptions), stopWatch);
logger.Debug("Silo statistics manager started successfully.");
// Finally, initialize the deployment load collector, for grains with load-based placement
await StartAsyncTaskWithPerfAnalysis("Start deployment load collector", StartDeploymentLoadCollector, stopWatch);
async Task StartDeploymentLoadCollector()
{
var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
await this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
.WithTimeout(this.initTimeout);
logger.Debug("Silo deployment load publisher started successfully.");
}
// Start background timer tick to watch for platform execution stalls, such as when GC kicks in
this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants, this.executorService, this.loggerFactory);
this.platformWatchdog.Start();
if (this.logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Silo platform watchdog started successfully.");
}
if (this.reminderService != null)
{
await StartAsyncTaskWithPerfAnalysis("Start reminder service", StartReminderService, stopWatch);
async Task StartReminderService()
{
// so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ??
this.fallbackScheduler.SchedulingContext;
await this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext)
.WithTimeout(this.initTimeout);
this.logger.Debug("Reminder service started successfully.");
}
}
StartTaskWithPerfAnalysis("Start gateway", StartGateway, stopWatch);
void StartGateway()
{
// Now that we're active, we can start the gateway
var mc = this.messageCenter as MessageCenter;
mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>());
logger.Debug("Message gateway service started successfully.");
}
this.SystemStatus = SystemStatus.Running;
}
catch (Exception exc)
{
this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
throw;
}
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Silo.Start complete: System status = {0}", this.SystemStatus);
}
}
public Task ReportStats(List <ICounter> statsCounters)
{
logger.Info("{0} ReportStats called", GetType().Name);
taskScheduler.QueueTask(() => grain.ReportStatsCalled(), schedulingContext).Ignore();
return(Task.CompletedTask);
}
public static IServiceCollection AddLegacyClusterConfigurationSupport(this IServiceCollection services, ClusterConfiguration configuration)
{
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
if (services.TryGetClusterConfiguration() != null)
{
throw new InvalidOperationException("Cannot configure legacy ClusterConfiguration support twice");
}
// these will eventually be removed once our code doesn't depend on the old ClientConfiguration
services.AddSingleton(configuration);
services.TryAddSingleton <LegacyConfigurationWrapper>();
services.TryAddSingleton(sp => sp.GetRequiredService <LegacyConfigurationWrapper>().ClusterConfig.Globals);
services.TryAddTransient(sp => sp.GetRequiredService <LegacyConfigurationWrapper>().NodeConfig);
services.TryAddSingleton <Factory <NodeConfiguration> >(
sp =>
{
var initializationParams = sp.GetRequiredService <LegacyConfigurationWrapper>();
return(() => initializationParams.NodeConfig);
});
services.Configure <SiloOptions>(options =>
{
if (string.IsNullOrWhiteSpace(options.ClusterId) && !string.IsNullOrWhiteSpace(configuration.Globals.ClusterId))
{
options.ClusterId = configuration.Globals.ClusterId;
}
if (options.ServiceId == Guid.Empty)
{
options.ServiceId = configuration.Globals.ServiceId;
}
options.FastKillOnCancelKeyPress = configuration.Globals.FastKillOnCancelKeyPress;
});
services.Configure <MultiClusterOptions>(options =>
{
var globals = configuration.Globals;
if (globals.HasMultiClusterNetwork)
{
options.HasMultiClusterNetwork = true;
options.BackgroundGossipInterval = globals.BackgroundGossipInterval;
options.DefaultMultiCluster = globals.DefaultMultiCluster?.ToList();
options.GlobalSingleInstanceNumberRetries = globals.GlobalSingleInstanceNumberRetries;
options.GlobalSingleInstanceRetryInterval = globals.GlobalSingleInstanceRetryInterval;
options.MaxMultiClusterGateways = globals.MaxMultiClusterGateways;
options.UseGlobalSingleInstanceByDefault = globals.UseGlobalSingleInstanceByDefault;
foreach (GlobalConfiguration.GossipChannelConfiguration channelConfig in globals.GossipChannels)
{
options.GossipChannels.Add(GlobalConfiguration.Remap(channelConfig.ChannelType), channelConfig.ConnectionString);
}
}
});
services.TryAddFromExisting <IMessagingConfiguration, GlobalConfiguration>();
services.AddOptions <SiloStatisticsOptions>()
.Configure <NodeConfiguration>((options, nodeConfig) => LegacyConfigurationExtensions.CopyStatisticsOptions(nodeConfig, options))
.Configure <GlobalConfiguration>((options, config) =>
{
options.DeploymentLoadPublisherRefreshTime = config.DeploymentLoadPublisherRefreshTime;
});
services.AddOptions <LoadSheddingOptions>()
.Configure <NodeConfiguration>((options, nodeConfig) =>
{
options.LoadSheddingEnabled = nodeConfig.LoadSheddingEnabled;
options.LoadSheddingLimit = nodeConfig.LoadSheddingLimit;
});
// Translate legacy configuration to new Options
services.AddOptions <SiloMessagingOptions>()
.Configure <GlobalConfiguration>((options, config) =>
{
LegacyConfigurationExtensions.CopyCommonMessagingOptions(config, options);
options.SiloSenderQueues = config.SiloSenderQueues;
options.GatewaySenderQueues = config.GatewaySenderQueues;
options.MaxForwardCount = config.MaxForwardCount;
options.ClientDropTimeout = config.ClientDropTimeout;
options.ClientRegistrationRefresh = config.ClientRegistrationRefresh;
options.MaxRequestProcessingTime = config.MaxRequestProcessingTime;
options.AssumeHomogenousSilosForTesting = config.AssumeHomogenousSilosForTesting;
})
.Configure <NodeConfiguration>((options, config) =>
{
options.PropagateActivityId = config.PropagateActivityId;
LimitValue requestLimit = config.LimitManager.GetLimit(LimitNames.LIMIT_MAX_ENQUEUED_REQUESTS);
options.MaxEnqueuedRequestsSoftLimit = requestLimit.SoftLimitThreshold;
options.MaxEnqueuedRequestsHardLimit = requestLimit.HardLimitThreshold;
LimitValue statelessWorkerRequestLimit = config.LimitManager.GetLimit(LimitNames.LIMIT_MAX_ENQUEUED_REQUESTS_STATELESS_WORKER);
options.MaxEnqueuedRequestsSoftLimit_StatelessWorker = statelessWorkerRequestLimit.SoftLimitThreshold;
options.MaxEnqueuedRequestsHardLimit_StatelessWorker = statelessWorkerRequestLimit.HardLimitThreshold;
});
services.Configure <NetworkingOptions>(options => LegacyConfigurationExtensions.CopyNetworkingOptions(configuration.Globals, options));
services.AddOptions <EndpointOptions>()
.Configure <IOptions <SiloOptions> >((options, siloOptions) =>
{
var nodeConfig = configuration.GetOrCreateNodeConfigurationForSilo(siloOptions.Value.SiloName);
if (options.IPAddress == null && string.IsNullOrWhiteSpace(options.HostNameOrIPAddress))
{
options.IPAddress = nodeConfig.Endpoint.Address;
options.Port = nodeConfig.Endpoint.Port;
}
if (options.ProxyPort == 0 && nodeConfig.ProxyGatewayEndpoint != null)
{
options.ProxyPort = nodeConfig.ProxyGatewayEndpoint.Port;
}
});
services.Configure <SerializationProviderOptions>(options =>
{
options.SerializationProviders = configuration.Globals.SerializationProviders;
options.FallbackSerializationProvider = configuration.Globals.FallbackSerializationProvider;
});
services.Configure <TelemetryOptions>(options =>
{
LegacyConfigurationExtensions.CopyTelemetryOptions(configuration.Defaults.TelemetryConfiguration, services, options);
});
services.AddOptions <GrainClassOptions>().Configure <IOptions <SiloOptions> >((options, siloOptions) =>
{
var nodeConfig = configuration.GetOrCreateNodeConfigurationForSilo(siloOptions.Value.SiloName);
options.ExcludedGrainTypes.AddRange(nodeConfig.ExcludedGrainTypes);
});
LegacyMembershipConfigurator.ConfigureServices(configuration.Globals, services);
services.AddOptions <SchedulingOptions>()
.Configure <GlobalConfiguration>((options, config) =>
{
options.AllowCallChainReentrancy = config.AllowCallChainReentrancy;
options.PerformDeadlockDetection = config.PerformDeadlockDetection;
})
.Configure <NodeConfiguration>((options, nodeConfig) =>
{
options.MaxActiveThreads = nodeConfig.MaxActiveThreads;
options.DelayWarningThreshold = nodeConfig.DelayWarningThreshold;
options.ActivationSchedulingQuantum = nodeConfig.ActivationSchedulingQuantum;
options.TurnWarningLengthThreshold = nodeConfig.TurnWarningLengthThreshold;
options.EnableWorkerThreadInjection = nodeConfig.EnableWorkerThreadInjection;
LimitValue itemLimit = nodeConfig.LimitManager.GetLimit(LimitNames.LIMIT_MAX_PENDING_ITEMS);
options.MaxPendingWorkItemsSoftLimit = itemLimit.SoftLimitThreshold;
options.MaxPendingWorkItemsHardLimit = itemLimit.HardLimitThreshold;
});
services.AddOptions <GrainCollectionOptions>().Configure <GlobalConfiguration>((options, config) =>
{
options.CollectionQuantum = config.CollectionQuantum;
options.CollectionAge = config.Application.DefaultCollectionAgeLimit;
foreach (GrainTypeConfiguration grainConfig in config.Application.ClassSpecific)
{
if (grainConfig.CollectionAgeLimit.HasValue)
{
options.ClassSpecificCollectionAge.Add(grainConfig.FullTypeName, grainConfig.CollectionAgeLimit.Value);
}
}
;
});
services.TryAddSingleton <LegacyProviderConfigurator.ScheduleTask>(sp =>
{
OrleansTaskScheduler scheduler = sp.GetRequiredService <OrleansTaskScheduler>();
SystemTarget fallbackSystemTarget = sp.GetRequiredService <FallbackSystemTarget>();
return((taskFunc) => scheduler.QueueTask(taskFunc, fallbackSystemTarget.SchedulingContext));
});
LegacyProviderConfigurator <ISiloLifecycle> .ConfigureServices(configuration.Globals.ProviderConfigurations, services, SiloDefaultProviderInitStage, SiloDefaultProviderStartStage);
services.AddOptions <GrainPlacementOptions>().Configure <GlobalConfiguration>((options, config) =>
{
options.DefaultPlacementStrategy = config.DefaultPlacementStrategy;
options.ActivationCountPlacementChooseOutOf = config.ActivationCountBasedPlacementChooseOutOf;
});
services.AddOptions <StaticClusterDeploymentOptions>().Configure <ClusterConfiguration>((options, config) =>
{
options.SiloNames = config.Overrides.Keys.ToList();
});
// add grain service configs as keyed services
short id = 0;
foreach (IGrainServiceConfiguration grainServiceConfiguration in configuration.Globals.GrainServiceConfigurations.GrainServices.Values)
{
services.AddSingletonKeyedService <long, IGrainServiceConfiguration>(id++, (sp, k) => grainServiceConfiguration);
}
// populate grain service options
id = 0;
services.AddOptions <GrainServiceOptions>().Configure <GlobalConfiguration>((options, config) =>
{
foreach (IGrainServiceConfiguration grainServiceConfiguration in config.GrainServiceConfigurations.GrainServices.Values)
{
options.GrainServices.Add(new KeyValuePair <string, short>(grainServiceConfiguration.ServiceType, id++));
}
});
services.AddOptions <ConsistentRingOptions>().Configure <GlobalConfiguration>((options, config) =>
{
options.UseVirtualBucketsConsistentRing = config.UseVirtualBucketsConsistentRing;
options.NumVirtualBucketsConsistentRing = config.NumVirtualBucketsConsistentRing;
});
services.AddOptions <MembershipOptions>()
.Configure <GlobalConfiguration>((options, config) =>
{
options.NumMissedTableIAmAliveLimit = config.NumMissedTableIAmAliveLimit;
options.LivenessEnabled = config.LivenessEnabled;
options.ProbeTimeout = config.ProbeTimeout;
options.TableRefreshTimeout = config.TableRefreshTimeout;
options.DeathVoteExpirationTimeout = config.DeathVoteExpirationTimeout;
options.IAmAliveTablePublishTimeout = config.IAmAliveTablePublishTimeout;
options.MaxJoinAttemptTime = config.MaxJoinAttemptTime;
options.ExpectedClusterSize = config.ExpectedClusterSize;
options.ValidateInitialConnectivity = config.ValidateInitialConnectivity;
options.NumMissedProbesLimit = config.NumMissedProbesLimit;
options.UseLivenessGossip = config.UseLivenessGossip;
options.NumProbedSilos = config.NumProbedSilos;
options.NumVotesForDeathDeclaration = config.NumVotesForDeathDeclaration;
})
.Configure <ClusterConfiguration>((options, config) =>
{
options.IsRunningAsUnitTest = config.IsRunningAsUnitTest;
});
services.AddOptions <ReminderOptions>()
.Configure <GlobalConfiguration>((options, config) =>
{
options.ReminderService = GlobalConfiguration.Remap(config.ReminderServiceType);
options.ReminderTableAssembly = config.ReminderTableAssembly;
options.UseMockReminderTable = config.UseMockReminderTable;
options.MockReminderTableTimeout = config.MockReminderTableTimeout;
});
services.AddOptions <GrainVersioningOptions>()
.Configure <GlobalConfiguration>((options, config) =>
{
options.DefaultCompatibilityStrategy = config.DefaultCompatibilityStrategy?.GetType().Name ?? GrainVersioningOptions.DEFAULT_COMPATABILITY_STRATEGY;
options.DefaultVersionSelectorStrategy = config.DefaultVersionSelectorStrategy?.GetType().Name ?? GrainVersioningOptions.DEFAULT_VERSION_SELECTOR_STRATEGY;
});
services.AddOptions <ThreadPoolOptions>()
.Configure <NodeConfiguration>((options, config) =>
{
options.MinDotNetThreadPoolSize = config.MinDotNetThreadPoolSize;
});
services.AddOptions <ServicePointOptions>()
.Configure <NodeConfiguration>((options, config) =>
{
options.DefaultConnectionLimit = config.DefaultConnectionLimit;
options.Expect100Continue = config.Expect100Continue;
options.UseNagleAlgorithm = config.UseNagleAlgorithm;
});
services.AddOptions <StorageOptions>()
.Configure <GlobalConfiguration>((options, config) =>
{
options.DataConnectionString = config.DataConnectionString;
options.DataConnectionStringForReminders = config.DataConnectionStringForReminders;
});
services.AddOptions <AdoNetOptions>()
.Configure <GlobalConfiguration>((options, config) =>
{
options.Invariant = config.AdoInvariant;
options.InvariantForReminders = config.AdoInvariantForReminders;
});
services.AddOptions <TypeManagementOptions>()
.Configure <GlobalConfiguration>((options, config) =>
{
options.TypeMapRefreshInterval = config.TypeMapRefreshInterval;
});
services.AddOptions <GrainDirectoryOptions>()
.Configure <GlobalConfiguration>((options, config) =>
{
options.CachingStrategy = GlobalConfiguration.Remap(config.DirectoryCachingStrategy);
options.CacheSize = config.CacheSize;
options.InitialCacheTTL = config.InitialCacheTTL;
options.MaximumCacheTTL = config.MaximumCacheTTL;
options.CacheTTLExtensionFactor = config.CacheTTLExtensionFactor;
options.LazyDeregistrationDelay = config.DirectoryLazyDeregistrationDelay;
});
return(services);
}
public static IServiceCollection AddLegacyClusterConfigurationSupport(this IServiceCollection services, ClusterConfiguration configuration)
{
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
if (services.Any(service => service.ServiceType == typeof(ClusterConfiguration)))
{
throw new InvalidOperationException("Cannot configure legacy ClusterConfiguration support twice");
}
// these will eventually be removed once our code doesn't depend on the old ClientConfiguration
services.AddSingleton(configuration);
services.TryAddSingleton <LegacyConfigurationWrapper>();
services.TryAddSingleton(sp => sp.GetRequiredService <LegacyConfigurationWrapper>().ClusterConfig.Globals);
services.TryAddTransient(sp => sp.GetRequiredService <LegacyConfigurationWrapper>().NodeConfig);
services.TryAddSingleton <Factory <NodeConfiguration> >(
sp =>
{
var initializationParams = sp.GetRequiredService <LegacyConfigurationWrapper>();
return(() => initializationParams.NodeConfig);
});
services.Configure <SiloOptions>(options =>
{
if (string.IsNullOrWhiteSpace(options.ClusterId) && !string.IsNullOrWhiteSpace(configuration.Globals.ClusterId))
{
options.ClusterId = configuration.Globals.ClusterId;
}
if (options.ServiceId == Guid.Empty)
{
options.ServiceId = configuration.Globals.ServiceId;
}
});
services.Configure <MultiClusterOptions>(options =>
{
var globals = configuration.Globals;
if (globals.HasMultiClusterNetwork)
{
options.HasMultiClusterNetwork = true;
options.BackgroundGossipInterval = globals.BackgroundGossipInterval;
options.DefaultMultiCluster = globals.DefaultMultiCluster?.ToList();
options.GlobalSingleInstanceNumberRetries = globals.GlobalSingleInstanceNumberRetries;
options.GlobalSingleInstanceRetryInterval = globals.GlobalSingleInstanceRetryInterval;
options.MaxMultiClusterGateways = globals.MaxMultiClusterGateways;
options.UseGlobalSingleInstanceByDefault = globals.UseGlobalSingleInstanceByDefault;
}
});
services.TryAddFromExisting <IMessagingConfiguration, GlobalConfiguration>();
services.AddOptions <StatisticsOptions>()
.Configure <NodeConfiguration>((options, nodeConfig) => LegacyConfigurationExtensions.CopyStatisticsOptions(nodeConfig, options));
// Translate legacy configuration to new Options
services.Configure <SiloMessagingOptions>(options =>
{
LegacyConfigurationExtensions.CopyCommonMessagingOptions(configuration.Globals, options);
options.SiloSenderQueues = configuration.Globals.SiloSenderQueues;
options.GatewaySenderQueues = configuration.Globals.GatewaySenderQueues;
options.MaxForwardCount = configuration.Globals.MaxForwardCount;
options.ClientDropTimeout = configuration.Globals.ClientDropTimeout;
});
services.Configure <NetworkingOptions>(options => LegacyConfigurationExtensions.CopyNetworkingOptions(configuration.Globals, options));
services.AddOptions <EndpointOptions>()
.Configure <IOptions <SiloOptions> >((options, siloOptions) =>
{
var nodeConfig = configuration.GetOrCreateNodeConfigurationForSilo(siloOptions.Value.SiloName);
if (options.IPAddress == null && string.IsNullOrWhiteSpace(options.HostNameOrIPAddress))
{
options.IPAddress = nodeConfig.Endpoint.Address;
options.Port = nodeConfig.Endpoint.Port;
}
if (options.ProxyPort == 0 && nodeConfig.ProxyGatewayEndpoint != null)
{
options.ProxyPort = nodeConfig.ProxyGatewayEndpoint.Port;
}
});
services.Configure <SerializationProviderOptions>(options =>
{
options.SerializationProviders = configuration.Globals.SerializationProviders;
options.FallbackSerializationProvider = configuration.Globals.FallbackSerializationProvider;
});
services.AddOptions <GrainClassOptions>().Configure <IOptions <SiloOptions> >((options, siloOptions) =>
{
var nodeConfig = configuration.GetOrCreateNodeConfigurationForSilo(siloOptions.Value.SiloName);
options.ExcludedGrainTypes.AddRange(nodeConfig.ExcludedGrainTypes);
});
LegacyMembershipConfigurator.ConfigureServices(configuration.Globals, services);
services.AddOptions <SchedulingOptions>().Configure <GlobalConfiguration>((options, config) =>
{
options.AllowCallChainReentrancy = config.AllowCallChainReentrancy;
options.PerformDeadlockDetection = config.PerformDeadlockDetection;
});
services.TryAddSingleton <LegacyProviderConfigurator.ScheduleTask>(sp =>
{
OrleansTaskScheduler scheduler = sp.GetRequiredService <OrleansTaskScheduler>();
SystemTarget fallbackSystemTarget = sp.GetRequiredService <FallbackSystemTarget>();
return((taskFunc) => scheduler.QueueTask(taskFunc, fallbackSystemTarget.SchedulingContext));
});
LegacyProviderConfigurator <ISiloLifecycle> .ConfigureServices(configuration.Globals.ProviderConfigurations, services, SiloDefaultProviderInitStage, SiloDefaultProviderStartStage);
return(services);
}
private void DoStart()
{
lock (lockable)
{
if (!this.SystemStatus.Equals(SystemStatus.Created))
{
throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", this.SystemStatus));
}
this.SystemStatus = SystemStatus.Starting;
}
logger.Info(ErrorCode.SiloStarting, "Silo Start()");
// Hook up to receive notification of process exit / Ctrl-C events
AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
if (GlobalConfig.FastKillOnCancelKeyPress)
{
Console.CancelKeyPress += HandleProcessExit;
}
ConfigureThreadPoolAndServicePointSettings();
// This has to start first so that the directory system target factory gets loaded before we start the router.
grainTypeManager.Start();
runtimeClient.Start();
// The order of these 4 is pretty much arbitrary.
scheduler.Start();
messageCenter.Start();
incomingPingAgent.Start();
incomingSystemAgent.Start();
incomingAgent.Start();
LocalGrainDirectory.Start();
// Set up an execution context for this thread so that the target creation steps can use asynch values.
RuntimeContext.InitializeMainThread();
// Initialize the implicit stream subscribers table.
var implicitStreamSubscriberTable = Services.GetRequiredService <ImplicitStreamSubscriberTable>();
implicitStreamSubscriberTable.InitImplicitStreamSubscribers(this.grainTypeManager.GrainClassTypeData.Select(t => t.Value.Type).ToArray());
var siloProviderRuntime = Services.GetRequiredService <SiloProviderRuntime>();
runtimeClient.CurrentStreamProviderRuntime = siloProviderRuntime;
statisticsProviderManager = this.Services.GetRequiredService <StatisticsProviderManager>();
string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
.WaitForResultWithThrow(initTimeout);
if (statsProviderName != null)
{
LocalConfig.StatisticsProviderName = statsProviderName;
}
// can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
siloStatistics.SetSiloStatsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout);
siloStatistics.SetSiloMetricsTableDataManager(this, LocalConfig).WaitWithThrow(initTimeout);
// This has to follow the above steps that start the runtime components
CreateSystemTargets();
InjectDependencies();
// Validate the configuration.
GlobalConfig.Application.ValidateConfiguration(logger);
// Initialize storage providers once we have a basic silo runtime environment operating
storageProviderManager = this.Services.GetRequiredService <StorageProviderManager>();
scheduler.QueueTask(
() => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
ITransactionAgent transactionAgent = this.Services.GetRequiredService <ITransactionAgent>();
ISchedulingContext transactionAgentContext = (transactionAgent as SystemTarget)?.SchedulingContext;
scheduler.QueueTask(transactionAgent.Start, transactionAgentContext)
.WaitWithThrow(initTimeout);
var versionStore = Services.GetService <IVersionStore>() as GrainVersionStore;
versionStore?.SetStorageManager(storageProviderManager);
if (logger.IsVerbose)
{
logger.Verbose("Storage provider manager created successfully.");
}
// Initialize log consistency providers once we have a basic silo runtime environment operating
logConsistencyProviderManager = this.Services.GetRequiredService <LogConsistencyProviderManager>();
scheduler.QueueTask(
() => logConsistencyProviderManager.LoadLogConsistencyProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Log consistency provider manager created successfully.");
}
// Load and init stream providers before silo becomes active
var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager;
scheduler.QueueTask(
() => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, siloProviderRuntime),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
runtimeClient.CurrentStreamProviderManager = siloStreamProviderManager;
if (logger.IsVerbose)
{
logger.Verbose("Stream provider manager created successfully.");
}
// Load and init grain services before silo becomes active.
CreateGrainServices(GlobalConfig.GrainServiceConfigurations);
this.membershipOracleContext = (this.membershipOracle as SystemTarget)?.SchedulingContext ??
this.providerManagerSystemTarget.SchedulingContext;
scheduler.QueueTask(() => this.membershipOracle.Start(), this.membershipOracleContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Local silo status oracle created successfully.");
}
scheduler.QueueTask(this.membershipOracle.BecomeActive, this.membershipOracleContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Local silo status oracle became active successfully.");
}
scheduler.QueueTask(() => this.typeManager.Initialize(versionStore), this.typeManager.SchedulingContext)
.WaitWithThrow(this.initTimeout);
//if running in multi cluster scenario, start the MultiClusterNetwork Oracle
if (GlobalConfig.HasMultiClusterNetwork)
{
logger.Info("Starting multicluster oracle with my ServiceId={0} and ClusterId={1}.",
GlobalConfig.ServiceId, GlobalConfig.ClusterId);
this.multiClusterOracleContext = (multiClusterOracle as SystemTarget)?.SchedulingContext ??
this.providerManagerSystemTarget.SchedulingContext;
scheduler.QueueTask(() => multiClusterOracle.Start(), multiClusterOracleContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("multicluster oracle created successfully.");
}
}
try
{
this.siloStatistics.Start(this.LocalConfig);
if (this.logger.IsVerbose)
{
this.logger.Verbose("Silo statistics manager started successfully.");
}
// Finally, initialize the deployment load collector, for grains with load-based placement
var deploymentLoadPublisher = Services.GetRequiredService <DeploymentLoadPublisher>();
this.scheduler.QueueTask(deploymentLoadPublisher.Start, deploymentLoadPublisher.SchedulingContext)
.WaitWithThrow(this.initTimeout);
if (this.logger.IsVerbose)
{
this.logger.Verbose("Silo deployment load publisher started successfully.");
}
// Start background timer tick to watch for platform execution stalls, such as when GC kicks in
this.platformWatchdog = new Watchdog(this.LocalConfig.StatisticsLogWriteInterval, this.healthCheckParticipants);
this.platformWatchdog.Start();
if (this.logger.IsVerbose)
{
this.logger.Verbose("Silo platform watchdog started successfully.");
}
if (this.reminderService != null)
{
// so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
this.reminderServiceContext = (this.reminderService as SystemTarget)?.SchedulingContext ??
this.providerManagerSystemTarget.SchedulingContext;
this.scheduler.QueueTask(this.reminderService.Start, this.reminderServiceContext)
.WaitWithThrow(this.initTimeout);
if (this.logger.IsVerbose)
{
this.logger.Verbose("Reminder service started successfully.");
}
}
this.bootstrapProviderManager = this.Services.GetRequiredService <BootstrapProviderManager>();
this.scheduler.QueueTask(
() => this.bootstrapProviderManager.LoadAppBootstrapProviders(siloProviderRuntime, this.GlobalConfig.ProviderConfigurations),
this.providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(this.initTimeout);
this.BootstrapProviders = this.bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics
if (this.logger.IsVerbose)
{
this.logger.Verbose("App bootstrap calls done successfully.");
}
// Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
// also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
this.scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, this.providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(this.initTimeout);
if (this.logger.IsVerbose)
{
this.logger.Verbose("Stream providers started successfully.");
}
// Now that we're active, we can start the gateway
var mc = this.messageCenter as MessageCenter;
mc?.StartGateway(this.Services.GetRequiredService <ClientObserverRegistrar>());
if (this.logger.IsVerbose)
{
this.logger.Verbose("Message gateway service started successfully.");
}
this.SystemStatus = SystemStatus.Running;
}
catch (Exception exc)
{
this.SafeExecute(() => this.logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", this.SiloAddress), exc));
this.FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
throw;
}
if (logger.IsVerbose)
{
logger.Verbose("Silo.Start complete: System status = {0}", this.SystemStatus);
}
}
private void DoStart()
{
lock (lockable)
{
if (!SystemStatus.Current.Equals(SystemStatus.Created))
{
throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", SystemStatus.Current));
}
SystemStatus.Current = SystemStatus.Starting;
}
logger.Info(ErrorCode.SiloStarting, "Silo Start()");
// Hook up to receive notification of process exit / Ctrl-C events
AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
Console.CancelKeyPress += HandleProcessExit;
ConfigureThreadPoolAndServicePointSettings();
// This has to start first so that the directory system target factory gets loaded before we start the router.
typeManager.Start();
InsideRuntimeClient.Current.Start();
// The order of these 4 is pretty much arbitrary.
scheduler.Start();
messageCenter.Start();
incomingPingAgent.Start();
incomingSystemAgent.Start();
incomingAgent.Start();
LocalGrainDirectory.Start();
// Set up an execution context for this thread so that the target creation steps can use asynch values.
RuntimeContext.InitializeMainThread();
SiloProviderRuntime.Initialize(GlobalConfig, SiloIdentity, grainFactory, Services);
InsideRuntimeClient.Current.CurrentStreamProviderRuntime = SiloProviderRuntime.Instance;
statisticsProviderManager = new StatisticsProviderManager("Statistics", SiloProviderRuntime.Instance);
string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
.WaitForResultWithThrow(initTimeout);
if (statsProviderName != null)
{
LocalConfig.StatisticsProviderName = statsProviderName;
}
allSiloProviders.AddRange(statisticsProviderManager.GetProviders());
// can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
siloStatistics.SetSiloStatsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
siloStatistics.SetSiloMetricsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
IMembershipTable membershipTable = membershipFactory.GetMembershipTable(GlobalConfig.LivenessType, GlobalConfig.MembershipTableAssembly);
membershipOracle = membershipFactory.CreateMembershipOracle(this, membershipTable);
multiClusterOracle = multiClusterFactory.CreateGossipOracle(this).WaitForResultWithThrow(initTimeout);
// This has to follow the above steps that start the runtime components
CreateSystemTargets();
InjectDependencies();
// Validate the configuration.
GlobalConfig.Application.ValidateConfiguration(logger);
// ensure this runs in the grain context, wait for it to complete
scheduler.QueueTask(CreateSystemGrains, catalog.SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("System grains created successfully.");
}
// Initialize storage providers once we have a basic silo runtime environment operating
storageProviderManager = new StorageProviderManager(grainFactory, Services);
scheduler.QueueTask(
() => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
catalog.SetStorageManager(storageProviderManager);
allSiloProviders.AddRange(storageProviderManager.GetProviders());
if (logger.IsVerbose)
{
logger.Verbose("Storage provider manager created successfully.");
}
// Load and init stream providers before silo becomes active
var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager;
scheduler.QueueTask(
() => siloStreamProviderManager.LoadStreamProviders(GlobalConfig.ProviderConfigurations, SiloProviderRuntime.Instance),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
InsideRuntimeClient.Current.CurrentStreamProviderManager = siloStreamProviderManager;
allSiloProviders.AddRange(siloStreamProviderManager.GetProviders());
if (logger.IsVerbose)
{
logger.Verbose("Stream provider manager created successfully.");
}
ISchedulingContext statusOracleContext = ((SystemTarget)LocalSiloStatusOracle).SchedulingContext;
bool waitForPrimaryToStart = globalConfig.PrimaryNodeIsRequired && siloType != SiloType.Primary;
if (waitForPrimaryToStart) // only in MembershipTableGrain case.
{
scheduler.QueueTask(() => membershipFactory.WaitForTableToInit(membershipTable), statusOracleContext)
.WaitWithThrow(initTimeout);
}
scheduler.QueueTask(() => membershipTable.InitializeMembershipTable(GlobalConfig, true, LogManager.GetLogger(membershipTable.GetType().Name)), statusOracleContext)
.WaitWithThrow(initTimeout);
scheduler.QueueTask(() => LocalSiloStatusOracle.Start(), statusOracleContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Local silo status oracle created successfully.");
}
scheduler.QueueTask(LocalSiloStatusOracle.BecomeActive, statusOracleContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Local silo status oracle became active successfully.");
}
//if running in multi cluster scenario, start the MultiClusterNetwork Oracle
if (GlobalConfig.HasMultiClusterNetwork)
{
logger.Info("Creating multicluster oracle with my ServiceId={0} and ClusterId={1}.",
GlobalConfig.ServiceId, GlobalConfig.ClusterId);
ISchedulingContext clusterStatusContext = ((SystemTarget)multiClusterOracle).SchedulingContext;
scheduler.QueueTask(() => multiClusterOracle.Start(LocalSiloStatusOracle), clusterStatusContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("multicluster oracle created successfully.");
}
}
try
{
siloStatistics.Start(LocalConfig);
if (logger.IsVerbose)
{
logger.Verbose("Silo statistics manager started successfully.");
}
// Finally, initialize the deployment load collector, for grains with load-based placement
scheduler.QueueTask(DeploymentLoadPublisher.Instance.Start, DeploymentLoadPublisher.Instance.SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Silo deployment load publisher started successfully.");
}
// Start background timer tick to watch for platform execution stalls, such as when GC kicks in
platformWatchdog = new Watchdog(nodeConfig.StatisticsLogWriteInterval, healthCheckParticipants);
platformWatchdog.Start();
if (logger.IsVerbose)
{
logger.Verbose("Silo platform watchdog started successfully.");
}
if (reminderService != null)
{
// so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
scheduler.QueueTask(reminderService.Start, ((SystemTarget)reminderService).SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Reminder service started successfully.");
}
}
bootstrapProviderManager = new BootstrapProviderManager();
scheduler.QueueTask(
() => bootstrapProviderManager.LoadAppBootstrapProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
BootstrapProviders = bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics
allSiloProviders.AddRange(BootstrapProviders);
if (logger.IsVerbose)
{
logger.Verbose("App bootstrap calls done successfully.");
}
// Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
// also after bootstrap provider started so bootstrap provider can initialize everything stream before events from this silo arrive.
scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Stream providers started successfully.");
}
// Now that we're active, we can start the gateway
var mc = messageCenter as MessageCenter;
if (mc != null)
{
mc.StartGateway(clientRegistrar);
}
if (logger.IsVerbose)
{
logger.Verbose("Message gateway service started successfully.");
}
SystemStatus.Current = SystemStatus.Running;
}
catch (Exception exc)
{
SafeExecute(() => logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", SiloAddress), exc));
FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
throw;
}
if (logger.IsVerbose)
{
logger.Verbose("Silo.Start complete: System status = {0}", SystemStatus.Current);
}
}
