Esempio n. 1
0
        public static void LogWarning(Orleans.Runtime.Logger orleansLog, string type, string method, string format, params object[] arguments)
        {
#if DEBUG
            var message  = string.Format(format, arguments);
            var threadId = Thread.CurrentThread.ManagedThreadId;

            message = string.Format("[{0}] {1}::{2}: {3}", threadId, type, method, message);
            //Trace.TraceInformation(message);
            // Debug.WriteLine(message);
            Console.WriteLine(message);
            orleansLog.Warn(0, message);
#endif
        }
Esempio n. 2
0
        /// <summary>
        /// Check whether this activation is overloaded. 
        /// Returns LimitExceededException if overloaded, otherwise <c>null</c>c>
        /// </summary>
        /// <param name="log">Logger to use for reporting any overflow condition</param>
        /// <returns>Returns LimitExceededException if overloaded, otherwise <c>null</c>c></returns>
        public LimitExceededException CheckOverloaded(Logger log)
        {
            LimitValue limitValue = GetMaxEnqueuedRequestLimit();

            int maxRequestsHardLimit = limitValue.HardLimitThreshold;
            int maxRequestsSoftLimit = limitValue.SoftLimitThreshold;

            if (maxRequestsHardLimit <= 0 && maxRequestsSoftLimit <= 0) return null; // No limits are set

            int count = GetRequestCount();

            if (maxRequestsHardLimit > 0 && count > maxRequestsHardLimit) // Hard limit
            {
                log.Warn(ErrorCode.Catalog_Reject_ActivationTooManyRequests, 
                    String.Format("Overload - {0} enqueued requests for activation {1}, exceeding hard limit rejection threshold of {2}",
                        count, this, maxRequestsHardLimit));

                return new LimitExceededException(limitValue.Name, count, maxRequestsHardLimit, this.ToString());
            }
            if (maxRequestsSoftLimit > 0 && count > maxRequestsSoftLimit) // Soft limit
            {
                log.Warn(ErrorCode.Catalog_Warn_ActivationTooManyRequests,
                    String.Format("Hot - {0} enqueued requests for activation {1}, exceeding soft limit warning threshold of {2}",
                        count, this, maxRequestsSoftLimit));
                return null;
            }

            return null;
        }
Esempio n. 3
0
        internal Silo(SiloInitializationParameters initializationParams)
        {
            string name = initializationParams.Name;
            ClusterConfiguration config = initializationParams.ClusterConfig;
            this.initializationParams = initializationParams;

            SystemStatus.Current = SystemStatus.Creating;

            CurrentSilo = this;

            var startTime = DateTime.UtcNow;
            
            siloTerminatedEvent = new ManualResetEvent(false);
            
            if (!LogManager.IsInitialized)
                LogManager.Initialize(LocalConfig);

            config.OnConfigChange("Defaults/Tracing", () => LogManager.Initialize(LocalConfig, true), false);
            MultiClusterRegistrationStrategy.Initialize(config.Globals);
            StatisticsCollector.Initialize(LocalConfig);
            
            SerializationManager.Initialize(GlobalConfig.SerializationProviders, this.GlobalConfig.FallbackSerializationProvider);
            initTimeout = GlobalConfig.MaxJoinAttemptTime;
            if (Debugger.IsAttached)
            {
                initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), GlobalConfig.MaxJoinAttemptTime);
                stopTimeout = initTimeout;
            }

            var localEndpoint = this.initializationParams.SiloAddress.Endpoint;
            LogManager.MyIPEndPoint = localEndpoint;
            logger = LogManager.GetLogger("Silo", LoggerType.Runtime);

            logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode));
            if (!GCSettings.IsServerGC || !GCSettings.LatencyMode.Equals(GCLatencyMode.Batch))
            {
                logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on or with GCLatencyMode.Batch enabled - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\"> and <configuration>-<runtime>-<gcConcurrent enabled=\"false\"/>");
                logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines).");
            }

            logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------",
                this.initializationParams.Type, LocalConfig.DNSHostName, Environment.MachineName, localEndpoint, this.initializationParams.SiloAddress.Generation);
            logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with the following configuration= " + Environment.NewLine + "{1}",
                name, config.ToString(name));
            
            // Configure DI using Startup type
            this.Services = StartupBuilder.ConfigureStartup(
                this.LocalConfig.StartupTypeName,
                (services, usingCustomServiceProvider) =>
                {
                    // add system types
                    // Note: you can replace IGrainFactory with your own implementation, but 
                    // we don't recommend it, in the aspect of performance and usability
                    services.TryAddSingleton(sp => sp);
                    services.TryAddSingleton(this);
                    services.TryAddSingleton(initializationParams);
                    services.TryAddSingleton<ILocalSiloDetails>(initializationParams);
                    services.TryAddSingleton(initializationParams.ClusterConfig);
                    services.TryAddSingleton(initializationParams.GlobalConfig);
                    services.TryAddTransient(sp => initializationParams.NodeConfig);
                    services.TryAddSingleton<ITimerRegistry, TimerRegistry>();
                    services.TryAddSingleton<IReminderRegistry, ReminderRegistry>();
                    services.TryAddSingleton<IStreamProviderManager, StreamProviderManager>();
                    services.TryAddSingleton<GrainRuntime>();
                    services.TryAddSingleton<IGrainRuntime, GrainRuntime>();
                    services.TryAddSingleton<OrleansTaskScheduler>();
                    services.TryAddSingleton<GrainFactory>(sp => sp.GetService<InsideRuntimeClient>().ConcreteGrainFactory);
                    services.TryAddExisting<IGrainFactory, GrainFactory>();
                    services.TryAddExisting<IInternalGrainFactory, GrainFactory>();
                    services.TryAddSingleton<TypeMetadataCache>();
                    services.TryAddSingleton<AssemblyProcessor>();
                    services.TryAddSingleton<ActivationDirectory>();
                    services.TryAddSingleton<LocalGrainDirectory>();
                    services.TryAddExisting<ILocalGrainDirectory, LocalGrainDirectory>();
                    services.TryAddSingleton<SiloStatisticsManager>();
                    services.TryAddSingleton<ISiloPerformanceMetrics>(sp => sp.GetRequiredService<SiloStatisticsManager>().MetricsTable);
                    services.TryAddSingleton<SiloAssemblyLoader>();
                    services.TryAddSingleton<GrainTypeManager>();
                    services.TryAddExisting<IMessagingConfiguration, GlobalConfiguration>();
                    services.TryAddSingleton<MessageCenter>();
                    services.TryAddExisting<IMessageCenter, MessageCenter>();
                    services.TryAddExisting<ISiloMessageCenter, MessageCenter>();
                    services.TryAddSingleton<Catalog>();
                    services.TryAddSingleton(sp => sp.GetRequiredService<Catalog>().Dispatcher);
                    services.TryAddSingleton<InsideRuntimeClient>();
                    services.TryAddExisting<IRuntimeClient, InsideRuntimeClient>();
                    services.TryAddExisting<ISiloRuntimeClient, InsideRuntimeClient>();
                    services.TryAddSingleton<MembershipFactory>();
                    services.TryAddSingleton<MultiClusterOracleFactory>();
                    services.TryAddSingleton<LocalReminderServiceFactory>();
                    services.TryAddSingleton<DeploymentLoadPublisher>();
                    services.TryAddSingleton<IMembershipTable>(
                        sp => sp.GetRequiredService<MembershipFactory>().GetMembershipTable(sp.GetRequiredService<GlobalConfiguration>()));
                    services.TryAddSingleton<MembershipOracle>(
                        sp =>
                        sp.GetRequiredService<MembershipFactory>()
                          .CreateMembershipOracle(sp.GetRequiredService<Silo>(), sp.GetRequiredService<IMembershipTable>()));
                    services.TryAddExisting<IMembershipOracle, MembershipOracle>();
                    services.TryAddExisting<ISiloStatusOracle, MembershipOracle>();
                    services.TryAddSingleton<Func<ISiloStatusOracle>>(sp => () => sp.GetRequiredService<ISiloStatusOracle>());
                    services.TryAddSingleton<MultiClusterOracleFactory>();
                    services.TryAddSingleton<LocalReminderServiceFactory>();
                    services.TryAddSingleton<ClientObserverRegistrar>();
                    services.TryAddSingleton<SiloProviderRuntime>();
                    services.TryAddExisting<IStreamProviderRuntime, SiloProviderRuntime>();
                    services.TryAddSingleton<ImplicitStreamSubscriberTable>();

                    // Placement
                    services.TryAddSingleton<PlacementDirectorsManager>();
                    services.TryAddSingleton<IPlacementDirector<RandomPlacement>, RandomPlacementDirector>();
                    services.TryAddSingleton<IPlacementDirector<PreferLocalPlacement>, PreferLocalPlacementDirector>();
                    services.TryAddSingleton<IPlacementDirector<StatelessWorkerPlacement>, StatelessWorkerDirector>();
                    services.TryAddSingleton<IPlacementDirector<ActivationCountBasedPlacement>, ActivationCountPlacementDirector>();
                    services.TryAddSingleton<DefaultPlacementStrategy>();
                    services.TryAddSingleton<ClientObserversPlacementDirector>();
                    
                    services.TryAddSingleton<Func<IGrainRuntime>>(sp => () => sp.GetRequiredService<IGrainRuntime>());
                    services.TryAddSingleton<GrainCreator>();

                    if (initializationParams.GlobalConfig.UseVirtualBucketsConsistentRing)
                    {
                        services.TryAddSingleton<IConsistentRingProvider>(
                            sp =>
                            new VirtualBucketsRingProvider(
                                this.initializationParams.SiloAddress,
                                this.initializationParams.GlobalConfig.NumVirtualBucketsConsistentRing));
                    }
                    else
                    {
                        services.TryAddSingleton<IConsistentRingProvider>(
                            sp => new ConsistentRingProvider(this.initializationParams.SiloAddress));
                    }
                });

            this.assemblyProcessor = this.Services.GetRequiredService<AssemblyProcessor>();
            this.assemblyProcessor.Initialize();

            BufferPool.InitGlobalBufferPool(GlobalConfig);

            UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnobservedExceptionHandler);
            AppDomain.CurrentDomain.UnhandledException += this.DomainUnobservedExceptionHandler;

            try
            {
                grainFactory = Services.GetRequiredService<GrainFactory>();
            }
            catch (InvalidOperationException exc)
            {
                logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc);
                throw;
            }

            grainTypeManager = Services.GetRequiredService<GrainTypeManager>();

            // Performance metrics
            siloStatistics = Services.GetRequiredService<SiloStatisticsManager>();

            // The scheduler
            scheduler = Services.GetRequiredService<OrleansTaskScheduler>();
            healthCheckParticipants.Add(scheduler);
            
            runtimeClient = Services.GetRequiredService<InsideRuntimeClient>();

            // Initialize the message center
            messageCenter = Services.GetRequiredService<MessageCenter>();
            messageCenter.RerouteHandler = runtimeClient.RerouteMessage;
            messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage;

            // GrainRuntime can be created only here, after messageCenter was created.
            grainRuntime = Services.GetRequiredService<IGrainRuntime>();

            // Now the router/directory service
            // This has to come after the message center //; note that it then gets injected back into the message center.;
            localGrainDirectory = Services.GetRequiredService<LocalGrainDirectory>(); 
            
            // Now the activation directory.
            activationDirectory = Services.GetRequiredService<ActivationDirectory>();
            
            // Now the consistent ring provider
            RingProvider = Services.GetRequiredService<IConsistentRingProvider>();

            // to preserve backwards compatibility, only use the service provider to inject grain dependencies if the user supplied his own
            // service provider, meaning that he is explicitly opting into it.
            catalog = Services.GetRequiredService<Catalog>();

            siloStatistics.MetricsTable.Scheduler = scheduler;
            siloStatistics.MetricsTable.ActivationDirectory = activationDirectory;
            siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector;
            siloStatistics.MetricsTable.MessageCenter = messageCenter;
            
            // Now the incoming message agents
            incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, catalog.Dispatcher);
            incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, catalog.Dispatcher);
            incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, catalog.Dispatcher);

            membershipFactory = Services.GetRequiredService<MembershipFactory>();
            membershipOracle = Services.GetRequiredService<IMembershipOracle>();
            
            SystemStatus.Current = SystemStatus.Created;

            StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
                () => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.

            logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
        }
Esempio n. 4
0
        public static bool CheckTimerDelay(DateTime previousTickTime, int totalNumTicks, 
                        TimeSpan dueTime, TimeSpan timerFrequency, Logger logger, Func<string> getName, ErrorCode errorCode, bool freezeCheck)
        {
            TimeSpan timeSinceLastTick = DateTime.UtcNow - previousTickTime;
            TimeSpan exceptedTimeToNexTick = totalNumTicks == 0 ? dueTime : timerFrequency;
            TimeSpan exceptedTimeWithSlack;
            if (exceptedTimeToNexTick >= TimeSpan.FromSeconds(6))
            {
                exceptedTimeWithSlack = exceptedTimeToNexTick + TimeSpan.FromSeconds(3);
            }
            else
            {
                exceptedTimeWithSlack = exceptedTimeToNexTick.Multiply(1.5);
            }
            if (timeSinceLastTick <= exceptedTimeWithSlack) return true;

            // did not tick in the last period.
            var errMsg = String.Format("{0}{1} did not fire on time. Last fired at {2}, {3} since previous fire, should have fired after {4}.",
                freezeCheck ? "Watchdog Freeze Alert: " : "-", // 0
                getName == null ? "" : getName(),   // 1
                LogFormatter.PrintDate(previousTickTime), // 2
                timeSinceLastTick,                  // 3
                exceptedTimeToNexTick);             // 4

            if(freezeCheck)
            {
                logger.Error(errorCode, errMsg);
            }else
            {
                logger.Warn(errorCode, errMsg);
            }
            return false;
        }
Esempio n. 5
0
 private static void CheckYourOwnHealth(DateTime lastCheckt, Logger logger)
 {
     var timeSinceLastTick = (DateTime.UtcNow - lastCheckt);
     if (timeSinceLastTick > heartbeatPeriod.Multiply(2))
     {
         var gc = new[] { GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2) };
         logger.Warn(ErrorCode.SiloHeartbeatTimerStalled,
             ".NET Runtime Platform stalled for {0} - possibly GC? We are now using total of {1}MB memory. gc={2}, {3}, {4}",
             timeSinceLastTick,
             GC.GetTotalMemory(false) / (1024 * 1024),
             gc[0],
             gc[1],
             gc[2]);
     }
 }
Esempio n. 6
0
        public static void LogForDebug(Orleans.Runtime.Logger orleansLog, string format, params object[] arguments)
        {
#if DEBUG
            orleansLog.Warn(0, format, arguments);
#endif
        }
Esempio n. 7
0
 public static void LogForRelease(Orleans.Runtime.Logger orleansLog, string format, params object[] arguments)
 {
     orleansLog.Warn(0, format, arguments);
 }
Esempio n. 8
0
        internal static bool AnalyzeReadException(Exception exc, int iteration, string tableName, Logger logger)
        {
            bool isLastErrorRetriable;
            var we = exc as WebException;
            if (we != null)
            {
                isLastErrorRetriable = true;
                var statusCode = we.Status;
                logger.Warn(ErrorCode.AzureTable_10,
                    $"Intermediate issue reading Azure storage table {tableName}: HTTP status code={statusCode} Exception Type={exc.GetType().FullName} Message='{exc.Message}'",
                    exc);
            }
            else
            {
                HttpStatusCode httpStatusCode;
                string restStatus;
                if (EvaluateException(exc, out httpStatusCode, out restStatus, true))
                {
                    if (StorageErrorCodeStrings.ResourceNotFound.Equals(restStatus))
                    {
                        if (logger.IsVerbose) logger.Verbose(ErrorCode.AzureTable_DataNotFound,
                            "DataNotFound reading Azure storage table {0}:{1} HTTP status code={2} REST status code={3} Exception={4}",
                            tableName,
                            iteration == 0 ? "" : (" Repeat=" + iteration),
                            httpStatusCode,
                            restStatus,
                            exc);

                        isLastErrorRetriable = false;
                    }
                    else
                    {
                        isLastErrorRetriable = IsRetriableHttpError(httpStatusCode, restStatus);

                        logger.Warn(ErrorCode.AzureTable_11,
                            $"Intermediate issue reading Azure storage table {tableName}:{(iteration == 0 ? "" : (" Repeat=" + iteration))} IsRetriable={isLastErrorRetriable} HTTP status code={httpStatusCode} REST status code={restStatus} Exception Type={exc.GetType().FullName} Message='{exc.Message}'",
                                exc);
                    }
                }
                else
                {
                    logger.Error(ErrorCode.AzureTable_12,
                        $"Unexpected issue reading Azure storage table {tableName}: Exception Type={exc.GetType().FullName} Message='{exc.Message}'",
                                 exc);
                    isLastErrorRetriable = false;
                }
            }
            return isLastErrorRetriable;
        }
Esempio n. 9
0
        private List <string> EnumerateApprovedAssemblies()
        {
            var assemblies = new List <string>();

            foreach (var i in dirEnumArgs)
            {
                var pathName     = i.Key;
                var searchOption = i.Value;

                if (!Directory.Exists(pathName))
                {
                    logger.Warn(ErrorCode.Loader_DirNotFound, "Unable to find directory {0}; skipping.", pathName);
                    continue;
                }

                logger.Info(
                    searchOption == SearchOption.TopDirectoryOnly ?
                    "Searching for assemblies in {0}..." :
                    "Recursively searching for assemblies in {0}...",
                    pathName);

                var candidates =
                    Directory.EnumerateFiles(pathName, "*.dll", searchOption)
                    .Select(Path.GetFullPath)
                    .Distinct()
                    .ToArray();

                // This is a workaround for the behavior of ReflectionOnlyLoad/ReflectionOnlyLoadFrom
                // that appear not to automatically resolve dependencies.
                // We are trying to pre-load all dlls we find in the folder, so that if one of these
                // assemblies happens to be a dependency of an assembly we later on call
                // Assembly.DefinedTypes on, the dependency will be already loaded and will get
                // automatically resolved. Ugly, but seems to solve the problem.

                foreach (var j in candidates)
                {
                    try
                    {
                        if (logger.IsVerbose)
                        {
                            logger.Verbose("Trying to pre-load {0} to reflection-only context.", j);
                        }
                        Assembly.ReflectionOnlyLoadFrom(j);
                    }
                    catch (Exception)
                    {
                        if (logger.IsVerbose)
                        {
                            logger.Verbose("Failed to pre-load assembly {0} in reflection-only context.", j);
                        }
                    }
                }

                foreach (var j in candidates)
                {
                    if (AssemblyPassesLoadCriteria(j))
                    {
                        assemblies.Add(j);
                    }
                }
            }

            return(assemblies);
        }
Esempio n. 10
0
        internal Silo(string name, SiloType siloType, ClusterConfiguration config, ILocalDataStore keyStore)
        {
            SystemStatus.Current = SystemStatus.Creating;

            CurrentSilo = this;

            var startTime = DateTime.UtcNow;

            this.siloType = siloType;
            Name          = name;

            siloTerminatedEvent = new ManualResetEvent(false);

            OrleansConfig = config;
            globalConfig  = config.Globals;
            config.OnConfigChange("Defaults", () => nodeConfig = config.GetOrCreateNodeConfigurationForSilo(name));

            if (!LogManager.IsInitialized)
            {
                LogManager.Initialize(nodeConfig);
            }

            config.OnConfigChange("Defaults/Tracing", () => LogManager.Initialize(nodeConfig, true), false);
            MultiClusterRegistrationStrategy.Initialize(config.Globals);
            ActivationData.Init(config, nodeConfig);
            StatisticsCollector.Initialize(nodeConfig);

            SerializationManager.Initialize(globalConfig.SerializationProviders, this.globalConfig.FallbackSerializationProvider);
            initTimeout = globalConfig.MaxJoinAttemptTime;
            if (Debugger.IsAttached)
            {
                initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), globalConfig.MaxJoinAttemptTime);
                stopTimeout = initTimeout;
            }

            IPEndPoint here       = nodeConfig.Endpoint;
            int        generation = nodeConfig.Generation;

            if (generation == 0)
            {
                generation            = SiloAddress.AllocateNewGeneration();
                nodeConfig.Generation = generation;
            }
            LogManager.MyIPEndPoint = here;
            logger = LogManager.GetLogger("Silo", LoggerType.Runtime);

            logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode));
            if (!GCSettings.IsServerGC || !GCSettings.LatencyMode.Equals(GCLatencyMode.Batch))
            {
                logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on or with GCLatencyMode.Batch enabled - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\"> and <configuration>-<runtime>-<gcConcurrent enabled=\"false\"/>");
                logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines).");
            }

            logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------",
                        siloType, nodeConfig.DNSHostName, Environment.MachineName, here, generation);
            logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with the following configuration= " + Environment.NewLine + "{1}",
                        name, config.ToString(name));

            if (keyStore != null)
            {
                // Re-establish reference to shared local key store in this app domain
                LocalDataStoreInstance.LocalDataStore = keyStore;
            }

            // Configure DI using Startup type
            bool usingCustomServiceProvider;

            Services = StartupBuilder.ConfigureStartup(nodeConfig.StartupTypeName, out usingCustomServiceProvider);

            healthCheckParticipants = new List <IHealthCheckParticipant>();
            allSiloProviders        = new List <IProvider>();

            BufferPool.InitGlobalBufferPool(globalConfig);
            PlacementStrategy.Initialize(globalConfig);

            UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnobservedExceptionHandler);
            AppDomain.CurrentDomain.UnhandledException +=
                (obj, ev) => DomainUnobservedExceptionHandler(obj, (Exception)ev.ExceptionObject);

            try
            {
                grainFactory = Services.GetRequiredService <GrainFactory>();
            }
            catch (InvalidOperationException exc)
            {
                logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc);
                throw;
            }

            typeManager = new GrainTypeManager(
                here.Address.Equals(IPAddress.Loopback),
                grainFactory,
                new SiloAssemblyLoader(OrleansConfig.Defaults.AdditionalAssemblyDirectories));

            // Performance metrics
            siloStatistics = new SiloStatisticsManager(globalConfig, nodeConfig);
            config.OnConfigChange("Defaults/LoadShedding", () => siloStatistics.MetricsTable.NodeConfig = nodeConfig, false);

            // The scheduler
            scheduler = new OrleansTaskScheduler(globalConfig, nodeConfig);
            healthCheckParticipants.Add(scheduler);

            // Initialize the message center
            var mc = new MessageCenter(here, generation, globalConfig, siloStatistics.MetricsTable);

            if (nodeConfig.IsGatewayNode)
            {
                mc.InstallGateway(nodeConfig.ProxyGatewayEndpoint);
            }

            messageCenter = mc;

            SiloIdentity = SiloAddress.ToLongString();

            // GrainRuntime can be created only here, after messageCenter was created.
            grainRuntime = new GrainRuntime(
                globalConfig.ServiceId,
                SiloIdentity,
                grainFactory,
                new TimerRegistry(),
                new ReminderRegistry(),
                new StreamProviderManager(),
                Services);


            // Now the router/directory service
            // This has to come after the message center //; note that it then gets injected back into the message center.;
            localGrainDirectory = new LocalGrainDirectory(this);

            RegistrarManager.InitializeGrainDirectoryManager(localGrainDirectory, globalConfig.GlobalSingleInstanceNumberRetries);

            // Now the activation directory.
            // This needs to know which router to use so that it can keep the global directory in synch with the local one.
            activationDirectory = new ActivationDirectory();

            // Now the consistent ring provider
            RingProvider = GlobalConfig.UseVirtualBucketsConsistentRing ?
                           (IConsistentRingProvider) new VirtualBucketsRingProvider(SiloAddress, GlobalConfig.NumVirtualBucketsConsistentRing)
                : new ConsistentRingProvider(SiloAddress);

            // to preserve backwards compatibility, only use the service provider to inject grain dependencies if the user supplied his own
            // service provider, meaning that he is explicitly opting into it.
            var grainCreator = new GrainCreator(grainRuntime, usingCustomServiceProvider ? Services : null);

            Action <Dispatcher> setDispatcher;

            catalog = new Catalog(Constants.CatalogId, SiloAddress, Name, LocalGrainDirectory, typeManager, scheduler, activationDirectory, config, grainCreator, out setDispatcher);
            var dispatcher = new Dispatcher(scheduler, messageCenter, catalog, config);

            setDispatcher(dispatcher);

            RuntimeClient.Current = new InsideRuntimeClient(
                dispatcher,
                catalog,
                LocalGrainDirectory,
                SiloAddress,
                config,
                RingProvider,
                typeManager,
                grainFactory);
            messageCenter.RerouteHandler       = InsideRuntimeClient.Current.RerouteMessage;
            messageCenter.SniffIncomingMessage = InsideRuntimeClient.Current.SniffIncomingMessage;

            siloStatistics.MetricsTable.Scheduler           = scheduler;
            siloStatistics.MetricsTable.ActivationDirectory = activationDirectory;
            siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector;
            siloStatistics.MetricsTable.MessageCenter       = messageCenter;

            DeploymentLoadPublisher.CreateDeploymentLoadPublisher(this, globalConfig);
            PlacementDirectorsManager.CreatePlacementDirectorsManager(globalConfig);

            // Now the incoming message agents
            incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, dispatcher);
            incomingPingAgent   = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, dispatcher);
            incomingAgent       = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, dispatcher);

            membershipFactory   = new MembershipFactory();
            multiClusterFactory = new MultiClusterOracleFactory();
            reminderFactory     = new LocalReminderServiceFactory();

            SystemStatus.Current = SystemStatus.Created;

            StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
                                              () => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.

            logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
        }