static CachedReflectionOnlyTypeResolver()
{
Instance = new CachedReflectionOnlyTypeResolver();
logger = TraceLogger.GetLogger("AssemblyLoader.CachedReflectionOnlyTypeResolver");
}
internal Silo(string name, SiloType siloType, ClusterConfiguration config, ILocalDataStore keyStore)
{
SystemStatus.Current = SystemStatus.Creating;
CurrentSilo = this;
var startTime = DateTime.UtcNow;
this.siloType = siloType;
Name = name;
siloTerminatedEvent = new ManualResetEvent(false);
OrleansConfig = config;
globalConfig = config.Globals;
config.OnConfigChange("Defaults", () => nodeConfig = config.GetConfigurationForNode(name));
if (!TraceLogger.IsInitialized)
{
TraceLogger.Initialize(nodeConfig);
}
config.OnConfigChange("Defaults/Tracing", () => TraceLogger.Initialize(nodeConfig, true), false);
ActivationData.Init(config, nodeConfig);
StatisticsCollector.Initialize(nodeConfig);
SerializationManager.Initialize(globalConfig.UseStandardSerializer);
initTimeout = globalConfig.MaxJoinAttemptTime;
if (Debugger.IsAttached)
{
initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), globalConfig.MaxJoinAttemptTime);
stopTimeout = initTimeout;
}
IPEndPoint here = nodeConfig.Endpoint;
int generation = nodeConfig.Generation;
if (generation == 0)
{
generation = SiloAddress.AllocateNewGeneration();
nodeConfig.Generation = generation;
}
TraceLogger.MyIPEndPoint = here;
logger = TraceLogger.GetLogger("Silo", TraceLogger.LoggerType.Runtime);
logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on {1} at {2}, gen {3} --------------", siloType, nodeConfig.DNSHostName, here, generation);
logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with runtime Version='{1}' Config= " + Environment.NewLine + "{2}", name, RuntimeVersion.Current, config.ToString(name));
if (keyStore != null)
{
// Re-establish reference to shared local key store in this app domain
LocalDataStoreInstance.LocalDataStore = keyStore;
}
healthCheckParticipants = new List <IHealthCheckParticipant>();
allSiloProviders = new List <IProvider>();
BufferPool.InitGlobalBufferPool(globalConfig);
PlacementStrategy.Initialize(globalConfig);
UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnobservedExceptionHandler);
AppDomain.CurrentDomain.UnhandledException +=
(obj, ev) => DomainUnobservedExceptionHandler(obj, (Exception)ev.ExceptionObject);
grainFactory = new GrainFactory();
typeManager = new GrainTypeManager(here.Address.Equals(IPAddress.Loopback), grainFactory);
// Performance metrics
siloStatistics = new SiloStatisticsManager(globalConfig, nodeConfig);
config.OnConfigChange("Defaults/LoadShedding", () => siloStatistics.MetricsTable.NodeConfig = nodeConfig, false);
// The scheduler
scheduler = new OrleansTaskScheduler(globalConfig, nodeConfig);
healthCheckParticipants.Add(scheduler);
// Initialize the message center
var mc = new MessageCenter(here, generation, globalConfig, siloStatistics.MetricsTable);
if (nodeConfig.IsGatewayNode)
{
mc.InstallGateway(nodeConfig.ProxyGatewayEndpoint);
}
messageCenter = mc;
SiloIdentity = SiloAddress.ToLongString();
// GrainRuntime can be created only here, after messageCenter was created.
grainRuntime = new GrainRuntime(
globalConfig.ServiceId,
SiloIdentity,
grainFactory,
new TimerRegistry(),
new ReminderRegistry(),
new StreamProviderManager());
// Now the router/directory service
// This has to come after the message center //; note that it then gets injected back into the message center.;
localGrainDirectory = new LocalGrainDirectory(this);
// Now the activation directory.
// This needs to know which router to use so that it can keep the global directory in synch with the local one.
activationDirectory = new ActivationDirectory();
// Now the consistent ring provider
RingProvider = GlobalConfig.UseVirtualBucketsConsistentRing ?
(IConsistentRingProvider) new VirtualBucketsRingProvider(SiloAddress, GlobalConfig.NumVirtualBucketsConsistentRing)
: new ConsistentRingProvider(SiloAddress);
Action <Dispatcher> setDispatcher;
catalog = new Catalog(Constants.CatalogId, SiloAddress, Name, LocalGrainDirectory, typeManager, scheduler, activationDirectory, config, grainRuntime, out setDispatcher);
var dispatcher = new Dispatcher(scheduler, messageCenter, catalog, config);
setDispatcher(dispatcher);
RuntimeClient.Current = new InsideRuntimeClient(
dispatcher,
catalog,
LocalGrainDirectory,
SiloAddress,
config,
RingProvider,
typeManager,
grainFactory);
messageCenter.RerouteHandler = InsideRuntimeClient.Current.RerouteMessage;
messageCenter.SniffIncomingMessage = InsideRuntimeClient.Current.SniffIncomingMessage;
siloStatistics.MetricsTable.Scheduler = scheduler;
siloStatistics.MetricsTable.ActivationDirectory = activationDirectory;
siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector;
siloStatistics.MetricsTable.MessageCenter = messageCenter;
DeploymentLoadPublisher.CreateDeploymentLoadPublisher(this, globalConfig);
PlacementDirectorsManager.CreatePlacementDirectorsManager(globalConfig);
// Now the incoming message agents
incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, dispatcher);
incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, dispatcher);
incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, dispatcher);
membershipFactory = new MembershipFactory();
reminderFactory = new LocalReminderServiceFactory();
SystemStatus.Current = SystemStatus.Created;
StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
() => TraceLogger.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.
TestHookup = new TestHookups(this);
logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
}
private void DoStart()
{
lock (lockable)
{
if (!SystemStatus.Current.Equals(SystemStatus.Created))
{
throw new InvalidOperationException(String.Format("Calling Silo.Start() on a silo which is not in the Created state. This silo is in the {0} state.", SystemStatus.Current));
}
SystemStatus.Current = SystemStatus.Starting;
}
logger.Info(ErrorCode.SiloStarting, "Silo Start()");
// Hook up to receive notification of process exit / Ctrl-C events
AppDomain.CurrentDomain.ProcessExit += HandleProcessExit;
Console.CancelKeyPress += HandleProcessExit;
ConfigureThreadPoolAndServicePointSettings();
// This has to start first so that the directory system target factory gets loaded before we start the router.
typeManager.Start();
InsideRuntimeClient.Current.Start();
// The order of these 4 is pretty much arbitrary.
scheduler.Start();
messageCenter.Start();
incomingPingAgent.Start();
incomingSystemAgent.Start();
incomingAgent.Start();
LocalGrainDirectory.Start();
// Set up an execution context for this thread so that the target creation steps can use asynch values.
RuntimeContext.InitializeMainThread();
SiloProviderRuntime.Initialize(GlobalConfig, SiloIdentity, grainFactory);
InsideRuntimeClient.Current.CurrentStreamProviderRuntime = SiloProviderRuntime.Instance;
statisticsProviderManager = new StatisticsProviderManager("Statistics", SiloProviderRuntime.Instance);
string statsProviderName = statisticsProviderManager.LoadProvider(GlobalConfig.ProviderConfigurations)
.WaitForResultWithThrow(initTimeout);
if (statsProviderName != null)
{
LocalConfig.StatisticsProviderName = statsProviderName;
}
allSiloProviders.AddRange(statisticsProviderManager.GetProviders());
// can call SetSiloMetricsTableDataManager only after MessageCenter is created (dependency on this.SiloAddress).
siloStatistics.SetSiloStatsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
siloStatistics.SetSiloMetricsTableDataManager(this, nodeConfig).WaitWithThrow(initTimeout);
IMembershipTable membershipTable = membershipFactory.GetMembershipTable(GlobalConfig.LivenessType, GlobalConfig.MembershipTableAssembly);
membershipOracle = membershipFactory.CreateMembershipOracle(this, membershipTable);
// This has to follow the above steps that start the runtime components
CreateSystemTargets();
InjectDependencies();
// Validate the configuration.
GlobalConfig.Application.ValidateConfiguration(logger);
// ensure this runs in the grain context, wait for it to complete
scheduler.QueueTask(CreateSystemGrains, catalog.SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("System grains created successfully.");
}
// Initialize storage providers once we have a basic silo runtime environment operating
storageProviderManager = new StorageProviderManager(grainFactory);
scheduler.QueueTask(
() => storageProviderManager.LoadStorageProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
catalog.SetStorageManager(storageProviderManager);
allSiloProviders.AddRange(storageProviderManager.GetProviders());
if (logger.IsVerbose)
{
logger.Verbose("Storage provider manager created successfully.");
}
// Load and init stream providers before silo becomes active
var siloStreamProviderManager = (StreamProviderManager)grainRuntime.StreamProviderManager;
scheduler.QueueTask(
() => siloStreamProviderManager.LoadStreamProviders(this.GlobalConfig.ProviderConfigurations, SiloProviderRuntime.Instance),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
InsideRuntimeClient.Current.CurrentStreamProviderManager = siloStreamProviderManager;
allSiloProviders.AddRange(siloStreamProviderManager.GetProviders());
if (logger.IsVerbose)
{
logger.Verbose("Stream provider manager created successfully.");
}
ISchedulingContext statusOracleContext = ((SystemTarget)LocalSiloStatusOracle).SchedulingContext;
bool waitForPrimaryToStart = globalConfig.PrimaryNodeIsRequired && siloType != SiloType.Primary;
if (waitForPrimaryToStart) // only in MembershipTableGrain case.
{
scheduler.QueueTask(() => membershipFactory.WaitForTableToInit(membershipTable), statusOracleContext)
.WaitWithThrow(initTimeout);
}
scheduler.QueueTask(() => membershipTable.InitializeMembershipTable(this.GlobalConfig, true, TraceLogger.GetLogger(membershipTable.GetType().Name)), statusOracleContext)
.WaitWithThrow(initTimeout);
scheduler.QueueTask(() => LocalSiloStatusOracle.Start(), statusOracleContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Local silo status oracle created successfully.");
}
scheduler.QueueTask(LocalSiloStatusOracle.BecomeActive, statusOracleContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Local silo status oracle became active successfully.");
}
try
{
siloStatistics.Start(LocalConfig);
if (logger.IsVerbose)
{
logger.Verbose("Silo statistics manager started successfully.");
}
// Finally, initialize the deployment load collector, for grains with load-based placement
scheduler.QueueTask(DeploymentLoadPublisher.Instance.Start, DeploymentLoadPublisher.Instance.SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Silo deployment load publisher started successfully.");
}
// Start background timer tick to watch for platform execution stalls, such as when GC kicks in
platformWatchdog = new Watchdog(nodeConfig.StatisticsLogWriteInterval, healthCheckParticipants);
platformWatchdog.Start();
if (logger.IsVerbose)
{
logger.Verbose("Silo platform watchdog started successfully.");
}
if (reminderService != null)
{
// so, we have the view of the membership in the consistentRingProvider. We can start the reminder service
scheduler.QueueTask(reminderService.Start, ((SystemTarget)reminderService).SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Reminder service started successfully.");
}
}
// Start stream providers after silo is active (so the pulling agents don't start sending messages before silo is active).
scheduler.QueueTask(siloStreamProviderManager.StartStreamProviders, providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Stream providers started successfully.");
}
var bootstrapProviderManager = new BootstrapProviderManager();
scheduler.QueueTask(
() => bootstrapProviderManager.LoadAppBootstrapProviders(GlobalConfig.ProviderConfigurations),
providerManagerSystemTarget.SchedulingContext)
.WaitWithThrow(initTimeout);
BootstrapProviders = bootstrapProviderManager.GetProviders(); // Data hook for testing & diagnotics
allSiloProviders.AddRange(BootstrapProviders);
if (logger.IsVerbose)
{
logger.Verbose("App bootstrap calls done successfully.");
}
// Now that we're active, we can start the gateway
var mc = messageCenter as MessageCenter;
if (mc != null)
{
mc.StartGateway(clientRegistrar);
}
if (logger.IsVerbose)
{
logger.Verbose("Message gateway service started successfully.");
}
scheduler.QueueTask(clientRegistrar.Start, clientRegistrar.SchedulingContext)
.WaitWithThrow(initTimeout);
if (logger.IsVerbose)
{
logger.Verbose("Client registrar service started successfully.");
}
SystemStatus.Current = SystemStatus.Running;
}
catch (Exception exc)
{
SafeExecute(() => logger.Error(ErrorCode.Runtime_Error_100330, String.Format("Error starting silo {0}. Going to FastKill().", SiloAddress), exc));
FastKill(); // if failed after Membership became active, mark itself as dead in Membership abale.
throw;
}
if (logger.IsVerbose)
{
logger.Verbose("Silo.Start complete: System status = {0}", SystemStatus.Current);
}
}
public static void SafeExecute(Action action, TraceLogger logger = null, string caller = null)
{
SafeExecute(action, logger, caller == null ? (Func <string>)null : () => caller);
}
public static bool CheckTimerDelay(DateTime previousTickTime, int totalNumTicks,
TimeSpan dueTime, TimeSpan timerFrequency, TraceLogger logger, Func <string> getName, ErrorCode errorCode, bool freezeCheck)
{
TimeSpan timeSinceLastTick = DateTime.UtcNow - previousTickTime;
TimeSpan exceptedTimeToNexTick = totalNumTicks == 0 ? dueTime : timerFrequency;
TimeSpan exceptedTimeWithSlack;
if (exceptedTimeToNexTick >= TimeSpan.FromSeconds(6))
{
exceptedTimeWithSlack = exceptedTimeToNexTick + TimeSpan.FromSeconds(3);
}
else
{
exceptedTimeWithSlack = exceptedTimeToNexTick.Multiply(1.5);
}
if (timeSinceLastTick <= exceptedTimeWithSlack)
{
return(true);
}
// did not tick in the last period.
var errMsg = String.Format("{0}{1} did not fire on time. Last fired at {2}, {3} since previous fire, should have fired after {4}.",
freezeCheck ? "Watchdog Freeze Alert: " : "-", // 0
getName == null ? "" : getName(), // 1
TraceLogger.PrintDate(previousTickTime), // 2
timeSinceLastTick, // 3
exceptedTimeToNexTick); // 4
if (freezeCheck)
{
logger.Error(errorCode, errMsg);
}
else
{
logger.Warn(errorCode, errMsg);
}
return(false);
}
internal LocalReminderServiceFactory()
{
logger = TraceLogger.GetLogger("ReminderFactory", TraceLogger.LoggerType.Runtime);
}
static Message()
{
lifecycleStatistics = new Dictionary <string, TransitionStats[, ]>();
logger = TraceLogger.GetLogger("Message", TraceLogger.LoggerType.Runtime);
}
internal ClientStatisticsManager(IStatisticsConfiguration config)
{
runtimeStats = new RuntimeStatisticsGroup();
logStatistics = new LogStatistics(config.StatisticsLogWriteInterval, false);
logger = TraceLogger.GetLogger(GetType().Name);
}
/// <summary>
/// Initializes static members of the <see cref="AssemblyProcessor"/> class.
/// </summary>
static AssemblyProcessor()
{
Logger = TraceLogger.GetLogger("AssemblyProcessor");
}
public Logger GetLogger(string loggerName, TraceLogger.LoggerType logType)
{
return(TraceLogger.GetLogger(loggerName, logType));
}
private static void AgentThreadProc(Object obj)
{
var agent = obj as AsynchAgent;
if (agent == null)
{
var log = TraceLogger.GetLogger("RuntimeCore.AsynchAgent");
log.Error(ErrorCode.Runtime_Error_100022, "Agent thread started with incorrect parameter type");
return;
}
try
{
LogStatus(agent.Log, "Starting AsyncAgent {0} on managed thread {1}", agent.Name, Thread.CurrentThread.ManagedThreadId);
CounterStatistic.SetOrleansManagedThread(); // do it before using CounterStatistic.
CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.RUNTIME_THREADS_ASYNC_AGENT_PERAGENTTYPE, agent.type)).Increment();
CounterStatistic.FindOrCreate(StatisticNames.RUNTIME_THREADS_ASYNC_AGENT_TOTAL_THREADS_CREATED).Increment();
agent.Run();
}
catch (Exception exc)
{
if (agent.State == ThreadState.Running) // If we're stopping, ignore exceptions
{
var log = agent.Log;
switch (agent.OnFault)
{
case FaultBehavior.CrashOnFault:
Console.WriteLine(
"The {0} agent has thrown an unhandled exception, {1}. The process will be terminated.",
agent.Name, exc);
log.Error(ErrorCode.Runtime_Error_100023,
"AsynchAgent Run method has thrown an unhandled exception. The process will be terminated.",
exc);
log.Fail(ErrorCode.Runtime_Error_100024, "Terminating process because of an unhandled exception caught in AsynchAgent.Run.");
break;
case FaultBehavior.IgnoreFault:
log.Error(ErrorCode.Runtime_Error_100025, "AsynchAgent Run method has thrown an unhandled exception. The agent will exit.",
exc);
agent.State = ThreadState.Stopped;
break;
case FaultBehavior.RestartOnFault:
log.Error(ErrorCode.Runtime_Error_100026,
"AsynchAgent Run method has thrown an unhandled exception. The agent will be restarted.",
exc);
agent.State = ThreadState.Stopped;
try
{
agent.Start();
}
catch (Exception ex)
{
log.Error(ErrorCode.Runtime_Error_100027, "Unable to restart AsynchAgent", ex);
agent.State = ThreadState.Stopped;
}
break;
}
}
}
finally
{
CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.RUNTIME_THREADS_ASYNC_AGENT_PERAGENTTYPE, agent.type)).DecrementBy(1);
agent.Log.Info(ErrorCode.Runtime_Error_100328, "Stopping AsyncAgent {0} that runs on managed thread {1}", agent.Name, Thread.CurrentThread.ManagedThreadId);
}
}
static Message()
{
logger = TraceLogger.GetLogger("Message", TraceLogger.LoggerType.Runtime);
}
internal PluginManager(TraceLogger logger)
{
this.logger = logger;
}
internal LogStatistics(TimeSpan writeInterval, bool isSilo)
{
reportFrequency = writeInterval;
logger = TraceLogger.GetLogger(isSilo ? "SiloLogStatistics" : "ClientLogStatistics", TraceLogger.LoggerType.Runtime);
}
