public static IEnumerable <TypeInfo> GetDefinedTypes(Assembly assembly, Logger logger)
{
try
{
return(assembly.DefinedTypes);
}
catch (Exception exception)
{
if (logger != null && logger.IsWarning)
{
var message =
$"Exception loading types from assembly '{assembly.FullName}': {LogFormatter.PrintException(exception)}.";
logger.Warn(ErrorCode.Loader_TypeLoadError_5, message, exception);
}
var typeLoadException = exception as ReflectionTypeLoadException;
if (typeLoadException != null)
{
return(typeLoadException.Types?.Where(type => type != null).Select(type => type.GetTypeInfo()) ??
Enumerable.Empty <TypeInfo>());
}
return(Enumerable.Empty <TypeInfo>());
}
}
private static string FormatLogMessage_Impl(
DateTime timestamp,
Severity severity,
LoggerType loggerType,
string caller,
string message,
IPEndPoint myIPEndPoint,
Exception exception,
int errorCode,
bool includeStackTrace)
{
if (severity == Severity.Error)
{
message = "!!!!!!!!!! " + message;
}
string ip = myIPEndPoint == null ? String.Empty : myIPEndPoint.ToString();
if (loggerType.Equals(LoggerType.Grain))
{
// Grain identifies itself, so I don't want an additional long string in the prefix.
// This is just a temporal solution to ease the dev. process, can remove later.
ip = String.Empty;
}
string exc = includeStackTrace ? LogFormatter.PrintException(exception) : LogFormatter.PrintExceptionWithoutStackTrace(exception);
string msg = String.Format("[{0} {1,5}\t{2}\t{3}\t{4}\t{5}]\t{6}\t{7}",
LogManager.ShowDate ? LogFormatter.PrintDate(timestamp) : LogFormatter.PrintTime(timestamp), //0
Thread.CurrentThread.ManagedThreadId, //1
LogManager.SeverityTable[(int)severity], //2
errorCode, //3
caller, //4
ip, //5
message, //6
exc); //7
return(msg);
}
public async Task Invoke(IGrainContext target, Message message)
{
try
{
// Don't process messages that have already timed out
if (message.IsExpired)
{
this.messagingTrace.OnDropExpiredMessage(message, MessagingStatisticsGroup.Phase.Invoke);
return;
}
RequestContextExtensions.Import(message.RequestContextData);
bool startNewTransaction = false;
ITransactionInfo transactionInfo = message.TransactionInfo;
if (message.IsTransactionRequired && transactionInfo == null)
{
// TODO: this should be a configurable parameter
var transactionTimeout = Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(10);
// Start a new transaction
transactionInfo = await this.transactionAgent.StartTransaction(message.IsReadOnly, transactionTimeout);
startNewTransaction = true;
}
if (transactionInfo != null)
{
TransactionContext.SetTransactionInfo(transactionInfo);
}
object resultObject;
try
{
var request = (InvokeMethodRequest)message.BodyObject;
if (request.Arguments != null)
{
CancellationSourcesExtension.RegisterCancellationTokens(target, request);
}
if (!this.invokers.TryGet(message.InterfaceType, out var invoker))
{
throw new KeyNotFoundException($"Could not find an invoker for interface {message.InterfaceType}");
}
messagingTrace.OnInvokeMessage(message);
var requestInvoker = new GrainMethodInvoker(target, request, invoker, GrainCallFilters, interfaceToImplementationMapping);
await requestInvoker.Invoke();
resultObject = requestInvoker.Result;
}
catch (Exception exc1)
{
if (message.Direction == Message.Directions.OneWay)
{
this.invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message + ": " + LogFormatter.PrintException(exc1), exc1);
}
else if (invokeExceptionLogger.IsEnabled(LogLevel.Debug))
{
this.invokeExceptionLogger.Debug(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message + ": " + LogFormatter.PrintException(exc1), exc1);
}
if (transactionInfo != null)
{
transactionInfo.ReconcilePending();
// Record reason for abort, if not already set.
transactionInfo.RecordException(exc1, serializationManager);
if (startNewTransaction)
{
exc1 = transactionInfo.MustAbort(serializationManager);
await this.transactionAgent.Abort(transactionInfo);
TransactionContext.Clear();
}
}
// If a grain allowed an inconsistent state exception to escape and the exception originated from
// this activation, then deactivate it.
var ise = exc1 as InconsistentStateException;
if (ise != null && ise.IsSourceActivation)
{
// Mark the exception so that it doesn't deactivate any other activations.
ise.IsSourceActivation = false;
this.invokeExceptionLogger.Info($"Deactivating {target} due to inconsistent state.");
this.DeactivateOnIdle(target.ActivationId);
}
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc1);
}
return;
}
OrleansTransactionException transactionException = null;
if (transactionInfo != null)
{
try
{
transactionInfo.ReconcilePending();
transactionException = transactionInfo.MustAbort(serializationManager);
// This request started the transaction, so we try to commit before returning,
// or if it must abort, tell participants that it aborted
if (startNewTransaction)
{
try
{
if (transactionException is null)
{
var(status, exception) = await this.transactionAgent.Resolve(transactionInfo);
if (status != TransactionalStatus.Ok)
{
transactionException = status.ConvertToUserException(transactionInfo.Id, exception);
}
}
else
{
await this.transactionAgent.Abort(transactionInfo);
}
}
finally
{
TransactionContext.Clear();
}
}
}
catch (Exception e)
{
// we should never hit this, but if we do, the following message will help us diagnose
this.logger.LogError(e, "Error in transaction post-grain-method-invocation code");
throw;
}
}
if (message.Direction != Message.Directions.OneWay)
{
if (transactionException != null)
{
SafeSendExceptionResponse(message, transactionException);
}
else
{
SafeSendResponse(message, resultObject);
}
}
return;
}
catch (Exception exc2)
{
this.logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2);
TransactionContext.Clear();
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc2);
}
}
finally
{
RequestContext.Clear();
}
}
private static bool IsCompatibleWithCurrentProcess(string fileName, out string[] complaints)
{
complaints = null;
Stream peImage = null;
try
{
peImage = File.Open(fileName, FileMode.Open, FileAccess.Read, FileShare.Read);
using (var peReader = new PEReader(peImage, PEStreamOptions.PrefetchMetadata))
{
peImage = null;
if (peReader.HasMetadata)
{
var processorArchitecture = ProcessorArchitecture.MSIL;
var isPureIL = (peReader.PEHeaders.CorHeader.Flags & CorFlags.ILOnly) != 0;
if (peReader.PEHeaders.PEHeader.Magic == PEMagic.PE32Plus)
{
processorArchitecture = ProcessorArchitecture.Amd64;
}
else if ((peReader.PEHeaders.CorHeader.Flags & CorFlags.Requires32Bit) != 0 || !isPureIL)
{
processorArchitecture = ProcessorArchitecture.X86;
}
var isLoadable = (isPureIL && processorArchitecture == ProcessorArchitecture.MSIL) ||
(Environment.Is64BitProcess && processorArchitecture == ProcessorArchitecture.Amd64) ||
(!Environment.Is64BitProcess && processorArchitecture == ProcessorArchitecture.X86);
if (!isLoadable)
{
complaints = new[] { $"The file {fileName} is not loadable into this process, either it is not an MSIL assembly or the complied for a different processor architecture." };
}
return(isLoadable);
}
else
{
complaints = new[] { $"The file {fileName} does not contain any CLR metadata, probably it is a native file." };
return(false);
}
}
}
catch (IOException)
{
return(false);
}
catch (BadImageFormatException)
{
return(false);
}
catch (UnauthorizedAccessException)
{
return(false);
}
catch (MissingMethodException)
{
complaints = new[] { "MissingMethodException occurred. Please try to add a BindingRedirect for System.Collections.ImmutableCollections to the App.config file to correct this error." };
return(false);
}
catch (Exception ex)
{
complaints = new[] { LogFormatter.PrintException(ex) };
return(false);
}
finally
{
peImage?.Dispose();
}
}
public async Task Invoke(IAddressable target, IInvokable invokable, Message message)
{
try
{
// Don't process messages that have already timed out
if (message.IsExpired)
{
this.messagingTrace.OnDropExpiredMessage(message, MessagingStatisticsGroup.Phase.Invoke);
return;
}
RequestContextExtensions.Import(message.RequestContextData);
if (schedulingOptions.PerformDeadlockDetection && !message.TargetGrain.IsSystemTarget)
{
UpdateDeadlockInfoInRequestContext(new RequestInvocationHistory(message.TargetGrain, message.TargetActivation));
// RequestContext is automatically saved in the msg upon send and propagated to the next hop
// in RuntimeClient.CreateMessage -> RequestContextExtensions.ExportToMessage(message);
}
bool startNewTransaction = false;
ITransactionInfo transactionInfo = message.TransactionInfo;
if (message.IsTransactionRequired && transactionInfo == null)
{
// TODO: this should be a configurable parameter
var transactionTimeout = Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(10);
// Start a new transaction
transactionInfo = await this.transactionAgent.StartTransaction(message.IsReadOnly, transactionTimeout);
startNewTransaction = true;
}
if (transactionInfo != null)
{
TransactionContext.SetTransactionInfo(transactionInfo);
}
object resultObject;
try
{
var request = (InvokeMethodRequest)message.BodyObject;
if (request.Arguments != null)
{
CancellationSourcesExtension.RegisterCancellationTokens(target, request, this.loggerFactory, logger, this, this.cancellationTokenRuntime);
}
var invoker = invokable.GetInvoker(typeManager, request.InterfaceId, message.GenericGrainType);
if (invoker is IGrainExtensionMethodInvoker &&
!(target is IGrainExtension) &&
!TryInstallExtension(request.InterfaceId, invokable, message.GenericGrainType, ref invoker))
{
// We are trying the invoke a grain extension method on a grain
// -- most likely reason is that the dynamic extension is not installed for this grain
// So throw a specific exception here rather than a general InvalidCastException
var error = String.Format(
"Extension not installed on grain {0} attempting to invoke type {1} from invokable {2}",
target.GetType().FullName, invoker.GetType().FullName, invokable.GetType().FullName);
var exc = new GrainExtensionNotInstalledException(error);
string extraDebugInfo = null;
#if DEBUG
extraDebugInfo = Utils.GetStackTrace();
#endif
this.logger.Warn(ErrorCode.Stream_ExtensionNotInstalled,
string.Format("{0} for message {1} {2}", error, message, extraDebugInfo), exc);
throw exc;
}
messagingTrace.OnInvokeMessage(message);
var requestInvoker = new GrainMethodInvoker(target, request, invoker, GrainCallFilters, interfaceToImplementationMapping);
await requestInvoker.Invoke();
resultObject = requestInvoker.Result;
}
catch (Exception exc1)
{
if (message.Direction == Message.Directions.OneWay)
{
this.invokeExceptionLogger.Warn(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message + ": " + LogFormatter.PrintException(exc1), exc1);
}
else if (invokeExceptionLogger.IsEnabled(LogLevel.Debug))
{
this.invokeExceptionLogger.Debug(ErrorCode.GrainInvokeException,
"Exception during Grain method call of message: " + message + ": " + LogFormatter.PrintException(exc1), exc1);
}
if (transactionInfo != null)
{
transactionInfo.ReconcilePending();
// Record reason for abort, if not alread set
transactionInfo.RecordException(exc1, serializationManager);
if (startNewTransaction)
{
exc1 = transactionInfo.MustAbort(serializationManager);
await this.transactionAgent.Abort(transactionInfo);
TransactionContext.Clear();
}
}
// If a grain allowed an inconsistent state exception to escape and the exception originated from
// this activation, then deactivate it.
var ise = exc1 as InconsistentStateException;
if (ise != null && ise.IsSourceActivation)
{
// Mark the exception so that it doesn't deactivate any other activations.
ise.IsSourceActivation = false;
var activation = (target as Grain)?.Data;
if (activation != null)
{
this.invokeExceptionLogger.Info($"Deactivating {activation} due to inconsistent state.");
this.DeactivateOnIdle(activation.ActivationId);
}
}
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc1);
}
return;
}
OrleansTransactionException transactionException = null;
if (transactionInfo != null)
{
try
{
transactionInfo.ReconcilePending();
transactionException = transactionInfo.MustAbort(serializationManager);
// This request started the transaction, so we try to commit before returning,
// or if it must abort, tell participants that it aborted
if (startNewTransaction)
{
try
{
if (transactionException == null)
{
var status = await this.transactionAgent.Resolve(transactionInfo);
if (status != TransactionalStatus.Ok)
{
transactionException = status.ConvertToUserException(transactionInfo.Id);
}
}
else
{
await this.transactionAgent.Abort(transactionInfo);
}
}
finally
{
TransactionContext.Clear();
}
}
}
catch (Exception e)
{
// we should never hit this, but if we do, the following message will help us diagnose
this.logger.LogError(e, "Error in transaction post-grain-method-invocation code");
throw;
}
}
if (message.Direction != Message.Directions.OneWay)
{
if (transactionException != null)
{
SafeSendExceptionResponse(message, transactionException);
}
else
{
SafeSendResponse(message, resultObject);
}
}
return;
}
catch (Exception exc2)
{
this.logger.Warn(ErrorCode.Runtime_Error_100329, "Exception during Invoke of message: " + message, exc2);
TransactionContext.Clear();
if (message.Direction != Message.Directions.OneWay)
{
SafeSendExceptionResponse(message, exc2);
}
}
finally
{
TransactionContext.Clear();
}
}
public Silo(ILocalSiloDetails siloDetails, IServiceProvider services)
{
string name = siloDetails.Name;
// Temporarily still require this. Hopefuly gone when 2.0 is released.
this.siloDetails = siloDetails;
this.SystemStatus = SystemStatus.Creating;
var startTime = DateTime.UtcNow;
IOptions <ClusterMembershipOptions> clusterMembershipOptions = services.GetRequiredService <IOptions <ClusterMembershipOptions> >();
initTimeout = clusterMembershipOptions.Value.MaxJoinAttemptTime;
if (Debugger.IsAttached)
{
initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), clusterMembershipOptions.Value.MaxJoinAttemptTime);
stopTimeout = initTimeout;
}
var localEndpoint = this.siloDetails.SiloAddress.Endpoint;
this.Services = services;
//set PropagateActivityId flag from node config
IOptions <SiloMessagingOptions> messagingOptions = services.GetRequiredService <IOptions <SiloMessagingOptions> >();
RequestContext.PropagateActivityId = messagingOptions.Value.PropagateActivityId;
this.waitForMessageToBeQueuedForOutbound = messagingOptions.Value.WaitForMessageToBeQueuedForOutboundTime;
this.loggerFactory = this.Services.GetRequiredService <ILoggerFactory>();
logger = this.loggerFactory.CreateLogger <Silo>();
logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode));
if (!GCSettings.IsServerGC)
{
logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\">");
logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines).");
}
if (logger.IsEnabled(LogLevel.Debug))
{
var highestLogLevel = logger.IsEnabled(LogLevel.Trace) ? nameof(LogLevel.Trace) : nameof(LogLevel.Debug);
logger.LogWarning(
new EventId((int)ErrorCode.SiloGcWarning),
$"A verbose logging level ({highestLogLevel}) is configured. This will impact performance. The recommended log level is {nameof(LogLevel.Information)}.");
}
logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing silo on host {0} MachineName {1} at {2}, gen {3} --------------",
this.siloDetails.DnsHostName, Environment.MachineName, localEndpoint, this.siloDetails.SiloAddress.Generation);
logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0}", name);
try
{
grainFactory = Services.GetRequiredService <GrainFactory>();
}
catch (InvalidOperationException exc)
{
logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc);
throw;
}
// Performance metrics
siloStatistics = Services.GetRequiredService <SiloStatisticsManager>();
runtimeClient = Services.GetRequiredService <InsideRuntimeClient>();
// Initialize the message center
messageCenter = Services.GetRequiredService <MessageCenter>();
messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage;
// Now the router/directory service
// This has to come after the message center //; note that it then gets injected back into the message center.;
localGrainDirectory = Services.GetRequiredService <LocalGrainDirectory>();
// Now the consistent ring provider
RingProvider = Services.GetRequiredService <IConsistentRingProvider>();
catalog = Services.GetRequiredService <Catalog>();
siloStatusOracle = Services.GetRequiredService <ISiloStatusOracle>();
this.membershipService = Services.GetRequiredService <IMembershipService>();
this.SystemStatus = SystemStatus.Created;
StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
() => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.
this.siloLifecycle = this.Services.GetRequiredService <ISiloLifecycleSubject>();
// register all lifecycle participants
IEnumerable <ILifecycleParticipant <ISiloLifecycle> > lifecycleParticipants = this.Services.GetServices <ILifecycleParticipant <ISiloLifecycle> >();
foreach (ILifecycleParticipant <ISiloLifecycle> participant in lifecycleParticipants)
{
participant?.Participate(this.siloLifecycle);
}
// register all named lifecycle participants
IKeyedServiceCollection <string, ILifecycleParticipant <ISiloLifecycle> > namedLifecycleParticipantCollection = this.Services.GetService <IKeyedServiceCollection <string, ILifecycleParticipant <ISiloLifecycle> > >();
foreach (ILifecycleParticipant <ISiloLifecycle> participant in namedLifecycleParticipantCollection
?.GetServices(this.Services)
?.Select(s => s.GetService(this.Services)))
{
participant?.Participate(this.siloLifecycle);
}
// add self to lifecycle
this.Participate(this.siloLifecycle);
logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
}
public Silo(ILocalSiloDetails siloDetails, IServiceProvider services)
{
string name = siloDetails.Name;
// Temporarily still require this. Hopefuly gone when 2.0 is released.
this.legacyConfiguration = services.GetRequiredService <LegacyConfigurationWrapper>();
this.siloDetails = siloDetails;
this.SystemStatus = SystemStatus.Creating;
AsynchAgent.IsStarting = true;
var startTime = DateTime.UtcNow;
StatisticsCollector.Initialize(LocalConfig.StatisticsCollectionLevel);
initTimeout = GlobalConfig.MaxJoinAttemptTime;
if (Debugger.IsAttached)
{
initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), GlobalConfig.MaxJoinAttemptTime);
stopTimeout = initTimeout;
}
var localEndpoint = this.siloDetails.SiloAddress.Endpoint;
services.GetService <SerializationManager>().RegisterSerializers(services.GetService <IApplicationPartManager>());
this.Services = services;
this.Services.InitializeSiloUnobservedExceptionsHandler();
//set PropagateActivityId flag from node cofnig
RequestContext.PropagateActivityId = this.legacyConfiguration.NodeConfig.PropagateActivityId;
this.loggerFactory = this.Services.GetRequiredService <ILoggerFactory>();
logger = this.loggerFactory.CreateLogger <Silo>();
logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode));
if (!GCSettings.IsServerGC)
{
logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\">");
logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines).");
}
logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------",
this.legacyConfiguration.Type, this.siloDetails.DnsHostName, Environment.MachineName, localEndpoint, this.siloDetails.SiloAddress.Generation);
logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with the following configuration= " + Environment.NewLine + "{1}",
name, this.legacyConfiguration.ClusterConfig.ToString(name));
var siloMessagingOptions = this.Services.GetRequiredService <IOptions <SiloMessagingOptions> >();
BufferPool.InitGlobalBufferPool(siloMessagingOptions);
try
{
grainFactory = Services.GetRequiredService <GrainFactory>();
}
catch (InvalidOperationException exc)
{
logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc);
throw;
}
// Performance metrics
siloStatistics = Services.GetRequiredService <SiloStatisticsManager>();
// The scheduler
scheduler = Services.GetRequiredService <OrleansTaskScheduler>();
healthCheckParticipants.Add(scheduler);
runtimeClient = Services.GetRequiredService <InsideRuntimeClient>();
// Initialize the message center
messageCenter = Services.GetRequiredService <MessageCenter>();
var dispatcher = this.Services.GetRequiredService <Dispatcher>();
messageCenter.RerouteHandler = dispatcher.RerouteMessage;
messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage;
// GrainRuntime can be created only here, after messageCenter was created.
this.grainRuntime = Services.GetRequiredService <IGrainRuntime>();
// Now the router/directory service
// This has to come after the message center //; note that it then gets injected back into the message center.;
localGrainDirectory = Services.GetRequiredService <LocalGrainDirectory>();
// Now the activation directory.
activationDirectory = Services.GetRequiredService <ActivationDirectory>();
// Now the consistent ring provider
RingProvider = Services.GetRequiredService <IConsistentRingProvider>();
catalog = Services.GetRequiredService <Catalog>();
siloStatistics.MetricsTable.Scheduler = scheduler;
siloStatistics.MetricsTable.ActivationDirectory = activationDirectory;
siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector;
siloStatistics.MetricsTable.MessageCenter = messageCenter;
executorService = Services.GetRequiredService <ExecutorService>();
// Now the incoming message agents
var messageFactory = this.Services.GetRequiredService <MessageFactory>();
incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory);
incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory);
incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, executorService, this.loggerFactory);
membershipOracle = Services.GetRequiredService <IMembershipOracle>();
this.siloOptions = Services.GetRequiredService <IOptions <SiloOptions> >().Value;
var multiClusterOptions = Services.GetRequiredService <IOptions <MultiClusterOptions> >().Value;
if (!multiClusterOptions.HasMultiClusterNetwork)
{
logger.Info("Skip multicluster oracle creation (no multicluster network configured)");
}
else
{
multiClusterOracle = Services.GetRequiredService <IMultiClusterOracle>();
}
this.SystemStatus = SystemStatus.Created;
AsynchAgent.IsStarting = false;
StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
() => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.
var fullSiloLifecycle = this.Services.GetRequiredService <SiloLifecycle>();
this.siloLifecycle = fullSiloLifecycle;
// register all lifecycle participants
IEnumerable <ILifecycleParticipant <ISiloLifecycle> > lifecycleParticipants = this.Services.GetServices <ILifecycleParticipant <ISiloLifecycle> >();
foreach (ILifecycleParticipant <ISiloLifecycle> participant in lifecycleParticipants)
{
participant?.Participate(fullSiloLifecycle);
}
// register all named lifecycle participants
IKeyedServiceCollection <string, ILifecycleParticipant <ISiloLifecycle> > namedLifecycleParticipantCollection = this.Services.GetService <IKeyedServiceCollection <string, ILifecycleParticipant <ISiloLifecycle> > >();
foreach (ILifecycleParticipant <ISiloLifecycle> participant in namedLifecycleParticipantCollection
?.GetServices(this.Services)
?.Select(s => s.GetService(this.Services)))
{
participant?.Participate(fullSiloLifecycle);
}
// add self to lifecycle
this.Participate(fullSiloLifecycle);
logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
}
private void InternalUnobservedTaskExceptionHandler(object sender, UnobservedTaskExceptionEventArgs e)
{
var aggrException = e.Exception;
var baseException = aggrException.GetBaseException();
var tplTask = (Task)sender;
var contextObj = tplTask.AsyncState;
var context = contextObj as ISchedulingContext;
try
{
UnobservedExceptionHandler(context, baseException);
}
finally
{
if (e.Observed)
{
logger.Info(ErrorCode.Runtime_Error_100311, "Silo caught an UnobservedTaskException which was successfully observed and recovered from. BaseException = {0}. Exception = {1}",
baseException.Message, LogFormatter.PrintException(aggrException));
}
else
{
var errorStr = String.Format("Silo Caught an UnobservedTaskException event sent by {0}. Exception = {1}",
OrleansTaskExtentions.ToString((Task)sender), LogFormatter.PrintException(aggrException));
logger.Error(ErrorCode.Runtime_Error_100005, errorStr);
logger.Error(ErrorCode.Runtime_Error_100006, "Exception remained UnObserved!!! The subsequent behavior depends on the ThrowUnobservedTaskExceptions setting in app config and .NET version.");
}
}
}
internal Silo(SiloInitializationParameters initializationParams, IServiceProvider services)
{
string name = initializationParams.Name;
ClusterConfiguration config = initializationParams.ClusterConfig;
this.initializationParams = initializationParams;
this.SystemStatus = SystemStatus.Creating;
AsynchAgent.IsStarting = true;
var startTime = DateTime.UtcNow;
services?.GetService <TelemetryManager>()?.AddFromConfiguration(services, LocalConfig.TelemetryConfiguration);
StatisticsCollector.Initialize(LocalConfig);
initTimeout = GlobalConfig.MaxJoinAttemptTime;
if (Debugger.IsAttached)
{
initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), GlobalConfig.MaxJoinAttemptTime);
stopTimeout = initTimeout;
}
var localEndpoint = this.initializationParams.SiloAddress.Endpoint;
// Configure DI using Startup type
if (services == null)
{
var serviceCollection = new ServiceCollection();
serviceCollection.AddSingleton <Silo>(this);
serviceCollection.AddSingleton(initializationParams);
serviceCollection.AddLegacyClusterConfigurationSupport(config);
serviceCollection.Configure <SiloIdentityOptions>(options => options.SiloName = name);
var hostContext = new HostBuilderContext(new Dictionary <object, object>());
DefaultSiloServices.AddDefaultServices(hostContext, serviceCollection);
var applicationPartManager = hostContext.GetApplicationPartManager();
applicationPartManager.AddApplicationPartsFromAppDomain();
applicationPartManager.AddApplicationPartsFromBasePath();
services = StartupBuilder.ConfigureStartup(this.LocalConfig.StartupTypeName, serviceCollection);
services.GetService <TelemetryManager>()?.AddFromConfiguration(services, LocalConfig.TelemetryConfiguration);
}
this.Services = services;
this.Services.InitializeSiloUnobservedExceptionsHandler();
//set PropagateActivityId flag from node cofnig
RequestContext.PropagateActivityId = this.initializationParams.NodeConfig.PropagateActivityId;
this.loggerFactory = this.Services.GetRequiredService <ILoggerFactory>();
logger = this.loggerFactory.CreateLogger <Silo>();
logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode));
if (!GCSettings.IsServerGC)
{
logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\">");
logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines).");
}
logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------",
this.initializationParams.Type, LocalConfig.DNSHostName, Environment.MachineName, localEndpoint, this.initializationParams.SiloAddress.Generation);
logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with the following configuration= " + Environment.NewLine + "{1}",
name, config.ToString(name));
var siloMessagingOptions = this.Services.GetRequiredService <IOptions <SiloMessagingOptions> >();
BufferPool.InitGlobalBufferPool(siloMessagingOptions);
try
{
grainFactory = Services.GetRequiredService <GrainFactory>();
}
catch (InvalidOperationException exc)
{
logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc);
throw;
}
grainTypeManager = Services.GetRequiredService <GrainTypeManager>();
// Performance metrics
siloStatistics = Services.GetRequiredService <SiloStatisticsManager>();
// The scheduler
scheduler = Services.GetRequiredService <OrleansTaskScheduler>();
healthCheckParticipants.Add(scheduler);
runtimeClient = Services.GetRequiredService <InsideRuntimeClient>();
// Initialize the message center
messageCenter = Services.GetRequiredService <MessageCenter>();
var dispatcher = this.Services.GetRequiredService <Dispatcher>();
messageCenter.RerouteHandler = dispatcher.RerouteMessage;
messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage;
// GrainRuntime can be created only here, after messageCenter was created.
grainRuntime = Services.GetRequiredService <IGrainRuntime>();
StreamProviderManager = Services.GetRequiredService <IStreamProviderManager>();
// Now the router/directory service
// This has to come after the message center //; note that it then gets injected back into the message center.;
localGrainDirectory = Services.GetRequiredService <LocalGrainDirectory>();
// Now the activation directory.
activationDirectory = Services.GetRequiredService <ActivationDirectory>();
// Now the consistent ring provider
RingProvider = Services.GetRequiredService <IConsistentRingProvider>();
catalog = Services.GetRequiredService <Catalog>();
siloStatistics.MetricsTable.Scheduler = scheduler;
siloStatistics.MetricsTable.ActivationDirectory = activationDirectory;
siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector;
siloStatistics.MetricsTable.MessageCenter = messageCenter;
// Now the incoming message agents
var messageFactory = this.Services.GetRequiredService <MessageFactory>();
incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, this.loggerFactory);
incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, this.loggerFactory);
incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, catalog.Dispatcher, messageFactory, this.loggerFactory);
membershipOracle = Services.GetRequiredService <IMembershipOracle>();
if (!this.GlobalConfig.HasMultiClusterNetwork)
{
logger.Info("Skip multicluster oracle creation (no multicluster network configured)");
}
else
{
multiClusterOracle = Services.GetRequiredService <IMultiClusterOracle>();
}
this.SystemStatus = SystemStatus.Created;
AsynchAgent.IsStarting = false;
StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
() => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.
logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
}
private void InternalUnobservedTaskExceptionHandler(object sender, System.Threading.Tasks.UnobservedTaskExceptionEventArgs e)
{
var aggrException = e.Exception;
var baseException = aggrException.GetBaseException();
var context = (sender as Task)?.AsyncState;
try
{
this.logger.LogError((int)ErrorCode.Runtime_Error_100104, baseException, "Silo caught an unobserved exception thrown from context {Context}: {Exception}", context, baseException);
}
finally
{
if (e.Observed)
{
logger.Info(ErrorCode.Runtime_Error_100311, "Silo caught an unobserved exception which was successfully observed and recovered from. BaseException = {0}. Exception = {1}",
baseException.Message, LogFormatter.PrintException(aggrException));
}
else
{
var errorStr = String.Format("Silo Caught an UnobservedTaskException event sent by {0}. Exception = {1}",
OrleansTaskExtentions.ToString((Task)sender), LogFormatter.PrintException(aggrException));
logger.Error(ErrorCode.Runtime_Error_100005, errorStr);
logger.Error(ErrorCode.Runtime_Error_100006, "Exception remained UnObserved!!! The subsequent behavior depends on the ThrowUnobservedTaskExceptions setting in app config and .NET version.");
}
}
}
internal Silo(string name, SiloType siloType, ClusterConfiguration config, ILocalDataStore keyStore)
{
SystemStatus.Current = SystemStatus.Creating;
CurrentSilo = this;
var startTime = DateTime.UtcNow;
this.siloType = siloType;
Name = name;
siloTerminatedEvent = new ManualResetEvent(false);
OrleansConfig = config;
globalConfig = config.Globals;
config.OnConfigChange("Defaults", () => nodeConfig = config.GetOrCreateNodeConfigurationForSilo(name));
if (!LogManager.IsInitialized)
{
LogManager.Initialize(nodeConfig);
}
config.OnConfigChange("Defaults/Tracing", () => LogManager.Initialize(nodeConfig, true), false);
MultiClusterRegistrationStrategy.Initialize(config.Globals);
ActivationData.Init(config, nodeConfig);
StatisticsCollector.Initialize(nodeConfig);
SerializationManager.Initialize(globalConfig.SerializationProviders, this.globalConfig.FallbackSerializationProvider);
initTimeout = globalConfig.MaxJoinAttemptTime;
if (Debugger.IsAttached)
{
initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), globalConfig.MaxJoinAttemptTime);
stopTimeout = initTimeout;
}
IPEndPoint here = nodeConfig.Endpoint;
int generation = nodeConfig.Generation;
if (generation == 0)
{
generation = SiloAddress.AllocateNewGeneration();
nodeConfig.Generation = generation;
}
LogManager.MyIPEndPoint = here;
logger = LogManager.GetLogger("Silo", LoggerType.Runtime);
logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode));
if (!GCSettings.IsServerGC || !GCSettings.LatencyMode.Equals(GCLatencyMode.Batch))
{
logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on or with GCLatencyMode.Batch enabled - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\"> and <configuration>-<runtime>-<gcConcurrent enabled=\"false\"/>");
logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines).");
}
logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------",
siloType, nodeConfig.DNSHostName, Environment.MachineName, here, generation);
logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with the following configuration= " + Environment.NewLine + "{1}",
name, config.ToString(name));
if (keyStore != null)
{
// Re-establish reference to shared local key store in this app domain
LocalDataStoreInstance.LocalDataStore = keyStore;
}
// Configure DI using Startup type
bool usingCustomServiceProvider;
Services = StartupBuilder.ConfigureStartup(nodeConfig.StartupTypeName, out usingCustomServiceProvider);
healthCheckParticipants = new List <IHealthCheckParticipant>();
allSiloProviders = new List <IProvider>();
BufferPool.InitGlobalBufferPool(globalConfig);
PlacementStrategy.Initialize(globalConfig);
UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnobservedExceptionHandler);
AppDomain.CurrentDomain.UnhandledException +=
(obj, ev) => DomainUnobservedExceptionHandler(obj, (Exception)ev.ExceptionObject);
try
{
grainFactory = Services.GetRequiredService <GrainFactory>();
}
catch (InvalidOperationException exc)
{
logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc);
throw;
}
typeManager = new GrainTypeManager(
here.Address.Equals(IPAddress.Loopback),
grainFactory,
new SiloAssemblyLoader(OrleansConfig.Defaults.AdditionalAssemblyDirectories));
// Performance metrics
siloStatistics = new SiloStatisticsManager(globalConfig, nodeConfig);
config.OnConfigChange("Defaults/LoadShedding", () => siloStatistics.MetricsTable.NodeConfig = nodeConfig, false);
// The scheduler
scheduler = new OrleansTaskScheduler(globalConfig, nodeConfig);
healthCheckParticipants.Add(scheduler);
// Initialize the message center
var mc = new MessageCenter(here, generation, globalConfig, siloStatistics.MetricsTable);
if (nodeConfig.IsGatewayNode)
{
mc.InstallGateway(nodeConfig.ProxyGatewayEndpoint);
}
messageCenter = mc;
SiloIdentity = SiloAddress.ToLongString();
// GrainRuntime can be created only here, after messageCenter was created.
grainRuntime = new GrainRuntime(
globalConfig.ServiceId,
SiloIdentity,
grainFactory,
new TimerRegistry(),
new ReminderRegistry(),
new StreamProviderManager(),
Services);
// Now the router/directory service
// This has to come after the message center //; note that it then gets injected back into the message center.;
localGrainDirectory = new LocalGrainDirectory(this);
RegistrarManager.InitializeGrainDirectoryManager(localGrainDirectory, globalConfig.GlobalSingleInstanceNumberRetries);
// Now the activation directory.
// This needs to know which router to use so that it can keep the global directory in synch with the local one.
activationDirectory = new ActivationDirectory();
// Now the consistent ring provider
RingProvider = GlobalConfig.UseVirtualBucketsConsistentRing ?
(IConsistentRingProvider) new VirtualBucketsRingProvider(SiloAddress, GlobalConfig.NumVirtualBucketsConsistentRing)
: new ConsistentRingProvider(SiloAddress);
// to preserve backwards compatibility, only use the service provider to inject grain dependencies if the user supplied his own
// service provider, meaning that he is explicitly opting into it.
var grainCreator = new GrainCreator(grainRuntime, usingCustomServiceProvider ? Services : null);
Action <Dispatcher> setDispatcher;
catalog = new Catalog(Constants.CatalogId, SiloAddress, Name, LocalGrainDirectory, typeManager, scheduler, activationDirectory, config, grainCreator, out setDispatcher);
var dispatcher = new Dispatcher(scheduler, messageCenter, catalog, config);
setDispatcher(dispatcher);
RuntimeClient.Current = new InsideRuntimeClient(
dispatcher,
catalog,
LocalGrainDirectory,
SiloAddress,
config,
RingProvider,
typeManager,
grainFactory);
messageCenter.RerouteHandler = InsideRuntimeClient.Current.RerouteMessage;
messageCenter.SniffIncomingMessage = InsideRuntimeClient.Current.SniffIncomingMessage;
siloStatistics.MetricsTable.Scheduler = scheduler;
siloStatistics.MetricsTable.ActivationDirectory = activationDirectory;
siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector;
siloStatistics.MetricsTable.MessageCenter = messageCenter;
DeploymentLoadPublisher.CreateDeploymentLoadPublisher(this, globalConfig);
PlacementDirectorsManager.CreatePlacementDirectorsManager(globalConfig);
// Now the incoming message agents
incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, dispatcher);
incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, dispatcher);
incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, dispatcher);
membershipFactory = new MembershipFactory();
multiClusterFactory = new MultiClusterOracleFactory();
reminderFactory = new LocalReminderServiceFactory();
SystemStatus.Current = SystemStatus.Created;
StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME,
() => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs.
logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode());
}
