private AzureTableBasedGossipChannel gossipTable; // This type is internal public AzureGossipTableTests() { logger = LogManager.GetLogger("AzureGossipTableTests", LoggerType.Application); globalServiceId = Guid.NewGuid(); deploymentId = "test-" + globalServiceId; IPAddress ip; if (!IPAddress.TryParse("127.0.0.1", out ip)) { logger.Error(-1, "Could not parse ip address"); return; } IPEndPoint ep1 = new IPEndPoint(ip, 21111); siloAddress1 = SiloAddress.New(ep1, 0); IPEndPoint ep2 = new IPEndPoint(ip, 21112); siloAddress2 = SiloAddress.New(ep2, 0); logger.Info("DeploymentId={0}", deploymentId); GlobalConfiguration config = new GlobalConfiguration { ServiceId = globalServiceId, ClusterId = "0", DeploymentId = deploymentId, DataConnectionString = TestDefaultConfiguration.DataConnectionString }; gossipTable = new AzureTableBasedGossipChannel(); var done = gossipTable.Initialize(config.ServiceId, config.DataConnectionString); if (!done.Wait(timeout)) { throw new TimeoutException("Could not create/read table."); } }
/// <summary> /// Adds CancellationToken to the grain extension /// so that it can be cancelled through remote call to the CancellationSourcesExtension. /// </summary> /// <param name="target"></param> /// <param name="request"></param> /// <param name="logger"></param> internal static void RegisterCancellationTokens(IAddressable target, InvokeMethodRequest request, Logger logger) { for (var i = 0; i < request.Arguments.Length; i++) { var arg = request.Arguments[i]; if (!(arg is GrainCancellationToken)) continue; var grainToken = ((GrainCancellationToken) request.Arguments[i]); CancellationSourcesExtension cancellationExtension; if (!SiloProviderRuntime.Instance.TryGetExtensionHandler(out cancellationExtension)) { cancellationExtension = new CancellationSourcesExtension(); if (!SiloProviderRuntime.Instance.TryAddExtension(cancellationExtension)) { logger.Error( ErrorCode.CancellationExtensionCreationFailed, $"Could not add cancellation token extension to: {target}"); return; } } // Replacing the half baked GrainCancellationToken that came from the wire with locally fully created one. request.Arguments[i] = cancellationExtension.RecordCancellationToken(grainToken.Id, grainToken.IsCancellationRequested); } }
public static void LogError(Orleans.Runtime.Logger orleansLog, string type, string method, string format, params object[] arguments) { var message = string.Format(format, arguments); var threadId = Thread.CurrentThread.ManagedThreadId; message = string.Format("[{0}] {1}::{2}: {3}", threadId, type, method, message); //Trace.TraceInformation(message); // Debug.WriteLine(message); //Console.WriteLine(message); orleansLog.Error(0, message); }
/// <summary> Initialization function for this storage provider. </summary> /// <see cref="IProvider#Init"/> public async Task Init(string name, IProviderRuntime providerRuntime, IProviderConfiguration config) { serviceId = providerRuntime.ServiceId.ToString(); Log = providerRuntime.GetLogger("StorageProvider.SimpleSQLServerStorage." + serviceId); try { Name = name; this.jsonSettings = OrleansJsonSerializer.UpdateSerializerSettings(OrleansJsonSerializer.GetDefaultSerializerSettings(), config); if (!config.Properties.ContainsKey(CONNECTION_STRING) || string.IsNullOrWhiteSpace(config.Properties[CONNECTION_STRING])) { throw new BadProviderConfigException($"Specify a value for: {CONNECTION_STRING}"); } var connectionString = config.Properties[CONNECTION_STRING]; sqlconnBuilder = new SqlConnectionStringBuilder(connectionString); //a validation of the connection would be wise to perform here var sqlCon = new SqlConnection(sqlconnBuilder.ConnectionString); await sqlCon.OpenAsync(); sqlCon.Close(); //initialize to use the default of JSON storage (this is to provide backwards compatiblity with previous version useJsonOrBinaryFormat = StorageFormatEnum.Binary; if (config.Properties.ContainsKey(USE_JSON_FORMAT_PROPERTY)) { if ("true".Equals(config.Properties[USE_JSON_FORMAT_PROPERTY], StringComparison.OrdinalIgnoreCase)) useJsonOrBinaryFormat = StorageFormatEnum.Json; if ("both".Equals(config.Properties[USE_JSON_FORMAT_PROPERTY], StringComparison.OrdinalIgnoreCase)) useJsonOrBinaryFormat = StorageFormatEnum.Both; } } catch (Exception ex) { Log.Error((int) SimpleSQLServerProviderErrorCodes.SimpleSQLServerProvider_InitProvider, ex.ToString(), ex); throw; } }
public GatewayManager(ClientConfiguration cfg, IGatewayListProvider gatewayListProvider) { config = cfg; knownDead = new Dictionary<Uri, DateTime>(); rand = new SafeRandom(); logger = LogManager.GetLogger("Messaging.GatewayManager", LoggerType.Runtime); lockable = new object(); gatewayRefreshCallInitiated = false; ListProvider = gatewayListProvider; var knownGateways = ListProvider.GetGateways().GetResult(); if (knownGateways.Count == 0) { string gatewayProviderType = gatewayListProvider.GetType().FullName; string err = String.Format("Could not find any gateway in {0}. Orleans client cannot initialize.", gatewayProviderType); logger.Error(ErrorCode.GatewayManager_NoGateways, err); throw new OrleansException(err); } logger.Info(ErrorCode.GatewayManager_FoundKnownGateways, "Found {0} knownGateways from Gateway listProvider {1}", knownGateways.Count, Utils.EnumerableToString(knownGateways)); if (ListProvider is IGatewayListObservable) { ((IGatewayListObservable)ListProvider).SubscribeToGatewayNotificationEvents(this); } roundRobinCounter = cfg.PreferedGatewayIndex >= 0 ? cfg.PreferedGatewayIndex : rand.Next(knownGateways.Count); cachedLiveGateways = knownGateways; lastRefreshTime = DateTime.UtcNow; if (ListProvider.IsUpdatable) { gatewayRefreshTimer = new SafeTimer(RefreshSnapshotLiveGateways_TimerCallback, null, config.GatewayListRefreshPeriod, config.GatewayListRefreshPeriod); } }
public OutsideRuntimeClient(ClientConfiguration cfg, GrainFactory grainFactory, bool secondary = false) { this.grainFactory = grainFactory; if (cfg == null) { Console.WriteLine("An attempt to create an OutsideRuntimeClient with null ClientConfiguration object."); throw new ArgumentException("OutsideRuntimeClient was attempted to be created with null ClientConfiguration object.", "cfg"); } this.config = cfg; if (!LogManager.IsInitialized) LogManager.Initialize(config); StatisticsCollector.Initialize(config); SerializationManager.Initialize(config.UseStandardSerializer, cfg.SerializationProviders, config.UseJsonFallbackSerializer); logger = LogManager.GetLogger("OutsideRuntimeClient", LoggerType.Runtime); appLogger = LogManager.GetLogger("Application", LoggerType.Application); BufferPool.InitGlobalBufferPool(config); this.handshakeClientId = GrainId.NewClientId(); try { LoadAdditionalAssemblies(); PlacementStrategy.Initialize(); callbacks = new ConcurrentDictionary<CorrelationId, CallbackData>(); localObjects = new ConcurrentDictionary<GuidId, LocalObjectData>(); if (!secondary) { UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnhandledException); } AppDomain.CurrentDomain.DomainUnload += CurrentDomain_DomainUnload; // Ensure SerializationManager static constructor is called before AssemblyLoad event is invoked SerializationManager.GetDeserializer(typeof(String)); clientProviderRuntime = new ClientProviderRuntime(grainFactory, null); statisticsProviderManager = new StatisticsProviderManager("Statistics", clientProviderRuntime); var statsProviderName = statisticsProviderManager.LoadProvider(config.ProviderConfigurations) .WaitForResultWithThrow(initTimeout); if (statsProviderName != null) { config.StatisticsProviderName = statsProviderName; } responseTimeout = Debugger.IsAttached ? Constants.DEFAULT_RESPONSE_TIMEOUT : config.ResponseTimeout; var localAddress = ClusterConfiguration.GetLocalIPAddress(config.PreferredFamily, config.NetInterface); // Client init / sign-on message logger.Info(ErrorCode.ClientInitializing, string.Format( "{0} Initializing OutsideRuntimeClient on {1} at {2} Client Id = {3} {0}", BARS, config.DNSHostName, localAddress, handshakeClientId)); string startMsg = string.Format("{0} Starting OutsideRuntimeClient with runtime Version='{1}' in AppDomain={2}", BARS, RuntimeVersion.Current, PrintAppDomainDetails()); startMsg = string.Format("{0} Config= " + Environment.NewLine + " {1}", startMsg, config); logger.Info(ErrorCode.ClientStarting, startMsg); if (TestOnlyThrowExceptionDuringInit) { throw new InvalidOperationException("TestOnlyThrowExceptionDuringInit"); } config.CheckGatewayProviderSettings(); var generation = -SiloAddress.AllocateNewGeneration(); // Client generations are negative var gatewayListProvider = GatewayProviderFactory.CreateGatewayListProvider(config) .WithTimeout(initTimeout).Result; transport = new ProxiedMessageCenter(config, localAddress, generation, handshakeClientId, gatewayListProvider); if (StatisticsCollector.CollectThreadTimeTrackingStats) { incomingMessagesThreadTimeTracking = new ThreadTrackingStatistic("ClientReceiver"); } } catch (Exception exc) { if (logger != null) logger.Error(ErrorCode.Runtime_Error_100319, "OutsideRuntimeClient constructor failed.", exc); ConstructorReset(); throw; } }
internal Silo(SiloInitializationParameters initializationParams) { string name = initializationParams.Name; ClusterConfiguration config = initializationParams.ClusterConfig; this.initializationParams = initializationParams; SystemStatus.Current = SystemStatus.Creating; CurrentSilo = this; var startTime = DateTime.UtcNow; siloTerminatedEvent = new ManualResetEvent(false); if (!LogManager.IsInitialized) LogManager.Initialize(LocalConfig); config.OnConfigChange("Defaults/Tracing", () => LogManager.Initialize(LocalConfig, true), false); MultiClusterRegistrationStrategy.Initialize(config.Globals); StatisticsCollector.Initialize(LocalConfig); SerializationManager.Initialize(GlobalConfig.SerializationProviders, this.GlobalConfig.FallbackSerializationProvider); initTimeout = GlobalConfig.MaxJoinAttemptTime; if (Debugger.IsAttached) { initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), GlobalConfig.MaxJoinAttemptTime); stopTimeout = initTimeout; } var localEndpoint = this.initializationParams.SiloAddress.Endpoint; LogManager.MyIPEndPoint = localEndpoint; logger = LogManager.GetLogger("Silo", LoggerType.Runtime); logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode)); if (!GCSettings.IsServerGC || !GCSettings.LatencyMode.Equals(GCLatencyMode.Batch)) { logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on or with GCLatencyMode.Batch enabled - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\"> and <configuration>-<runtime>-<gcConcurrent enabled=\"false\"/>"); logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines)."); } logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------", this.initializationParams.Type, LocalConfig.DNSHostName, Environment.MachineName, localEndpoint, this.initializationParams.SiloAddress.Generation); logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with the following configuration= " + Environment.NewLine + "{1}", name, config.ToString(name)); // Configure DI using Startup type this.Services = StartupBuilder.ConfigureStartup( this.LocalConfig.StartupTypeName, (services, usingCustomServiceProvider) => { // add system types // Note: you can replace IGrainFactory with your own implementation, but // we don't recommend it, in the aspect of performance and usability services.TryAddSingleton(sp => sp); services.TryAddSingleton(this); services.TryAddSingleton(initializationParams); services.TryAddSingleton<ILocalSiloDetails>(initializationParams); services.TryAddSingleton(initializationParams.ClusterConfig); services.TryAddSingleton(initializationParams.GlobalConfig); services.TryAddTransient(sp => initializationParams.NodeConfig); services.TryAddSingleton<ITimerRegistry, TimerRegistry>(); services.TryAddSingleton<IReminderRegistry, ReminderRegistry>(); services.TryAddSingleton<IStreamProviderManager, StreamProviderManager>(); services.TryAddSingleton<GrainRuntime>(); services.TryAddSingleton<IGrainRuntime, GrainRuntime>(); services.TryAddSingleton<OrleansTaskScheduler>(); services.TryAddSingleton<GrainFactory>(sp => sp.GetService<InsideRuntimeClient>().ConcreteGrainFactory); services.TryAddExisting<IGrainFactory, GrainFactory>(); services.TryAddExisting<IInternalGrainFactory, GrainFactory>(); services.TryAddSingleton<TypeMetadataCache>(); services.TryAddSingleton<AssemblyProcessor>(); services.TryAddSingleton<ActivationDirectory>(); services.TryAddSingleton<LocalGrainDirectory>(); services.TryAddExisting<ILocalGrainDirectory, LocalGrainDirectory>(); services.TryAddSingleton<SiloStatisticsManager>(); services.TryAddSingleton<ISiloPerformanceMetrics>(sp => sp.GetRequiredService<SiloStatisticsManager>().MetricsTable); services.TryAddSingleton<SiloAssemblyLoader>(); services.TryAddSingleton<GrainTypeManager>(); services.TryAddExisting<IMessagingConfiguration, GlobalConfiguration>(); services.TryAddSingleton<MessageCenter>(); services.TryAddExisting<IMessageCenter, MessageCenter>(); services.TryAddExisting<ISiloMessageCenter, MessageCenter>(); services.TryAddSingleton<Catalog>(); services.TryAddSingleton(sp => sp.GetRequiredService<Catalog>().Dispatcher); services.TryAddSingleton<InsideRuntimeClient>(); services.TryAddExisting<IRuntimeClient, InsideRuntimeClient>(); services.TryAddExisting<ISiloRuntimeClient, InsideRuntimeClient>(); services.TryAddSingleton<MembershipFactory>(); services.TryAddSingleton<MultiClusterOracleFactory>(); services.TryAddSingleton<LocalReminderServiceFactory>(); services.TryAddSingleton<DeploymentLoadPublisher>(); services.TryAddSingleton<IMembershipTable>( sp => sp.GetRequiredService<MembershipFactory>().GetMembershipTable(sp.GetRequiredService<GlobalConfiguration>())); services.TryAddSingleton<MembershipOracle>( sp => sp.GetRequiredService<MembershipFactory>() .CreateMembershipOracle(sp.GetRequiredService<Silo>(), sp.GetRequiredService<IMembershipTable>())); services.TryAddExisting<IMembershipOracle, MembershipOracle>(); services.TryAddExisting<ISiloStatusOracle, MembershipOracle>(); services.TryAddSingleton<Func<ISiloStatusOracle>>(sp => () => sp.GetRequiredService<ISiloStatusOracle>()); services.TryAddSingleton<MultiClusterOracleFactory>(); services.TryAddSingleton<LocalReminderServiceFactory>(); services.TryAddSingleton<ClientObserverRegistrar>(); services.TryAddSingleton<SiloProviderRuntime>(); services.TryAddExisting<IStreamProviderRuntime, SiloProviderRuntime>(); services.TryAddSingleton<ImplicitStreamSubscriberTable>(); // Placement services.TryAddSingleton<PlacementDirectorsManager>(); services.TryAddSingleton<IPlacementDirector<RandomPlacement>, RandomPlacementDirector>(); services.TryAddSingleton<IPlacementDirector<PreferLocalPlacement>, PreferLocalPlacementDirector>(); services.TryAddSingleton<IPlacementDirector<StatelessWorkerPlacement>, StatelessWorkerDirector>(); services.TryAddSingleton<IPlacementDirector<ActivationCountBasedPlacement>, ActivationCountPlacementDirector>(); services.TryAddSingleton<DefaultPlacementStrategy>(); services.TryAddSingleton<ClientObserversPlacementDirector>(); services.TryAddSingleton<Func<IGrainRuntime>>(sp => () => sp.GetRequiredService<IGrainRuntime>()); services.TryAddSingleton<GrainCreator>(); if (initializationParams.GlobalConfig.UseVirtualBucketsConsistentRing) { services.TryAddSingleton<IConsistentRingProvider>( sp => new VirtualBucketsRingProvider( this.initializationParams.SiloAddress, this.initializationParams.GlobalConfig.NumVirtualBucketsConsistentRing)); } else { services.TryAddSingleton<IConsistentRingProvider>( sp => new ConsistentRingProvider(this.initializationParams.SiloAddress)); } }); this.assemblyProcessor = this.Services.GetRequiredService<AssemblyProcessor>(); this.assemblyProcessor.Initialize(); BufferPool.InitGlobalBufferPool(GlobalConfig); UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnobservedExceptionHandler); AppDomain.CurrentDomain.UnhandledException += this.DomainUnobservedExceptionHandler; try { grainFactory = Services.GetRequiredService<GrainFactory>(); } catch (InvalidOperationException exc) { logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc); throw; } grainTypeManager = Services.GetRequiredService<GrainTypeManager>(); // Performance metrics siloStatistics = Services.GetRequiredService<SiloStatisticsManager>(); // The scheduler scheduler = Services.GetRequiredService<OrleansTaskScheduler>(); healthCheckParticipants.Add(scheduler); runtimeClient = Services.GetRequiredService<InsideRuntimeClient>(); // Initialize the message center messageCenter = Services.GetRequiredService<MessageCenter>(); messageCenter.RerouteHandler = runtimeClient.RerouteMessage; messageCenter.SniffIncomingMessage = runtimeClient.SniffIncomingMessage; // GrainRuntime can be created only here, after messageCenter was created. grainRuntime = Services.GetRequiredService<IGrainRuntime>(); // Now the router/directory service // This has to come after the message center //; note that it then gets injected back into the message center.; localGrainDirectory = Services.GetRequiredService<LocalGrainDirectory>(); // Now the activation directory. activationDirectory = Services.GetRequiredService<ActivationDirectory>(); // Now the consistent ring provider RingProvider = Services.GetRequiredService<IConsistentRingProvider>(); // to preserve backwards compatibility, only use the service provider to inject grain dependencies if the user supplied his own // service provider, meaning that he is explicitly opting into it. catalog = Services.GetRequiredService<Catalog>(); siloStatistics.MetricsTable.Scheduler = scheduler; siloStatistics.MetricsTable.ActivationDirectory = activationDirectory; siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector; siloStatistics.MetricsTable.MessageCenter = messageCenter; // Now the incoming message agents incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, catalog.Dispatcher); incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, catalog.Dispatcher); incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, catalog.Dispatcher); membershipFactory = Services.GetRequiredService<MembershipFactory>(); membershipOracle = Services.GetRequiredService<IMembershipOracle>(); SystemStatus.Current = SystemStatus.Created; StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME, () => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs. logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode()); }
public static bool CheckTimerDelay(DateTime previousTickTime, int totalNumTicks, TimeSpan dueTime, TimeSpan timerFrequency, Logger logger, Func<string> getName, ErrorCode errorCode, bool freezeCheck) { TimeSpan timeSinceLastTick = DateTime.UtcNow - previousTickTime; TimeSpan exceptedTimeToNexTick = totalNumTicks == 0 ? dueTime : timerFrequency; TimeSpan exceptedTimeWithSlack; if (exceptedTimeToNexTick >= TimeSpan.FromSeconds(6)) { exceptedTimeWithSlack = exceptedTimeToNexTick + TimeSpan.FromSeconds(3); } else { exceptedTimeWithSlack = exceptedTimeToNexTick.Multiply(1.5); } if (timeSinceLastTick <= exceptedTimeWithSlack) return true; // did not tick in the last period. var errMsg = String.Format("{0}{1} did not fire on time. Last fired at {2}, {3} since previous fire, should have fired after {4}.", freezeCheck ? "Watchdog Freeze Alert: " : "-", // 0 getName == null ? "" : getName(), // 1 LogFormatter.PrintDate(previousTickTime), // 2 timeSinceLastTick, // 3 exceptedTimeToNexTick); // 4 if(freezeCheck) { logger.Error(errorCode, errMsg); }else { logger.Warn(errorCode, errMsg); } return false; }
private async Task Update(IDictionary <string, ProviderCategoryConfiguration> streamProviderConfigurations) { ProviderCategoryConfiguration categoryConfig; streamProviderConfigurations.TryGetValue(ProviderCategoryConfiguration.STREAM_PROVIDER_CATEGORY_NAME, out categoryConfig); if (categoryConfig == null) { if (logger.IsVerbose) { logger.Verbose("streamProviderConfigurations does not contain '" + ProviderCategoryConfiguration.STREAM_PROVIDER_CATEGORY_NAME + "' element. Nothing to update."); } return; } IList <string> addList = new List <string>(); IList <string> removeList = new List <string>(); var siloStreamProviderManager = streamProviderManager; var existingProviders = siloStreamProviderManager.GetStreamProviders().Select(p => ((IProvider)p).Name).ToList(); var newProviderList = categoryConfig.Providers; foreach (var providerName in existingProviders) { if (!newProviderList.ContainsKey(providerName)) { removeList.Add(providerName); if (logger.IsVerbose) { logger.Verbose("Removing stream provider '" + providerName + "' from silo"); } } } foreach (var providerName in newProviderList.Keys) { if (!existingProviders.Contains(providerName)) { addList.Add(providerName); if (logger.IsVerbose) { logger.Verbose("Adding stream provider '" + providerName + "' to silo"); } } } try { // Removing providers from silo first await siloStreamProviderManager.RemoveProviders(removeList); // Adding new providers to silo await siloStreamProviderManager.LoadStreamProviders(streamProviderConfigurations, this.streamProviderRuntime); // Starting new providers await siloStreamProviderManager.StartStreamProviders(addList); } catch (ProviderStartException exc) { logger.Error(ErrorCode.Provider_ErrorFromInit, exc.Message, exc); throw; } catch (ProviderInitializationException exc) { logger.Error(ErrorCode.Provider_ErrorFromInit, exc.Message, exc); throw; } if (logger.IsVerbose) { logger.Verbose("Stream providers updated successfully."); } providerConfigurations[ProviderCategoryConfiguration.STREAM_PROVIDER_CATEGORY_NAME] = streamProviderConfigurations[ProviderCategoryConfiguration.STREAM_PROVIDER_CATEGORY_NAME]; }
public override Task OnActivateAsync() { logger = GetLogger(); var startMe = new Task( () => { logger.Info("OnActivateAsync"); watcher = GrainFactory.GetGrain<IActivateDeactivateWatcherGrain>(0); Assert.IsFalse(doingActivate, "Not doing Activate"); Assert.IsFalse(doingDeactivate, "Not doing Deactivate"); doingActivate = true; }); // we want to use Task.ContinueWith with an async lambda, an explicitly typed variable is required to avoid // writing code that doesn't do what i think it is doing. Func<Task> asyncCont = async () => { logger.Info("Started-OnActivateAsync"); Assert.IsTrue(doingActivate, "Doing Activate"); Assert.IsFalse(doingDeactivate, "Not doing Deactivate"); try { logger.Info("Calling RecordActivateCall"); await watcher.RecordActivateCall(Data.ActivationId.ToString()); logger.Info("Returned from calling RecordActivateCall"); } catch (Exception exc) { var msg = "RecordActivateCall failed with error " + exc; logger.Error(0, msg); Assert.Fail(msg); } Assert.IsTrue(doingActivate, "Doing Activate"); Assert.IsFalse(doingDeactivate, "Not doing Deactivate"); await Task.Delay(TimeSpan.FromSeconds(1)); doingActivate = false; logger.Info("Finished-OnActivateAsync"); }; var awaitMe = startMe.ContinueWith(_ => asyncCont()).Unwrap(); startMe.Start(); return awaitMe; }
internal static bool AnalyzeReadException(Exception exc, int iteration, string tableName, Logger logger) { bool isLastErrorRetriable; var we = exc as WebException; if (we != null) { isLastErrorRetriable = true; var statusCode = we.Status; logger.Warn(ErrorCode.AzureTable_10, $"Intermediate issue reading Azure storage table {tableName}: HTTP status code={statusCode} Exception Type={exc.GetType().FullName} Message='{exc.Message}'", exc); } else { HttpStatusCode httpStatusCode; string restStatus; if (EvaluateException(exc, out httpStatusCode, out restStatus, true)) { if (StorageErrorCodeStrings.ResourceNotFound.Equals(restStatus)) { if (logger.IsVerbose) logger.Verbose(ErrorCode.AzureTable_DataNotFound, "DataNotFound reading Azure storage table {0}:{1} HTTP status code={2} REST status code={3} Exception={4}", tableName, iteration == 0 ? "" : (" Repeat=" + iteration), httpStatusCode, restStatus, exc); isLastErrorRetriable = false; } else { isLastErrorRetriable = IsRetriableHttpError(httpStatusCode, restStatus); logger.Warn(ErrorCode.AzureTable_11, $"Intermediate issue reading Azure storage table {tableName}:{(iteration == 0 ? "" : (" Repeat=" + iteration))} IsRetriable={isLastErrorRetriable} HTTP status code={httpStatusCode} REST status code={restStatus} Exception Type={exc.GetType().FullName} Message='{exc.Message}'", exc); } } else { logger.Error(ErrorCode.AzureTable_12, $"Unexpected issue reading Azure storage table {tableName}: Exception Type={exc.GetType().FullName} Message='{exc.Message}'", exc); isLastErrorRetriable = false; } } return isLastErrorRetriable; }
internal static CloudQueueClient GetCloudQueueClient( string storageConnectionString, IRetryPolicy retryPolicy, TimeSpan timeout, Logger logger) { try { var storageAccount = GetCloudStorageAccount(storageConnectionString); CloudQueueClient operationsClient = storageAccount.CreateCloudQueueClient(); operationsClient.DefaultRequestOptions.RetryPolicy = retryPolicy; operationsClient.DefaultRequestOptions.ServerTimeout = timeout; return operationsClient; } catch (Exception exc) { logger.Error(ErrorCode.AzureQueue_14, String.Format("Error creating GetCloudQueueOperationsClient."), exc); throw; } }
internal Silo(string name, SiloType siloType, ClusterConfiguration config, ILocalDataStore keyStore) { SystemStatus.Current = SystemStatus.Creating; CurrentSilo = this; var startTime = DateTime.UtcNow; this.siloType = siloType; Name = name; siloTerminatedEvent = new ManualResetEvent(false); OrleansConfig = config; globalConfig = config.Globals; config.OnConfigChange("Defaults", () => nodeConfig = config.GetOrCreateNodeConfigurationForSilo(name)); if (!LogManager.IsInitialized) { LogManager.Initialize(nodeConfig); } config.OnConfigChange("Defaults/Tracing", () => LogManager.Initialize(nodeConfig, true), false); MultiClusterRegistrationStrategy.Initialize(config.Globals); ActivationData.Init(config, nodeConfig); StatisticsCollector.Initialize(nodeConfig); SerializationManager.Initialize(globalConfig.SerializationProviders, this.globalConfig.FallbackSerializationProvider); initTimeout = globalConfig.MaxJoinAttemptTime; if (Debugger.IsAttached) { initTimeout = StandardExtensions.Max(TimeSpan.FromMinutes(10), globalConfig.MaxJoinAttemptTime); stopTimeout = initTimeout; } IPEndPoint here = nodeConfig.Endpoint; int generation = nodeConfig.Generation; if (generation == 0) { generation = SiloAddress.AllocateNewGeneration(); nodeConfig.Generation = generation; } LogManager.MyIPEndPoint = here; logger = LogManager.GetLogger("Silo", LoggerType.Runtime); logger.Info(ErrorCode.SiloGcSetting, "Silo starting with GC settings: ServerGC={0} GCLatencyMode={1}", GCSettings.IsServerGC, Enum.GetName(typeof(GCLatencyMode), GCSettings.LatencyMode)); if (!GCSettings.IsServerGC || !GCSettings.LatencyMode.Equals(GCLatencyMode.Batch)) { logger.Warn(ErrorCode.SiloGcWarning, "Note: Silo not running with ServerGC turned on or with GCLatencyMode.Batch enabled - recommend checking app config : <configuration>-<runtime>-<gcServer enabled=\"true\"> and <configuration>-<runtime>-<gcConcurrent enabled=\"false\"/>"); logger.Warn(ErrorCode.SiloGcWarning, "Note: ServerGC only kicks in on multi-core systems (settings enabling ServerGC have no effect on single-core machines)."); } logger.Info(ErrorCode.SiloInitializing, "-------------- Initializing {0} silo on host {1} MachineName {2} at {3}, gen {4} --------------", siloType, nodeConfig.DNSHostName, Environment.MachineName, here, generation); logger.Info(ErrorCode.SiloInitConfig, "Starting silo {0} with the following configuration= " + Environment.NewLine + "{1}", name, config.ToString(name)); if (keyStore != null) { // Re-establish reference to shared local key store in this app domain LocalDataStoreInstance.LocalDataStore = keyStore; } // Configure DI using Startup type bool usingCustomServiceProvider; Services = StartupBuilder.ConfigureStartup(nodeConfig.StartupTypeName, out usingCustomServiceProvider); healthCheckParticipants = new List <IHealthCheckParticipant>(); allSiloProviders = new List <IProvider>(); BufferPool.InitGlobalBufferPool(globalConfig); PlacementStrategy.Initialize(globalConfig); UnobservedExceptionsHandlerClass.SetUnobservedExceptionHandler(UnobservedExceptionHandler); AppDomain.CurrentDomain.UnhandledException += (obj, ev) => DomainUnobservedExceptionHandler(obj, (Exception)ev.ExceptionObject); try { grainFactory = Services.GetRequiredService <GrainFactory>(); } catch (InvalidOperationException exc) { logger.Error(ErrorCode.SiloStartError, "Exception during Silo.Start, GrainFactory was not registered in Dependency Injection container", exc); throw; } typeManager = new GrainTypeManager( here.Address.Equals(IPAddress.Loopback), grainFactory, new SiloAssemblyLoader(OrleansConfig.Defaults.AdditionalAssemblyDirectories)); // Performance metrics siloStatistics = new SiloStatisticsManager(globalConfig, nodeConfig); config.OnConfigChange("Defaults/LoadShedding", () => siloStatistics.MetricsTable.NodeConfig = nodeConfig, false); // The scheduler scheduler = new OrleansTaskScheduler(globalConfig, nodeConfig); healthCheckParticipants.Add(scheduler); // Initialize the message center var mc = new MessageCenter(here, generation, globalConfig, siloStatistics.MetricsTable); if (nodeConfig.IsGatewayNode) { mc.InstallGateway(nodeConfig.ProxyGatewayEndpoint); } messageCenter = mc; SiloIdentity = SiloAddress.ToLongString(); // GrainRuntime can be created only here, after messageCenter was created. grainRuntime = new GrainRuntime( globalConfig.ServiceId, SiloIdentity, grainFactory, new TimerRegistry(), new ReminderRegistry(), new StreamProviderManager(), Services); // Now the router/directory service // This has to come after the message center //; note that it then gets injected back into the message center.; localGrainDirectory = new LocalGrainDirectory(this); RegistrarManager.InitializeGrainDirectoryManager(localGrainDirectory, globalConfig.GlobalSingleInstanceNumberRetries); // Now the activation directory. // This needs to know which router to use so that it can keep the global directory in synch with the local one. activationDirectory = new ActivationDirectory(); // Now the consistent ring provider RingProvider = GlobalConfig.UseVirtualBucketsConsistentRing ? (IConsistentRingProvider) new VirtualBucketsRingProvider(SiloAddress, GlobalConfig.NumVirtualBucketsConsistentRing) : new ConsistentRingProvider(SiloAddress); // to preserve backwards compatibility, only use the service provider to inject grain dependencies if the user supplied his own // service provider, meaning that he is explicitly opting into it. var grainCreator = new GrainCreator(grainRuntime, usingCustomServiceProvider ? Services : null); Action <Dispatcher> setDispatcher; catalog = new Catalog(Constants.CatalogId, SiloAddress, Name, LocalGrainDirectory, typeManager, scheduler, activationDirectory, config, grainCreator, out setDispatcher); var dispatcher = new Dispatcher(scheduler, messageCenter, catalog, config); setDispatcher(dispatcher); RuntimeClient.Current = new InsideRuntimeClient( dispatcher, catalog, LocalGrainDirectory, SiloAddress, config, RingProvider, typeManager, grainFactory); messageCenter.RerouteHandler = InsideRuntimeClient.Current.RerouteMessage; messageCenter.SniffIncomingMessage = InsideRuntimeClient.Current.SniffIncomingMessage; siloStatistics.MetricsTable.Scheduler = scheduler; siloStatistics.MetricsTable.ActivationDirectory = activationDirectory; siloStatistics.MetricsTable.ActivationCollector = catalog.ActivationCollector; siloStatistics.MetricsTable.MessageCenter = messageCenter; DeploymentLoadPublisher.CreateDeploymentLoadPublisher(this, globalConfig); PlacementDirectorsManager.CreatePlacementDirectorsManager(globalConfig); // Now the incoming message agents incomingSystemAgent = new IncomingMessageAgent(Message.Categories.System, messageCenter, activationDirectory, scheduler, dispatcher); incomingPingAgent = new IncomingMessageAgent(Message.Categories.Ping, messageCenter, activationDirectory, scheduler, dispatcher); incomingAgent = new IncomingMessageAgent(Message.Categories.Application, messageCenter, activationDirectory, scheduler, dispatcher); membershipFactory = new MembershipFactory(); multiClusterFactory = new MultiClusterOracleFactory(); reminderFactory = new LocalReminderServiceFactory(); SystemStatus.Current = SystemStatus.Created; StringValueStatistic.FindOrCreate(StatisticNames.SILO_START_TIME, () => LogFormatter.PrintDate(startTime)); // this will help troubleshoot production deployment when looking at MDS logs. logger.Info(ErrorCode.SiloInitializingFinished, "-------------- Started silo {0}, ConsistentHashCode {1:X} --------------", SiloAddress.ToLongString(), SiloAddress.GetConsistentHashCode()); }