/// <summary> /// Creates and runs a distributed cache service /// </summary> /// <exception cref="CacheException">thrown when cache startup fails</exception> public static Task RunWithConfigurationAsync( ILogger logger, IDistributedCacheServiceHost host, HostInfo hostInfo, ITelemetryFieldsProvider telemetryFieldsProvider, DistributedCacheServiceConfiguration config, CancellationToken token, string keyspace = null) { logger.Info($"CAS log severity set to {config.MinimumLogSeverity}"); var arguments = new DistributedCacheServiceArguments( logger: logger, copier: null, copyRequester: null, host: host, hostInfo: hostInfo, cancellation: token, dataRootPath: config.DataRootPath, configuration: config, keyspace: keyspace) { TelemetryFieldsProvider = telemetryFieldsProvider, BuildCopyInfrastructure = logger => BuildCopyInfrastructure(logger, config), }; return(RunAsync(arguments)); }
private static void AdjustCopyInfrastructure(DistributedCacheServiceArguments arguments) { if (arguments.BuildCopyInfrastructure != null) { var(copier, copyRequester) = arguments.BuildCopyInfrastructure(arguments.Logger); arguments.Copier = copier; arguments.CopyRequester = copyRequester; } }
/// <summary> /// Creates and runs a distributed cache service /// </summary> /// <exception cref="CacheException">thrown when cache startup fails</exception> public static async Task RunAsync(DistributedCacheServiceArguments arguments) { // Switching to another thread. await Task.Yield(); var host = arguments.Host; var logger = arguments.Logger; var factory = new ContentServerFactory(arguments); // await _lifetimeManager.StartingService(); await host.OnStartingServiceAsync(); using (var server = factory.Create()) { var context = new Context(logger); try { // Removed this, because it is kind-of useless... // _serviceHealthyAction(false); var startupResult = await server.StartupAsync(context); if (!startupResult) { throw new CacheException(startupResult.ToString()); } // _serviceHealthyAction(true); host.OnStartedService(); logger.Info("Service started"); await arguments.Cancellation.WaitForCancellationAsync(); logger.Always("Exit event set"); } finally { var timeoutInMinutes = arguments?.Configuration?.DistributedContentSettings?.MaxShutdownDurationInMinutes ?? 30; BoolResult result = await ShutdownWithTimeout(context, server, TimeSpan.FromMinutes(timeoutInMinutes)); if (!result) { logger.Warning("Failed to shutdown local content server: {0}", result); } // await _lifetimeManager.TeardownCompleted(); host.OnTeardownCompleted(); } } }
private static Task ReportStartingServiceAsync( OperationContext context, IDistributedCacheServiceHost host, DistributedCacheServiceArguments arguments) { var configuration = arguments.Configuration; var logIntervalSeconds = configuration.DistributedContentSettings.ServiceRunningLogInSeconds; var logInterval = logIntervalSeconds != null ? (TimeSpan?)TimeSpan.FromSeconds(logIntervalSeconds.Value) : null; var logFilePath = GetPathForLifetimeTracking(configuration); LifetimeTracker.ServiceStarting(context, logInterval, logFilePath, arguments.TelemetryFieldsProvider.ServiceName); return(host.OnStartingServiceAsync()); }
/// <summary> /// Creates and runs a distributed cache service /// </summary> /// <exception cref="CacheException">thrown when cache startup fails</exception> public static async Task RunAsync(DistributedCacheServiceArguments arguments) { // Switching to another thread. await Task.Yield(); var host = arguments.Host; var startUpTime = Stopwatch.StartNew(); // NOTE(jubayard): this is the entry point for running CASaaS. At this point, the Logger inside the // arguments holds the client's implementation of our logging interface ILogger. Here, we may override the // client's decision with our own. // The disposableToken helps ensure that we shutdown properly and all logs are sent to their final // destination. var loggerReplacement = CreateReplacementLogger(arguments); arguments.Logger = loggerReplacement.Logger; using var disposableToken = loggerReplacement.DisposableToken; if (arguments.BuildCopyInfrastructure != null) { var(copier, pathTransformer, copyRequester) = arguments.BuildCopyInfrastructure(arguments.Logger); arguments.Copier = copier; arguments.PathTransformer = pathTransformer; arguments.CopyRequester = copyRequester; } var logger = arguments.Logger; var factory = new CacheServerFactory(arguments); await host.OnStartingServiceAsync(); // Technically, this method doesn't own the file copier, but no one actually owns it. // So to clean up the resources (and print some stats) we dispose it here. using (arguments.Copier as IDisposable) using (var server = factory.Create()) { var context = new Context(logger); var ctx = new OperationContext(context); try { var startupResult = await server.StartupAsync(context); if (!startupResult) { throw new CacheException(startupResult.ToString()); } host.OnStartedService(); using var serviceRunningTracker = DistributedCacheServiceRunningTracker.Create( ctx, SystemClock.Instance, new PassThroughFileSystem(), arguments.Configuration, startUpTime.Elapsed).GetValueOrDefault(); await arguments.Cancellation.WaitForCancellationAsync(); context.Always("Exit event set"); } finally { var timeoutInMinutes = arguments?.Configuration?.DistributedContentSettings?.MaxShutdownDurationInMinutes ?? 30; BoolResult result = await ShutdownWithTimeoutAsync(context, server, TimeSpan.FromMinutes(timeoutInMinutes)); if (!result) { context.Warning($"Failed to shutdown local content server: {result}"); } host.OnTeardownCompleted(); } } }
private static async Task <AzureBlobStorageLog> CreateAzureBlobStorageLogAsync(OperationContext operationContext, DistributedCacheServiceArguments arguments, AzureBlobStorageLogPublicConfiguration configuration) { Contract.RequiresNotNull(configuration); // There is a big issue here: on the one hand, we'd like to be able to configure everything from the XML // instead of our JSON configuration, simply because the XML is self-contained. On the other hand, the XML // will likely be shared across all stamps, so there's no "stamp-specific" configuration in there. That // means all stamp-level configuration must be done through the JSON. AzureBlobStorageCredentials credentials = null; if (configuration.UseSasTokens) { var secrets = await arguments.Host.RetrieveSecretsAsync(new List <RetrieveSecretsRequest>() { new RetrieveSecretsRequest(configuration.SecretName, SecretKind.SasToken) }, token : operationContext.Token); credentials = new AzureBlobStorageCredentials((UpdatingSasToken)secrets[configuration.SecretName]); } else { var secrets = await arguments.Host.RetrieveSecretsAsync(new List <RetrieveSecretsRequest>() { new RetrieveSecretsRequest(configuration.SecretName, SecretKind.PlainText) }, token : operationContext.Token); credentials = new AzureBlobStorageCredentials((PlainTextSecret)secrets[configuration.SecretName]); } var azureBlobStorageLogConfiguration = ToInternalConfiguration(configuration); var azureBlobStorageLog = new AzureBlobStorageLog( configuration: azureBlobStorageLogConfiguration, context: operationContext, clock: SystemClock.Instance, fileSystem: new PassThroughFileSystem(), telemetryFieldsProvider: arguments.TelemetryFieldsProvider, credentials: credentials); await azureBlobStorageLog.StartupAsync().ThrowIfFailure(); return(azureBlobStorageLog); }
private static IStructuredLogger CreateNLogAdapter(OperationContext operationContext, DistributedCacheServiceArguments arguments) { Contract.RequiresNotNull(arguments.Configuration.LoggingSettings); // This is done for performance. See: https://github.com/NLog/NLog/wiki/performance#configure-nlog-to-not-scan-for-assemblies NLog.Config.ConfigurationItemFactory.Default = new NLog.Config.ConfigurationItemFactory(typeof(NLog.ILogger).GetTypeInfo().Assembly); // This is needed for dependency ingestion. See: https://github.com/NLog/NLog/wiki/Dependency-injection-with-NLog // The issue is that we need to construct a log, which requires access to both our config and the host. It // seems too much to put it into the AzureBlobStorageLogTarget itself, so we do it here. var defaultConstructor = NLog.Config.ConfigurationItemFactory.Default.CreateInstance; NLog.Config.ConfigurationItemFactory.Default.CreateInstance = type => { if (type == typeof(AzureBlobStorageLogTarget)) { var log = CreateAzureBlobStorageLogAsync(operationContext, arguments, arguments.Configuration.LoggingSettings.Configuration).Result; var target = new AzureBlobStorageLogTarget(log); return(target); } return(defaultConstructor(type)); }; NLog.Targets.Target.Register <AzureBlobStorageLogTarget>(nameof(AzureBlobStorageLogTarget)); // This is done in order to allow our logging configuration to access key telemetry information. var telemetryFieldsProvider = arguments.TelemetryFieldsProvider; NLog.LayoutRenderers.LayoutRenderer.Register("BuildId", _ => telemetryFieldsProvider.BuildId); NLog.LayoutRenderers.LayoutRenderer.Register("APEnvironment", _ => telemetryFieldsProvider.APEnvironment); NLog.LayoutRenderers.LayoutRenderer.Register("APCluster", _ => telemetryFieldsProvider.APCluster); NLog.LayoutRenderers.LayoutRenderer.Register("APMachineFunction", _ => telemetryFieldsProvider.APMachineFunction); NLog.LayoutRenderers.LayoutRenderer.Register("MachineName", _ => telemetryFieldsProvider.MachineName); NLog.LayoutRenderers.LayoutRenderer.Register("ServiceName", _ => telemetryFieldsProvider.ServiceName); NLog.LayoutRenderers.LayoutRenderer.Register("ServiceVersion", _ => telemetryFieldsProvider.ServiceVersion); NLog.LayoutRenderers.LayoutRenderer.Register("Stamp", _ => telemetryFieldsProvider.Stamp); NLog.LayoutRenderers.LayoutRenderer.Register("Ring", _ => telemetryFieldsProvider.Ring); NLog.LayoutRenderers.LayoutRenderer.Register("ConfigurationId", _ => telemetryFieldsProvider.ConfigurationId); NLog.LayoutRenderers.LayoutRenderer.Register("CacheVersion", _ => Utilities.Branding.Version); // Follows ISO8601 without timezone specification. // See: https://kusto.azurewebsites.net/docs/query/scalar-data-types/datetime.html // See: https://docs.microsoft.com/en-us/dotnet/standard/base-types/standard-date-and-time-format-strings?view=netframework-4.8#the-round-trip-o-o-format-specifier var processStartTimeUtc = SystemClock.Instance.UtcNow.ToString("o", System.Globalization.CultureInfo.InvariantCulture); NLog.LayoutRenderers.LayoutRenderer.Register("ProcessStartTimeUtc", _ => processStartTimeUtc); var configuration = new NLog.Config.XmlLoggingConfiguration(arguments.Configuration.LoggingSettings.NLogConfigurationPath); return(new NLogAdapter(operationContext.TracingContext.Logger, configuration)); }
/// <summary> /// This method allows CASaaS to replace the host's logger for our own logger. /// </summary> /// <remarks> /// Since we don't perform any kind of stats aggregation on our side (i.e. statsd, MDM, etc), we rely on /// the host to do it. This is done through <see cref="MetricsAdapter"/>. /// /// The situation with respect to shutdown is a little bit odd: we create a custom target, which holds some /// managed resources that need to be released (because this release ensures any remaining logs will be /// sent to Kusto). /// NLog will make sure to dispose those resources when we shut it down, but we may actually return the /// host's logger, in which case we don't consider that we own it, because clean up may happen on whatever /// code is actually using us, so we don't want to dispose in that case. /// </remarks> private static (ILogger Logger, IDisposable DisposableToken) CreateReplacementLogger(DistributedCacheServiceArguments arguments) { var logger = arguments.Logger; var loggingSettings = arguments.Configuration.LoggingSettings; if (string.IsNullOrEmpty(loggingSettings?.NLogConfigurationPath)) { return(logger, null); } Contract.RequiresNotNull(arguments.TelemetryFieldsProvider); // This context is associated to the host's logger. In this way, we can make sure that if we have any // issues with our logging, we can always go and read the host's logs to figure out what's going on. var context = new Context(logger); var operationContext = new OperationContext(context); context.Info($"Replacing cache logger for NLog-based implementation using configuration file at `{loggingSettings.NLogConfigurationPath}`"); try { var nLogAdapter = CreateNLogAdapter(operationContext, arguments); if (arguments.Logger is IOperationLogger operationLogger) { // NOTE(jubayard): the MetricsAdapter doesn't own the loggers, and hence won't dispose them. This // means we don't change the disposableToken. var wrapper = new MetricsAdapter(nLogAdapter, operationLogger); return(wrapper, nLogAdapter); } return(nLogAdapter, nLogAdapter); } catch (Exception e) { context.Error($"Failed to instantiate NLog-based logger with error: {e}"); return(logger, null); } }
/// <summary> /// Creates and runs a distributed cache service /// </summary> /// <exception cref="CacheException">thrown when cache startup fails</exception> public static async Task RunAsync(DistributedCacheServiceArguments arguments) { // Switching to another thread. await Task.Yield(); var host = arguments.Host; var timer = Stopwatch.StartNew(); InitializeGlobalLifetimeManager(host); // NOTE(jubayard): this is the entry point for running CASaaS. At this point, the Logger inside the // arguments holds the client's implementation of our logging interface ILogger. Here, we may override the // client's decision with our own. // The disposableToken helps ensure that we shutdown properly and all logs are sent to their final // destination. var loggerReplacement = CreateReplacementLogger(arguments); arguments.Logger = loggerReplacement.Logger; using var disposableToken = loggerReplacement.DisposableToken; AdjustCopyInfrastructure(arguments); var context = new Context(arguments.Logger); await ReportStartingServiceAsync(context, host); var factory = new CacheServerFactory(arguments); var operationContext = new OperationContext(context); ServiceOfflineDurationTracker serviceRunningTracker = null; // Technically, this method doesn't own the file copier, but no one actually owns it. // So to clean up the resources (and print some stats) we dispose it here. using (arguments.Copier as IDisposable) using (var server = factory.Create()) { try { var startupResult = await server.StartupAsync(context); if (!startupResult) { throw new CacheException(startupResult.ToString()); } var serviceRunningTrackerResult = ServiceOfflineDurationTracker.Create( operationContext, SystemClock.Instance, new PassThroughFileSystem(), arguments.Configuration); ReportServiceStarted(operationContext, host, serviceRunningTrackerResult, timer.Elapsed); serviceRunningTracker = serviceRunningTrackerResult.GetValueOrDefault(); await arguments.Cancellation.WaitForCancellationAsync(); ReportShuttingDownService(context); } catch (Exception e) { ReportServiceStartupFailed(context, e, timer.Elapsed); throw; } finally { // Writing the current time before shutting the service down. serviceRunningTracker?.LogCurrentTimeStampToFile(operationContext); timer.Reset(); var timeoutInMinutes = arguments.Configuration?.DistributedContentSettings?.MaxShutdownDurationInMinutes ?? 30; var result = await server .ShutdownAsync(context) .WithTimeoutAsync("Server shutdown", TimeSpan.FromMinutes(timeoutInMinutes)); serviceRunningTracker?.Dispose(); ReportServiceStopped(context, host, result, timer.Elapsed); } } }
/// <summary> /// Creates and runs a distributed cache service /// </summary> /// <exception cref="CacheException">thrown when cache startup fails</exception> public static async Task RunAsync(DistributedCacheServiceArguments arguments) { // Switching to another thread. await Task.Yield(); var host = arguments.Host; var timer = Stopwatch.StartNew(); InitializeLifetimeTracker(host); using var cts = CancellationTokenSource.CreateLinkedTokenSource(arguments.Cancellation); arguments.Cancellation = cts.Token; if (arguments.Configuration.RespectRequestTeardown) { LifetimeManager.OnTeardownRequested += _ => { cts.Cancel(); }; } // NOTE(jubayard): this is the entry point for running CASaaS. At this point, the Logger inside the // arguments holds the client's implementation of our logging interface ILogger. Here, we may override the // client's decision with our own. // The disposableToken helps ensure that we shutdown properly and all logs are sent to their final // destination. var loggerReplacement = LoggerFactory.CreateReplacementLogger(arguments); arguments.Logger = loggerReplacement.Logger; using var disposableToken = loggerReplacement.DisposableToken; var context = new Context(arguments.Logger); var operationContext = new OperationContext(context, arguments.Cancellation); InitializeActivityTrackerIfNeeded(context, arguments.Configuration.DistributedContentSettings); AdjustCopyInfrastructure(arguments); await ReportStartingServiceAsync(operationContext, host, arguments.Configuration); var factory = new CacheServerFactory(arguments); // Technically, this method doesn't own the file copier, but no one actually owns it. // So to clean up the resources (and print some stats) we dispose it here. using (arguments.Copier as IDisposable) using (var server = factory.Create()) { try { var startupResult = await server.StartupAsync(context); if (!startupResult) { throw new CacheException(startupResult.ToString()); } ReportServiceStarted(operationContext, host); await arguments.Cancellation.WaitForCancellationAsync(); await ReportShuttingDownServiceAsync(operationContext, host); } catch (Exception e) { ReportServiceStartupFailed(context, e, timer.Elapsed); throw; } finally { var timeoutInMinutes = arguments.Configuration?.DistributedContentSettings?.MaxShutdownDurationInMinutes ?? 30; var result = await server .ShutdownAsync(context) .WithTimeoutAsync("Server shutdown", TimeSpan.FromMinutes(timeoutInMinutes)); ReportServiceStopped(context, host, result); } } }