/// <summary>
/// Creates and runs a distributed cache service
/// </summary>
/// <exception cref="CacheException">thrown when cache startup fails</exception>
public static Task RunWithConfigurationAsync(
ILogger logger,
IDistributedCacheServiceHost host,
HostInfo hostInfo,
ITelemetryFieldsProvider telemetryFieldsProvider,
DistributedCacheServiceConfiguration config,
CancellationToken token,
string keyspace = null)
{
logger.Info($"CAS log severity set to {config.MinimumLogSeverity}");
var arguments = new DistributedCacheServiceArguments(
logger: logger,
copier: null,
copyRequester: null,
host: host,
hostInfo: hostInfo,
cancellation: token,
dataRootPath: config.DataRootPath,
configuration: config,
keyspace: keyspace)
{
TelemetryFieldsProvider = telemetryFieldsProvider,
BuildCopyInfrastructure = logger => BuildCopyInfrastructure(logger, config),
};
return(RunAsync(arguments));
}
private static void AdjustCopyInfrastructure(DistributedCacheServiceArguments arguments)
{
if (arguments.BuildCopyInfrastructure != null)
{
var(copier, copyRequester) = arguments.BuildCopyInfrastructure(arguments.Logger);
arguments.Copier = copier;
arguments.CopyRequester = copyRequester;
}
}
/// <summary>
/// Creates and runs a distributed cache service
/// </summary>
/// <exception cref="CacheException">thrown when cache startup fails</exception>
public static async Task RunAsync(DistributedCacheServiceArguments arguments)
{
// Switching to another thread.
await Task.Yield();
var host = arguments.Host;
var logger = arguments.Logger;
var factory = new ContentServerFactory(arguments);
// await _lifetimeManager.StartingService();
await host.OnStartingServiceAsync();
using (var server = factory.Create())
{
var context = new Context(logger);
try
{
// Removed this, because it is kind-of useless...
// _serviceHealthyAction(false);
var startupResult = await server.StartupAsync(context);
if (!startupResult)
{
throw new CacheException(startupResult.ToString());
}
// _serviceHealthyAction(true);
host.OnStartedService();
logger.Info("Service started");
await arguments.Cancellation.WaitForCancellationAsync();
logger.Always("Exit event set");
}
finally
{
var timeoutInMinutes = arguments?.Configuration?.DistributedContentSettings?.MaxShutdownDurationInMinutes ?? 30;
BoolResult result = await ShutdownWithTimeout(context, server, TimeSpan.FromMinutes(timeoutInMinutes));
if (!result)
{
logger.Warning("Failed to shutdown local content server: {0}", result);
}
// await _lifetimeManager.TeardownCompleted();
host.OnTeardownCompleted();
}
}
}
private static Task ReportStartingServiceAsync(
OperationContext context,
IDistributedCacheServiceHost host,
DistributedCacheServiceArguments arguments)
{
var configuration = arguments.Configuration;
var logIntervalSeconds = configuration.DistributedContentSettings.ServiceRunningLogInSeconds;
var logInterval = logIntervalSeconds != null ? (TimeSpan?)TimeSpan.FromSeconds(logIntervalSeconds.Value) : null;
var logFilePath = GetPathForLifetimeTracking(configuration);
LifetimeTracker.ServiceStarting(context, logInterval, logFilePath, arguments.TelemetryFieldsProvider.ServiceName);
return(host.OnStartingServiceAsync());
}
/// <summary>
/// Creates and runs a distributed cache service
/// </summary>
/// <exception cref="CacheException">thrown when cache startup fails</exception>
public static async Task RunAsync(DistributedCacheServiceArguments arguments)
{
// Switching to another thread.
await Task.Yield();
var host = arguments.Host;
var startUpTime = Stopwatch.StartNew();
// NOTE(jubayard): this is the entry point for running CASaaS. At this point, the Logger inside the
// arguments holds the client's implementation of our logging interface ILogger. Here, we may override the
// client's decision with our own.
// The disposableToken helps ensure that we shutdown properly and all logs are sent to their final
// destination.
var loggerReplacement = CreateReplacementLogger(arguments);
arguments.Logger = loggerReplacement.Logger;
using var disposableToken = loggerReplacement.DisposableToken;
if (arguments.BuildCopyInfrastructure != null)
{
var(copier, pathTransformer, copyRequester) = arguments.BuildCopyInfrastructure(arguments.Logger);
arguments.Copier = copier;
arguments.PathTransformer = pathTransformer;
arguments.CopyRequester = copyRequester;
}
var logger = arguments.Logger;
var factory = new CacheServerFactory(arguments);
await host.OnStartingServiceAsync();
// Technically, this method doesn't own the file copier, but no one actually owns it.
// So to clean up the resources (and print some stats) we dispose it here.
using (arguments.Copier as IDisposable)
using (var server = factory.Create())
{
var context = new Context(logger);
var ctx = new OperationContext(context);
try
{
var startupResult = await server.StartupAsync(context);
if (!startupResult)
{
throw new CacheException(startupResult.ToString());
}
host.OnStartedService();
using var serviceRunningTracker = DistributedCacheServiceRunningTracker.Create(
ctx,
SystemClock.Instance,
new PassThroughFileSystem(),
arguments.Configuration,
startUpTime.Elapsed).GetValueOrDefault();
await arguments.Cancellation.WaitForCancellationAsync();
context.Always("Exit event set");
}
finally
{
var timeoutInMinutes = arguments?.Configuration?.DistributedContentSettings?.MaxShutdownDurationInMinutes ?? 30;
BoolResult result = await ShutdownWithTimeoutAsync(context, server, TimeSpan.FromMinutes(timeoutInMinutes));
if (!result)
{
context.Warning($"Failed to shutdown local content server: {result}");
}
host.OnTeardownCompleted();
}
}
}
private static async Task <AzureBlobStorageLog> CreateAzureBlobStorageLogAsync(OperationContext operationContext, DistributedCacheServiceArguments arguments, AzureBlobStorageLogPublicConfiguration configuration)
{
Contract.RequiresNotNull(configuration);
// There is a big issue here: on the one hand, we'd like to be able to configure everything from the XML
// instead of our JSON configuration, simply because the XML is self-contained. On the other hand, the XML
// will likely be shared across all stamps, so there's no "stamp-specific" configuration in there. That
// means all stamp-level configuration must be done through the JSON.
AzureBlobStorageCredentials credentials = null;
if (configuration.UseSasTokens)
{
var secrets = await arguments.Host.RetrieveSecretsAsync(new List <RetrieveSecretsRequest>()
{
new RetrieveSecretsRequest(configuration.SecretName, SecretKind.SasToken)
}, token : operationContext.Token);
credentials = new AzureBlobStorageCredentials((UpdatingSasToken)secrets[configuration.SecretName]);
}
else
{
var secrets = await arguments.Host.RetrieveSecretsAsync(new List <RetrieveSecretsRequest>()
{
new RetrieveSecretsRequest(configuration.SecretName, SecretKind.PlainText)
}, token : operationContext.Token);
credentials = new AzureBlobStorageCredentials((PlainTextSecret)secrets[configuration.SecretName]);
}
var azureBlobStorageLogConfiguration = ToInternalConfiguration(configuration);
var azureBlobStorageLog = new AzureBlobStorageLog(
configuration: azureBlobStorageLogConfiguration,
context: operationContext,
clock: SystemClock.Instance,
fileSystem: new PassThroughFileSystem(),
telemetryFieldsProvider: arguments.TelemetryFieldsProvider,
credentials: credentials);
await azureBlobStorageLog.StartupAsync().ThrowIfFailure();
return(azureBlobStorageLog);
}
private static IStructuredLogger CreateNLogAdapter(OperationContext operationContext, DistributedCacheServiceArguments arguments)
{
Contract.RequiresNotNull(arguments.Configuration.LoggingSettings);
// This is done for performance. See: https://github.com/NLog/NLog/wiki/performance#configure-nlog-to-not-scan-for-assemblies
NLog.Config.ConfigurationItemFactory.Default = new NLog.Config.ConfigurationItemFactory(typeof(NLog.ILogger).GetTypeInfo().Assembly);
// This is needed for dependency ingestion. See: https://github.com/NLog/NLog/wiki/Dependency-injection-with-NLog
// The issue is that we need to construct a log, which requires access to both our config and the host. It
// seems too much to put it into the AzureBlobStorageLogTarget itself, so we do it here.
var defaultConstructor = NLog.Config.ConfigurationItemFactory.Default.CreateInstance;
NLog.Config.ConfigurationItemFactory.Default.CreateInstance = type =>
{
if (type == typeof(AzureBlobStorageLogTarget))
{
var log = CreateAzureBlobStorageLogAsync(operationContext, arguments, arguments.Configuration.LoggingSettings.Configuration).Result;
var target = new AzureBlobStorageLogTarget(log);
return(target);
}
return(defaultConstructor(type));
};
NLog.Targets.Target.Register <AzureBlobStorageLogTarget>(nameof(AzureBlobStorageLogTarget));
// This is done in order to allow our logging configuration to access key telemetry information.
var telemetryFieldsProvider = arguments.TelemetryFieldsProvider;
NLog.LayoutRenderers.LayoutRenderer.Register("BuildId", _ => telemetryFieldsProvider.BuildId);
NLog.LayoutRenderers.LayoutRenderer.Register("APEnvironment", _ => telemetryFieldsProvider.APEnvironment);
NLog.LayoutRenderers.LayoutRenderer.Register("APCluster", _ => telemetryFieldsProvider.APCluster);
NLog.LayoutRenderers.LayoutRenderer.Register("APMachineFunction", _ => telemetryFieldsProvider.APMachineFunction);
NLog.LayoutRenderers.LayoutRenderer.Register("MachineName", _ => telemetryFieldsProvider.MachineName);
NLog.LayoutRenderers.LayoutRenderer.Register("ServiceName", _ => telemetryFieldsProvider.ServiceName);
NLog.LayoutRenderers.LayoutRenderer.Register("ServiceVersion", _ => telemetryFieldsProvider.ServiceVersion);
NLog.LayoutRenderers.LayoutRenderer.Register("Stamp", _ => telemetryFieldsProvider.Stamp);
NLog.LayoutRenderers.LayoutRenderer.Register("Ring", _ => telemetryFieldsProvider.Ring);
NLog.LayoutRenderers.LayoutRenderer.Register("ConfigurationId", _ => telemetryFieldsProvider.ConfigurationId);
NLog.LayoutRenderers.LayoutRenderer.Register("CacheVersion", _ => Utilities.Branding.Version);
// Follows ISO8601 without timezone specification.
// See: https://kusto.azurewebsites.net/docs/query/scalar-data-types/datetime.html
// See: https://docs.microsoft.com/en-us/dotnet/standard/base-types/standard-date-and-time-format-strings?view=netframework-4.8#the-round-trip-o-o-format-specifier
var processStartTimeUtc = SystemClock.Instance.UtcNow.ToString("o", System.Globalization.CultureInfo.InvariantCulture);
NLog.LayoutRenderers.LayoutRenderer.Register("ProcessStartTimeUtc", _ => processStartTimeUtc);
var configuration = new NLog.Config.XmlLoggingConfiguration(arguments.Configuration.LoggingSettings.NLogConfigurationPath);
return(new NLogAdapter(operationContext.TracingContext.Logger, configuration));
}
/// <summary>
/// This method allows CASaaS to replace the host's logger for our own logger.
/// </summary>
/// <remarks>
/// Since we don't perform any kind of stats aggregation on our side (i.e. statsd, MDM, etc), we rely on
/// the host to do it. This is done through <see cref="MetricsAdapter"/>.
///
/// The situation with respect to shutdown is a little bit odd: we create a custom target, which holds some
/// managed resources that need to be released (because this release ensures any remaining logs will be
/// sent to Kusto).
/// NLog will make sure to dispose those resources when we shut it down, but we may actually return the
/// host's logger, in which case we don't consider that we own it, because clean up may happen on whatever
/// code is actually using us, so we don't want to dispose in that case.
/// </remarks>
private static (ILogger Logger, IDisposable DisposableToken) CreateReplacementLogger(DistributedCacheServiceArguments arguments)
{
var logger = arguments.Logger;
var loggingSettings = arguments.Configuration.LoggingSettings;
if (string.IsNullOrEmpty(loggingSettings?.NLogConfigurationPath))
{
return(logger, null);
}
Contract.RequiresNotNull(arguments.TelemetryFieldsProvider);
// This context is associated to the host's logger. In this way, we can make sure that if we have any
// issues with our logging, we can always go and read the host's logs to figure out what's going on.
var context = new Context(logger);
var operationContext = new OperationContext(context);
context.Info($"Replacing cache logger for NLog-based implementation using configuration file at `{loggingSettings.NLogConfigurationPath}`");
try
{
var nLogAdapter = CreateNLogAdapter(operationContext, arguments);
if (arguments.Logger is IOperationLogger operationLogger)
{
// NOTE(jubayard): the MetricsAdapter doesn't own the loggers, and hence won't dispose them. This
// means we don't change the disposableToken.
var wrapper = new MetricsAdapter(nLogAdapter, operationLogger);
return(wrapper, nLogAdapter);
}
return(nLogAdapter, nLogAdapter);
}
catch (Exception e)
{
context.Error($"Failed to instantiate NLog-based logger with error: {e}");
return(logger, null);
}
}
/// <summary>
/// Creates and runs a distributed cache service
/// </summary>
/// <exception cref="CacheException">thrown when cache startup fails</exception>
public static async Task RunAsync(DistributedCacheServiceArguments arguments)
{
// Switching to another thread.
await Task.Yield();
var host = arguments.Host;
var timer = Stopwatch.StartNew();
InitializeGlobalLifetimeManager(host);
// NOTE(jubayard): this is the entry point for running CASaaS. At this point, the Logger inside the
// arguments holds the client's implementation of our logging interface ILogger. Here, we may override the
// client's decision with our own.
// The disposableToken helps ensure that we shutdown properly and all logs are sent to their final
// destination.
var loggerReplacement = CreateReplacementLogger(arguments);
arguments.Logger = loggerReplacement.Logger;
using var disposableToken = loggerReplacement.DisposableToken;
AdjustCopyInfrastructure(arguments);
var context = new Context(arguments.Logger);
await ReportStartingServiceAsync(context, host);
var factory = new CacheServerFactory(arguments);
var operationContext = new OperationContext(context);
ServiceOfflineDurationTracker serviceRunningTracker = null;
// Technically, this method doesn't own the file copier, but no one actually owns it.
// So to clean up the resources (and print some stats) we dispose it here.
using (arguments.Copier as IDisposable)
using (var server = factory.Create())
{
try
{
var startupResult = await server.StartupAsync(context);
if (!startupResult)
{
throw new CacheException(startupResult.ToString());
}
var serviceRunningTrackerResult = ServiceOfflineDurationTracker.Create(
operationContext,
SystemClock.Instance,
new PassThroughFileSystem(),
arguments.Configuration);
ReportServiceStarted(operationContext, host, serviceRunningTrackerResult, timer.Elapsed);
serviceRunningTracker = serviceRunningTrackerResult.GetValueOrDefault();
await arguments.Cancellation.WaitForCancellationAsync();
ReportShuttingDownService(context);
}
catch (Exception e)
{
ReportServiceStartupFailed(context, e, timer.Elapsed);
throw;
}
finally
{
// Writing the current time before shutting the service down.
serviceRunningTracker?.LogCurrentTimeStampToFile(operationContext);
timer.Reset();
var timeoutInMinutes = arguments.Configuration?.DistributedContentSettings?.MaxShutdownDurationInMinutes ?? 30;
var result = await server
.ShutdownAsync(context)
.WithTimeoutAsync("Server shutdown", TimeSpan.FromMinutes(timeoutInMinutes));
serviceRunningTracker?.Dispose();
ReportServiceStopped(context, host, result, timer.Elapsed);
}
}
}
/// <summary>
/// Creates and runs a distributed cache service
/// </summary>
/// <exception cref="CacheException">thrown when cache startup fails</exception>
public static async Task RunAsync(DistributedCacheServiceArguments arguments)
{
// Switching to another thread.
await Task.Yield();
var host = arguments.Host;
var timer = Stopwatch.StartNew();
InitializeLifetimeTracker(host);
using var cts = CancellationTokenSource.CreateLinkedTokenSource(arguments.Cancellation);
arguments.Cancellation = cts.Token;
if (arguments.Configuration.RespectRequestTeardown)
{
LifetimeManager.OnTeardownRequested += _ =>
{
cts.Cancel();
};
}
// NOTE(jubayard): this is the entry point for running CASaaS. At this point, the Logger inside the
// arguments holds the client's implementation of our logging interface ILogger. Here, we may override the
// client's decision with our own.
// The disposableToken helps ensure that we shutdown properly and all logs are sent to their final
// destination.
var loggerReplacement = LoggerFactory.CreateReplacementLogger(arguments);
arguments.Logger = loggerReplacement.Logger;
using var disposableToken = loggerReplacement.DisposableToken;
var context = new Context(arguments.Logger);
var operationContext = new OperationContext(context, arguments.Cancellation);
InitializeActivityTrackerIfNeeded(context, arguments.Configuration.DistributedContentSettings);
AdjustCopyInfrastructure(arguments);
await ReportStartingServiceAsync(operationContext, host, arguments.Configuration);
var factory = new CacheServerFactory(arguments);
// Technically, this method doesn't own the file copier, but no one actually owns it.
// So to clean up the resources (and print some stats) we dispose it here.
using (arguments.Copier as IDisposable)
using (var server = factory.Create())
{
try
{
var startupResult = await server.StartupAsync(context);
if (!startupResult)
{
throw new CacheException(startupResult.ToString());
}
ReportServiceStarted(operationContext, host);
await arguments.Cancellation.WaitForCancellationAsync();
await ReportShuttingDownServiceAsync(operationContext, host);
}
catch (Exception e)
{
ReportServiceStartupFailed(context, e, timer.Elapsed);
throw;
}
finally
{
var timeoutInMinutes = arguments.Configuration?.DistributedContentSettings?.MaxShutdownDurationInMinutes ?? 30;
var result = await server
.ShutdownAsync(context)
.WithTimeoutAsync("Server shutdown", TimeSpan.FromMinutes(timeoutInMinutes));
ReportServiceStopped(context, host, result);
}
}
}
