public DistributedCacheServiceConfiguration(
LocalCasSettings localCasSettings,
DistributedContentSettings distributedCacheSettings)
{
LocalCasSettings = localCasSettings;
DistributedContentSettings = distributedCacheSettings;
}
internal DistributedCacheServiceArguments CreateDistributedCacheServiceArguments(
IAbsolutePathFileCopier copier,
IAbsolutePathTransformer pathTransformer,
DistributedContentSettings dcs,
HostInfo host,
string cacheName,
string cacheRootPath,
uint grpcPort,
int maxSizeQuotaMB,
string dataRootPath,
CancellationToken ct,
int?bufferSizeForGrpcCopies = null,
int?gzipBarrierSizeForGrpcCopies = null)
{
var distributedCacheServiceHost = new EnvironmentVariableHost();
var localCasSettings = LocalCasSettings.Default(
maxSizeQuotaMB: maxSizeQuotaMB,
cacheRootPath: cacheRootPath,
cacheName: cacheName,
grpcPort: grpcPort,
grpcPortFileName: _scenario);
localCasSettings.PreferredCacheDrive = Path.GetPathRoot(cacheRootPath);
localCasSettings.ServiceSettings = new LocalCasServiceSettings(60, scenarioName: _scenario, grpcPort: grpcPort, grpcPortFileName: _scenario, bufferSizeForGrpcCopies: bufferSizeForGrpcCopies, gzipBarrierSizeForGrpcCopies: gzipBarrierSizeForGrpcCopies);
var config = new DistributedCacheServiceConfiguration(localCasSettings, dcs);
return(new DistributedCacheServiceArguments(_logger, copier, pathTransformer, distributedCacheServiceHost, host, ct, dataRootPath, config, null));
}
internal DistributedCacheServiceArguments CreateDistributedCacheServiceArguments(
IRemoteFileCopier copier,
IContentCommunicationManager copyRequester,
DistributedContentSettings dcs,
HostInfo host,
string cacheName,
string cacheRootPath,
uint grpcPort,
int maxSizeQuotaMB,
string dataRootPath,
CancellationToken ct,
int?bufferSizeForGrpcCopies,
int?gzipBarrierSizeForGrpcCopies,
LoggingSettings loggingSettings,
ITelemetryFieldsProvider telemetryFieldsProvider)
{
var distributedCacheServiceHost = new EnvironmentVariableHost();
var localCasSettings = LocalCasSettings.Default(
maxSizeQuotaMB: maxSizeQuotaMB,
cacheRootPath: cacheRootPath,
cacheName: cacheName,
grpcPort: grpcPort,
grpcPortFileName: _scenario);
localCasSettings.PreferredCacheDrive = new AbsolutePath(cacheRootPath).GetPathRoot();
localCasSettings.ServiceSettings = new LocalCasServiceSettings(60, scenarioName: _scenario, grpcPort: grpcPort, grpcPortFileName: _scenario, bufferSizeForGrpcCopies: bufferSizeForGrpcCopies, gzipBarrierSizeForGrpcCopies: gzipBarrierSizeForGrpcCopies);
var config = new DistributedCacheServiceConfiguration(localCasSettings, dcs, loggingSettings);
return(new DistributedCacheServiceArguments(_logger, copier, copyRequester, distributedCacheServiceHost, host, ct, dataRootPath, config, null)
{
TelemetryFieldsProvider = telemetryFieldsProvider,
});
}
internal DistributedCacheServiceArguments CreateDistributedCacheServiceArguments(
IAbsolutePathFileCopier copier, IAbsolutePathTransformer pathTransformer, HostInfo host, string cacheName, string cacheRootPath, uint grpcPort, int maxSizeQuotaMB, string dataRootPath, CancellationToken ct, int?bufferSizeForGrpcCopies = null)
{
var distributedCacheServiceHost = new TestHost();
var localCasSettings = LocalCasSettings.Default(
maxSizeQuotaMB: maxSizeQuotaMB,
cacheRootPath: cacheRootPath,
cacheName: cacheName,
grpcPort: grpcPort,
grpcPortFileName: _scenario);
localCasSettings.PreferredCacheDrive = Path.GetPathRoot(cacheRootPath);
localCasSettings.ServiceSettings = new LocalCasServiceSettings(60, scenarioName: _scenario, grpcPort: grpcPort, grpcPortFileName: _scenario, bufferSizeForGrpcCopies: bufferSizeForGrpcCopies);
var redisConnectionString = Environment.GetEnvironmentVariable(EnvironmentConnectionStringProvider.RedisConnectionStringEnvironmentVariable);
var distributedContentSettings = DistributedContentSettings.CreateEnabled(new Dictionary <string, string>()
{
{ host.StampId, redisConnectionString }
});
var config = new DistributedCacheServiceConfiguration(localCasSettings, distributedContentSettings);
return(new DistributedCacheServiceArguments(_logger, copier, pathTransformer, distributedCacheServiceHost, host, ct, dataRootPath, config, null));
}
/// <summary>
/// Filters out <see cref="NamedCacheSettings"/> entries whose capability demands are not met by the host.
/// Also filters out entries pointing to e physical drive that doesn't exist.
/// </summary>
/// <param name="this"></param>
/// <param name="hostCapabilities">Capabilities provided by the host</param>
/// <param name="logger"></param>
/// <param name="driveExistenceOverride">For testing purposes</param>
/// <returns></returns>
public static LocalCasSettings FilterUnsupportedNamedCaches(
this LocalCasSettings @this, IEnumerable <string> hostCapabilities, ILogger logger, Predicate <string> driveExistenceOverride = null)
{
Predicate <string> checkDriveExists = driveExistenceOverride ?? Directory.Exists;
var result = new LocalCasSettings
{
CasClientSettings = @this.CasClientSettings,
ServiceSettings = @this.ServiceSettings,
DrivePreferenceOrder = new List <string>(@this.DrivePreferenceOrder)
};
var filteredCaches = new Dictionary <string, NamedCacheSettings>(@this.CacheSettingsByCacheName.Comparer);
foreach (KeyValuePair <string, NamedCacheSettings> kvp
in @this.CacheSettingsByCacheName)
{
// check that the stamp has the capabilities required by the named cache.
if (kvp.Value.RequiredCapabilities != null && kvp.Value.RequiredCapabilities.Count > 0)
{
string missingCaps = string.Join(",", kvp.Value.RequiredCapabilities
.Where(cap => !hostCapabilities.Contains(cap, StringComparer.OrdinalIgnoreCase)));
if (!string.IsNullOrEmpty(missingCaps))
{
logger.Debug(
"Named cache '{0}' was discarded since environment lacks required capabilities: {1}.",
kvp.Key, missingCaps);
continue;
}
}
// check that machine the drive required by named cache.
// TODO: Should remove this, after measuring this doesn't happens. If the drive layout doesn't match capability
// we'd rather fail.
AbsolutePath rootPath = @this.GetCacheRootPathWithScenario(kvp.Key);
string root = Path.GetPathRoot(rootPath.Path);
if (!checkDriveExists(root))
{
logger.Error(
"Named cache '{0}' was discarded since the drive required by {1} does not exist or is inaccessible on the machine.",
kvp.Key, rootPath);
continue;
}
filteredCaches.Add(kvp.Key, kvp.Value);
}
result.CacheSettingsByCacheName = filteredCaches;
result.DrivePreferenceOrder = GetSupportedDrivePreferenceOrder(@this.DrivePreferenceOrder, filteredCaches, logger);
return(result);
}
public void TestLocalCASSettings()
{
var input = LocalCasSettings.Default();
var options = new JsonSerializerOptions()
{
WriteIndented = true
};
var json = JsonSerializer.Serialize(input, options);
json.Should().Contain("CasClientSettings");
json.Should().Contain("CacheSettings");
}
private static ServiceConfiguration CreateServiceConfiguration(
ILogger logger,
IAbsFileSystem fileSystem,
LocalCasSettings localCasSettings,
DistributedContentSettings distributedSettings,
AbsolutePath dataRootPath,
bool isDistributed)
{
var namedCacheRoots = new Dictionary <string, AbsolutePath>(StringComparer.OrdinalIgnoreCase);
foreach (KeyValuePair <string, NamedCacheSettings> settings in localCasSettings.CacheSettingsByCacheName)
{
var rootPath = localCasSettings.GetCacheRootPathWithScenario(settings.Key);
logger.Debug($"Writing content store config file at {rootPath}.");
WriteContentStoreConfigFile(settings.Value.CacheSizeQuotaString, rootPath, fileSystem);
if (!isDistributed)
{
namedCacheRoots[settings.Key] = rootPath;
}
else
{
// Arbitrary set to match ServiceConfiguration and LocalContentServer pattern
namedCacheRoots[localCasSettings.CasClientSettings.DefaultCacheName] = rootPath;
}
}
if (!namedCacheRoots.Keys.Any(name => localCasSettings.CasClientSettings.DefaultCacheName.Equals(name, StringComparison.OrdinalIgnoreCase)))
{
throw new ArgumentException(
$"Must have the default cache name {localCasSettings.CasClientSettings.DefaultCacheName} as one of the named cache roots.");
}
var result = new ServiceConfiguration(
namedCacheRoots,
dataRootPath,
localCasSettings.ServiceSettings.MaxPipeListeners,
localCasSettings.ServiceSettings.GracefulShutdownSeconds,
(int)localCasSettings.ServiceSettings.GrpcPort,
grpcPortFileName: localCasSettings.ServiceSettings.GrpcPortFileName,
bufferSizeForGrpcCopies: localCasSettings.ServiceSettings.BufferSizeForGrpcCopies,
proactivePushCountLimit: localCasSettings.ServiceSettings.MaxProactivePushRequestHandlers,
logIncrementalStatsInterval: distributedSettings?.LogIncrementalStatsInterval,
logMachineStatsInterval: distributedSettings?.LogMachineStatsInterval,
logIncrementalStatsCounterNames: distributedSettings?.IncrementalStatisticsCounterNames,
asyncSessionShutdownTimeout: distributedSettings?.AsyncSessionShutdownTimeout);
ApplyIfNotNull(distributedSettings?.TraceServiceGrpcOperations, v => result.TraceGrpcOperation = v);
return(result);
}
/// <summary>
/// Filters out <see cref="NamedCacheSettings"/> entries whose capability demands are not met by the host.
/// Also filters out entries pointing to e physical drive that doesn't exist.
/// </summary>
public static LocalCasSettings FilterUnsupportedNamedCaches(
this LocalCasSettings @this, IEnumerable <string> hostCapabilities, ILogger logger)
{
Predicate <string> checkDriveExists = Directory.Exists;
var result = new LocalCasSettings
{
UseScenarioIsolation = @this.UseScenarioIsolation,
CasClientSettings = @this.CasClientSettings,
ServiceSettings = @this.ServiceSettings,
DrivePreferenceOrder = new List <string>(@this.DrivePreferenceOrder)
};
var filteredCaches = new Dictionary <string, NamedCacheSettings>(@this.CacheSettingsByCacheName.Comparer);
foreach (KeyValuePair <string, NamedCacheSettings> kvp in @this.CacheSettingsByCacheName)
{
// check that the stamp has the capabilities required by the named cache.
if (kvp.Value.RequiredCapabilities != null && kvp.Value.RequiredCapabilities.Count > 0)
{
string missingCaps = string.Join(",", kvp.Value.RequiredCapabilities
.Where(cap => !hostCapabilities.Contains(cap, StringComparer.OrdinalIgnoreCase)));
if (!string.IsNullOrEmpty(missingCaps))
{
logger.Debug(
"Named cache '{0}' was discarded since environment lacks required capabilities: {1}.",
kvp.Key, missingCaps);
continue;
}
}
AbsolutePath rootPath = @this.GetCacheRootPathWithScenario(kvp.Key);
string root = rootPath.GetPathRoot();
if (!checkDriveExists(root))
{
// Currently it's totally fine to have, for instance, both D and K drives configured for the entire stamp,
// even though only some machines in the stamp have both.
// For instance, GlobalCache machines usually do have K drive and that drive is preferred if available,
// but D drive should be used for CommandAgent machines.
// The next trace used to be an error, but in the current state this situation is happening on all CmdAgent machines almost everywhere.
logger.Debug(
"Named cache '{0}' was discarded since the drive required by {1} does not exist or is inaccessible on the machine.",
kvp.Key, rootPath);
continue;
}
filteredCaches.Add(kvp.Key, kvp.Value);
}
result.CacheSettingsByCacheName = filteredCaches;
result.DrivePreferenceOrder = GetSupportedDrivePreferenceOrder(@this.DrivePreferenceOrder, filteredCaches, logger);
if (result.CacheSettingsByCacheName.Count == 0)
{
// It seems that all the cache configs were filtered out. This is bad and the system can't work like that!
string message = $"All ({@this.CacheSettingsByCacheName.Count}) cache configs were discarded due to lack of capabilities. The cache service can't start without valid cache settings.";
throw new CacheException(message);
}
return(result);
}
/// <summary>
/// Variant of <see cref="FilterUnsupportedNamedCaches"/> which does not log.
/// For us in UTs and ConfigCop, where the logging is not plumbed anywhere. (And ommiting the type simplify assembly dependencies)
/// </summary>
/// <param name="this"></param>
/// <param name="hostCapabilities"></param>
/// <param name="driveExistenceOverride"></param>
/// <returns></returns>
public static LocalCasSettings FilterUnsupportedNamedCachesNoLogging(
this LocalCasSettings @this, IEnumerable <string> hostCapabilities, Predicate <string> driveExistenceOverride = null)
{
return(FilterUnsupportedNamedCaches(
@this, hostCapabilities, BuildXL.Cache.ContentStore.Logging.NullLogger.Instance, driveExistenceOverride));
}
public static AbsolutePath GetCacheRootPathWithScenario(
this LocalCasSettings @this, string cacheName)
{
return(new AbsolutePath(
@this.GetCacheRootPath(cacheName, @this.ServiceSettings.ScenarioName ?? DefaultScenario)));
}
internal void Service
(
[Description("Cache names")] string[] names,
[Description("Cache root paths")] string[] paths,
[DefaultValue(DefaultMaxConnections), Description(MaxConnectionsDescription)] uint maxConnections,
[DefaultValue(DefaultGracefulShutdownSeconds), Description(GracefulShutdownSecondsDescription)] uint gracefulShutdownSeconds,
[DefaultValue(ServiceConfiguration.GrpcDisabledPort), Description(GrpcPortDescription)] int grpcPort,
[Description("Name of the memory mapped file used to share GRPC port. 'CASaaS GRPC port' if not specified.")] string grpcPortFileName,
[DefaultValue(null), Description("Writable directory for service operations (use CWD if null)")] string dataRootPath,
[DefaultValue(null), Description("Duration of inactivity after which a session will be timed out.")] double?unusedSessionTimeoutSeconds,
[DefaultValue(null), Description("Duration of inactivity after which a session with a heartbeat will be timed out.")] double?unusedSessionHeartbeatTimeoutSeconds,
[DefaultValue(false), Description("Stop running service")] bool stop,
[DefaultValue(Constants.OneMB), Description("Max size quota in MB")] int maxSizeQuotaMB
)
{
Initialize();
if (stop)
{
IpcUtilities.SetShutdown(_scenario);
return;
}
if (names == null || paths == null)
{
throw new CacheException("At least one cache name/path is required.");
}
if (names.Length != paths.Length)
{
throw new CacheException("Mismatching lengths of names/paths arguments.");
}
var caches = new Dictionary <string, string>();
for (var i = 0; i < names.Length; i++)
{
caches.Add(names[i], paths[i]);
}
var serverDataRootPath = !string.IsNullOrWhiteSpace(dataRootPath)
? new AbsolutePath(dataRootPath)
: new AbsolutePath(Environment.CurrentDirectory);
var cancellationTokenSource = new CancellationTokenSource();
#if !FEATURE_CORECLR
var configuration = new ServiceConfiguration(caches, serverDataRootPath, maxConnections, gracefulShutdownSeconds, grpcPort, grpcPortFileName);
if (!configuration.IsValid)
{
throw new CacheException($"Invalid service configuration, error=[{configuration.Error}]");
}
var localContentServerConfiguration = new LocalServerConfiguration(configuration);
if (unusedSessionTimeoutSeconds != null)
{
localContentServerConfiguration.UnusedSessionTimeout = TimeSpan.FromSeconds(unusedSessionTimeoutSeconds.Value);
}
if (unusedSessionHeartbeatTimeoutSeconds != null)
{
localContentServerConfiguration.UnusedSessionHeartbeatTimeout = TimeSpan.FromSeconds(unusedSessionHeartbeatTimeoutSeconds.Value);
}
if (_scenario != null)
{
_logger.Debug($"scenario=[{_scenario}]");
}
var exitSignal = new ManualResetEvent(false);
Console.CancelKeyPress += (sender, args) =>
{
exitSignal.Set();
args.Cancel = true;
};
using (var exitEvent = IpcUtilities.GetShutdownWaitHandle(_scenario))
{
var server = new LocalContentServer(
_fileSystem,
_logger,
_scenario,
path =>
new FileSystemContentStore(
_fileSystem,
SystemClock.Instance,
path,
new ConfigurationModel(inProcessConfiguration: ContentStoreConfiguration.CreateWithMaxSizeQuotaMB((uint)maxSizeQuotaMB))),
localContentServerConfiguration);
using (server)
{
var context = new Context(_logger);
try
{
var result = server.StartupAsync(context).Result;
if (!result.Succeeded)
{
throw new CacheException(result.ErrorMessage);
}
int completedIndex = WaitHandle.WaitAny(new WaitHandle[] { exitSignal, exitEvent });
var source = completedIndex == 0 ? "control-C" : "exit event";
_tracer.Always(context, $"Shutdown by {source}.");
}
finally
{
var result = server.ShutdownAsync(context).Result;
if (!result.Succeeded)
{
_tracer.Warning(context, $"Failed to shutdown store: {result.ErrorMessage}");
}
}
}
}
#else
Console.CancelKeyPress += (sender, args) =>
{
cancellationTokenSource.Cancel();
args.Cancel = true;
};
var localCasSettings = LocalCasSettings.Default(maxSizeQuotaMB, serverDataRootPath.Path, names[0], (uint)grpcPort);
var distributedContentSettings = DistributedContentSettings.CreateDisabled();
var distributedCacheServiceConfiguration = new DistributedCacheServiceConfiguration(localCasSettings, distributedContentSettings);
// Ensure the computed keyspace is computed based on the hostInfo's StampId
distributedCacheServiceConfiguration.UseStampBasedIsolation = false;
var distributedCacheServiceArguments = new DistributedCacheServiceArguments(
logger: _logger,
copier: null,
pathTransformer: null,
host: new EnvironmentVariableHost(),
hostInfo: new HostInfo(null, null, new List <string>()),
cancellation: cancellationTokenSource.Token,
dataRootPath: serverDataRootPath.Path,
configuration: distributedCacheServiceConfiguration,
keyspace: null);
DistributedCacheServiceFacade.RunAsync(distributedCacheServiceArguments).GetAwaiter().GetResult();
// Because the facade completes immediately and named wait handles don't exist in CORECLR,
// completion here is gated on Control+C. In the future, this can be redone with another option,
// such as a MemoryMappedFile or GRPC heartbeat. This is just intended to be functional.
cancellationTokenSource.Token.WaitHandle.WaitOne();
#endif
}
internal void Service
(
[Description("Cache names")] string[] names,
[Description("Cache root paths")] string[] paths,
[DefaultValue(DefaultMaxConnections), Description(MaxConnectionsDescription)] uint maxConnections,
[DefaultValue(DefaultGracefulShutdownSeconds), Description(GracefulShutdownSecondsDescription)] uint gracefulShutdownSeconds,
[DefaultValue(ServiceConfiguration.GrpcDisabledPort), Description(GrpcPortDescription)] uint grpcPort,
[Description("Name of the memory mapped file used to share GRPC port. 'CASaaS GRPC port' if not specified.")] string grpcPortFileName,
[DefaultValue(null), Description("Writable directory for service operations (use CWD if null)")] string dataRootPath,
[DefaultValue(null), Description("Duration of inactivity after which a session will be timed out.")] double?unusedSessionTimeoutSeconds,
[DefaultValue(null), Description("Duration of inactivity after which a session with a heartbeat will be timed out.")] double?unusedSessionHeartbeatTimeoutSeconds,
[DefaultValue(false), Description("Stop running service")] bool stop,
[DefaultValue(Constants.OneGBInMB), Description("Max size quota in MB")] int maxSizeQuotaMB,
[DefaultValue(ServiceConfiguration.GrpcDisabledPort), Description(RemoteGrpcPortDescription)] uint backingGrpcPort,
[DefaultValue(null), Description("Name of scenario for backing CAS service")] string backingScenario,
[DefaultValue("None"), Description("Ring Id. Used only for telemetry.")] string ringId,
[DefaultValue("None"), Description("Stamp Id. Used only for telemetry.")] string stampId,
[DefaultValue(null), Description("nLog configuration path. If empty, it is disabled")] string nLogConfigurationPath,
[DefaultValue(null), Description("Whether to use Azure Blob logging or not")] string nLogToBlobStorageSecretName,
[DefaultValue(null), Description("If using Azure Blob logging, where to temporarily store logs")] string nLogToBlobStorageWorkspacePath,
[DefaultValue(false), Description("Enable metadata")] bool enableMetadata
)
{
Initialize();
if (stop)
{
IpcUtilities.SetShutdown(_scenario);
return;
}
if (names == null || paths == null)
{
throw new CacheException("At least one cache name/path is required.");
}
if (names.Length != paths.Length)
{
throw new CacheException("Mismatching lengths of names/paths arguments.");
}
var serverDataRootPath = !string.IsNullOrWhiteSpace(dataRootPath)
? new AbsolutePath(dataRootPath)
: new AbsolutePath(Environment.CurrentDirectory);
var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken);
if (_scenario != null)
{
_logger.Debug($"scenario=[{_scenario}]");
}
using var exitEvent = GetExitEvent();
var localCasSettings = LocalCasSettings.Default(maxSizeQuotaMB, paths[0], names[0], grpcPort, grpcPortFileName);
for (int i = 1; i < names.Length; i++)
{
localCasSettings.AddNamedCache(names[i], paths[i]);
}
localCasSettings.ServiceSettings.ScenarioName = _scenario;
if (unusedSessionTimeoutSeconds != null)
{
localCasSettings.ServiceSettings.UnusedSessionTimeoutMinutes = TimeSpan.FromSeconds(unusedSessionTimeoutSeconds.Value).TotalMinutes;
}
if (unusedSessionHeartbeatTimeoutSeconds != null)
{
localCasSettings.ServiceSettings.UnusedSessionHeartbeatTimeoutMinutes = TimeSpan.FromSeconds(unusedSessionHeartbeatTimeoutSeconds.Value).TotalMinutes;
}
var distributedContentSettings = DistributedContentSettings.CreateDisabled();
if (backingGrpcPort != ServiceConfiguration.GrpcDisabledPort)
{
distributedContentSettings.BackingGrpcPort = (int)backingGrpcPort;
distributedContentSettings.BackingScenario = backingScenario;
}
if (enableMetadata)
{
distributedContentSettings.EnableMetadataStore = true;
}
LoggingSettings loggingSettings = null;
if (!string.IsNullOrEmpty(nLogConfigurationPath))
{
loggingSettings = new LoggingSettings()
{
NLogConfigurationPath = nLogConfigurationPath,
Configuration = new AzureBlobStorageLogPublicConfiguration()
{
SecretName = nLogToBlobStorageSecretName,
WorkspaceFolderPath = nLogToBlobStorageWorkspacePath,
}
};
}
var distributedCacheServiceConfiguration = new DistributedCacheServiceConfiguration(localCasSettings, distributedContentSettings, loggingSettings);
// Ensure the computed keyspace is computed based on the hostInfo's StampId
distributedCacheServiceConfiguration.UseStampBasedIsolation = false;
var distributedCacheServiceArguments = new DistributedCacheServiceArguments(
logger: _logger,
copier: new DistributedCopier(),
copyRequester: null,
host: new EnvironmentVariableHost(),
hostInfo: new HostInfo(null, null, new List <string>()),
cancellation: cancellationTokenSource.Token,
dataRootPath: serverDataRootPath.Path,
configuration: distributedCacheServiceConfiguration,
keyspace: null)
{
TelemetryFieldsProvider = new TelemetryFieldsProvider(ringId, stampId, serviceName: "Service"),
};
var runTask = Task.Run(() => DistributedCacheServiceFacade.RunAsync(distributedCacheServiceArguments));
// Because the facade completes immediately and named wait handles don't exist in CORECLR,
// completion here is gated on Control+C. In the future, this can be redone with another option,
// such as a MemoryMappedFile or GRPC heartbeat. This is just intended to be functional.
int completedIndex = WaitHandle.WaitAny(new WaitHandle[] { cancellationTokenSource.Token.WaitHandle, exitEvent });
var source = completedIndex == 0 ? "control-C" : "exit event";
_logger.Always($"Shutdown by {source}.");
if (completedIndex == 1)
{
cancellationTokenSource.Cancel();
}
runTask.GetAwaiter().GetResult();
}
