/// <inheritdoc /> public async Task <BoolResult> ShutdownAsync(Context context) { ShutdownStarted = true; context.Debug($"Stopping service process {_process.Id} for scenario {_scenario}"); await Task.Run(() => { IpcUtilities.SetShutdown(_scenario); if (!_process.WaitForExit(_waitForExitTimeoutMs)) { context.Warning("Service process failed to exit, killing hard"); try { _process.Kill(); } catch (InvalidOperationException) { // the process may have exited, // in this case ignore the exception } } }); ShutdownCompleted = true; if (_process.ExitCode.HasValue && (_process.ExitCode != 0 || _process.GetStdErr().Length != 0)) { return(new BoolResult($"Process exited with code {_process.ExitCode}. Command line args: {_args}. StdErr: {_process.GetStdErr()} StdOut: {_process.GetStdOut()}")); } return(BoolResult.Success); }
private WaitHandle GetExitEvent() { if (OperatingSystemHelper.IsWindowsOS) { return(IpcUtilities.GetShutdownWaitHandle(_scenario)); } else { // Not supported on non-windows OS. Return no-op wait handle. return(CancellationToken.None.WaitHandle); } }
public void SetAndCloseAnOpenHandle() { var identifier = $"{nameof(IpcUtilitiesTests)}.{nameof(SetAndCloseAnOpenHandle)}"; using (var shutdownHandle = IpcUtilities.GetShutdownWaitHandle(identifier)) { // Open the handle, set it, and close it. This should not close the outer handle. IpcUtilities.SetShutdown(identifier).Should().BeTrue(); // Validate the outer handle is set but not closed. shutdownHandle.WaitOne(100).Should().BeTrue(); } }
public void ClosedHandleCannotBeReopened() { var identifier = $"{nameof(IpcUtilitiesTests)}.{nameof(ClosedHandleCannotBeReopened)}"; // Set the shutdown handle and immediately close it IpcUtilities.SetShutdown(identifier).Should().BeTrue(); using (var dupShutdownHandle = IpcUtilities.GetShutdownWaitHandle(identifier)) { // The handle should NOT be set dupShutdownHandle.WaitOne(100).Should().BeFalse(); } }
/// <summary> /// Check if the service is running. /// </summary> public static bool EnsureRunning(Context context, string scenario, int waitMs) { var currentUser = UserUtilities.CurrentUserName(); #if !PLATFORM_OSX context.TraceMessage(Severity.Debug, $"Opening ready event name=[{scenario}] for user=[{currentUser}] waitMs={waitMs}."); var stopwatch = Stopwatch.StartNew(); bool running = ensureRunningCore(); if (running) { context.TraceMessage(Severity.Debug, $"Opened ready event name=[{scenario}] by {stopwatch.ElapsedMilliseconds}ms."); } return(running); bool ensureRunningCore() { try { if (!IpcUtilities.TryOpenExistingReadyWaitHandle(scenario, out EventWaitHandle readyEvent, waitMs)) { context.TraceMessage(Severity.Debug, "Ready event does not exist"); return(false); } using (readyEvent) { var waitMsRemaining = Math.Max(0, waitMs - (int)stopwatch.ElapsedMilliseconds); if (!readyEvent.WaitOne(waitMsRemaining)) { context.TraceMessage(Severity.Debug, "Ready event was opened but remained unset until timeout"); return(false); } return(true); } } catch (UnauthorizedAccessException) { context.TraceMessage(Severity.Debug, "Ready event exists, but user does not have acceptable security access"); } return(false); } #else context.TraceMessage(Severity.Debug, $"Not validating ready event (OSX) name=[{scenario}] for user=[{currentUser}] waitMs={waitMs}"); return(true); #endif }
private static EventWaitHandle CreateShutdownEvent(ILogger logger, string scenario) { var currentUser = UserUtilities.CurrentUserName(); logger.Debug($"Creating shutdown event name=[{scenario}] for user=[{currentUser}]"); var shutdownEvent = IpcUtilities.GetShutdownWaitHandle(scenario); if (shutdownEvent.WaitOne(0)) { shutdownEvent.Dispose(); throw new CacheException($"Shutdown event name=[{scenario}] already exists"); } return(shutdownEvent); }
private static EventWaitHandle CreateReadyEvent(ILogger logger, string scenario) { var currentUser = UserUtilities.CurrentUserName(); logger.Debug($"Creating ready event name=[{scenario}] for user=[{currentUser}]"); var readyEvent = IpcUtilities.GetReadyWaitHandle(scenario); if (readyEvent.WaitOne(0)) { readyEvent.Dispose(); throw new CacheException("A service is already running"); } return(readyEvent); }
public void TwoHandlesWithSameNameAreIdentical() { var identifier = $"{nameof(IpcUtilitiesTests)}.{nameof(TwoHandlesWithSameNameAreIdentical)}"; using (var shutdownHandle = IpcUtilities.GetShutdownWaitHandle(identifier)) { // Open the handle for one thread shutdownHandle.Set().Should().BeTrue(); using (var dupShutdownHandle = IpcUtilities.GetShutdownWaitHandle(identifier)) { // Use handle's signal and reset dupShutdownHandle.WaitOne(100).Should().BeTrue(); } } }
public void IpcDoestNotLastAfterDispose() { var identifier = $"{nameof(IpcUtilitiesTests)}.{nameof(IpcDoestNotLastAfterDispose)}"; using (var readyHandle = IpcUtilities.GetReadyWaitHandle(identifier)) { // Ready handle uses ManualReset, so this should leave the handle open for all readyHandle.Set(); readyHandle.WaitOne(100).Should().BeTrue(); } var found = IpcUtilities.TryOpenExistingReadyWaitHandle(identifier, out EventWaitHandle existingReadyHandle); found.Should().BeFalse(); existingReadyHandle.Should().BeNull(); }
/// <summary> /// Attempt to open event that will signal an imminent service shutdown or restart. /// </summary> public static EventWaitHandle OpenShutdownEvent(Context context, string scenario) { var currentUser = UserUtilities.CurrentUserName(); context.TraceMessage(Severity.Debug, $"Opening shutdown event name=[{scenario}] for user=[{currentUser}]"); EventWaitHandle handle = null; try { if (!IpcUtilities.TryOpenExistingShutdownWaitHandle(scenario, out handle)) { context.TraceMessage(Severity.Debug, "Shutdown event does not exist"); } } catch (UnauthorizedAccessException) { context.TraceMessage(Severity.Debug, "Shutdown event exists, but user does not have acceptable security access"); } return(handle); }
internal void Service ( [Description("Cache names")] string[] names, [Description("Cache root paths")] string[] paths, [DefaultValue(DefaultMaxConnections), Description(MaxConnectionsDescription)] uint maxConnections, [DefaultValue(DefaultGracefulShutdownSeconds), Description(GracefulShutdownSecondsDescription)] uint gracefulShutdownSeconds, [DefaultValue(ServiceConfiguration.GrpcDisabledPort), Description(GrpcPortDescription)] int grpcPort, [Description("Name of the memory mapped file used to share GRPC port. 'CASaaS GRPC port' if not specified.")] string grpcPortFileName, [DefaultValue(null), Description("Writable directory for service operations (use CWD if null)")] string dataRootPath, [DefaultValue(null), Description("Duration of inactivity after which a session will be timed out.")] double?unusedSessionTimeoutSeconds, [DefaultValue(null), Description("Duration of inactivity after which a session with a heartbeat will be timed out.")] double?unusedSessionHeartbeatTimeoutSeconds, [DefaultValue(false), Description("Stop running service")] bool stop, [DefaultValue(Constants.OneMB), Description("Max size quota in MB")] int maxSizeQuotaMB ) { Initialize(); if (stop) { IpcUtilities.SetShutdown(_scenario); return; } if (names == null || paths == null) { throw new CacheException("At least one cache name/path is required."); } if (names.Length != paths.Length) { throw new CacheException("Mismatching lengths of names/paths arguments."); } var caches = new Dictionary <string, string>(); for (var i = 0; i < names.Length; i++) { caches.Add(names[i], paths[i]); } var serverDataRootPath = !string.IsNullOrWhiteSpace(dataRootPath) ? new AbsolutePath(dataRootPath) : new AbsolutePath(Environment.CurrentDirectory); var cancellationTokenSource = new CancellationTokenSource(); #if !FEATURE_CORECLR var configuration = new ServiceConfiguration(caches, serverDataRootPath, maxConnections, gracefulShutdownSeconds, grpcPort, grpcPortFileName); if (!configuration.IsValid) { throw new CacheException($"Invalid service configuration, error=[{configuration.Error}]"); } var localContentServerConfiguration = new LocalServerConfiguration(configuration); if (unusedSessionTimeoutSeconds != null) { localContentServerConfiguration.UnusedSessionTimeout = TimeSpan.FromSeconds(unusedSessionTimeoutSeconds.Value); } if (unusedSessionHeartbeatTimeoutSeconds != null) { localContentServerConfiguration.UnusedSessionHeartbeatTimeout = TimeSpan.FromSeconds(unusedSessionHeartbeatTimeoutSeconds.Value); } if (_scenario != null) { _logger.Debug($"scenario=[{_scenario}]"); } var exitSignal = new ManualResetEvent(false); Console.CancelKeyPress += (sender, args) => { exitSignal.Set(); args.Cancel = true; }; using (var exitEvent = IpcUtilities.GetShutdownWaitHandle(_scenario)) { var server = new LocalContentServer( _fileSystem, _logger, _scenario, path => new FileSystemContentStore( _fileSystem, SystemClock.Instance, path, new ConfigurationModel(inProcessConfiguration: ContentStoreConfiguration.CreateWithMaxSizeQuotaMB((uint)maxSizeQuotaMB))), localContentServerConfiguration); using (server) { var context = new Context(_logger); try { var result = server.StartupAsync(context).Result; if (!result.Succeeded) { throw new CacheException(result.ErrorMessage); } int completedIndex = WaitHandle.WaitAny(new WaitHandle[] { exitSignal, exitEvent }); var source = completedIndex == 0 ? "control-C" : "exit event"; _tracer.Always(context, $"Shutdown by {source}."); } finally { var result = server.ShutdownAsync(context).Result; if (!result.Succeeded) { _tracer.Warning(context, $"Failed to shutdown store: {result.ErrorMessage}"); } } } } #else Console.CancelKeyPress += (sender, args) => { cancellationTokenSource.Cancel(); args.Cancel = true; }; var localCasSettings = LocalCasSettings.Default(maxSizeQuotaMB, serverDataRootPath.Path, names[0], (uint)grpcPort); var distributedContentSettings = DistributedContentSettings.CreateDisabled(); var distributedCacheServiceConfiguration = new DistributedCacheServiceConfiguration(localCasSettings, distributedContentSettings); // Ensure the computed keyspace is computed based on the hostInfo's StampId distributedCacheServiceConfiguration.UseStampBasedIsolation = false; var distributedCacheServiceArguments = new DistributedCacheServiceArguments( logger: _logger, copier: null, pathTransformer: null, host: new EnvironmentVariableHost(), hostInfo: new HostInfo(null, null, new List <string>()), cancellation: cancellationTokenSource.Token, dataRootPath: serverDataRootPath.Path, configuration: distributedCacheServiceConfiguration, keyspace: null); DistributedCacheServiceFacade.RunAsync(distributedCacheServiceArguments).GetAwaiter().GetResult(); // Because the facade completes immediately and named wait handles don't exist in CORECLR, // completion here is gated on Control+C. In the future, this can be redone with another option, // such as a MemoryMappedFile or GRPC heartbeat. This is just intended to be functional. cancellationTokenSource.Token.WaitHandle.WaitOne(); #endif }
internal void Service ( [Description("Cache names")] string[] names, [Description("Cache root paths")] string[] paths, [DefaultValue(DefaultMaxConnections), Description(MaxConnectionsDescription)] uint maxConnections, [DefaultValue(DefaultGracefulShutdownSeconds), Description(GracefulShutdownSecondsDescription)] uint gracefulShutdownSeconds, [DefaultValue(ServiceConfiguration.GrpcDisabledPort), Description(GrpcPortDescription)] uint grpcPort, [Description("Name of the memory mapped file used to share GRPC port. 'CASaaS GRPC port' if not specified.")] string grpcPortFileName, [DefaultValue(null), Description("Writable directory for service operations (use CWD if null)")] string dataRootPath, [DefaultValue(null), Description("Duration of inactivity after which a session will be timed out.")] double?unusedSessionTimeoutSeconds, [DefaultValue(null), Description("Duration of inactivity after which a session with a heartbeat will be timed out.")] double?unusedSessionHeartbeatTimeoutSeconds, [DefaultValue(false), Description("Stop running service")] bool stop, [DefaultValue(Constants.OneGBInMB), Description("Max size quota in MB")] int maxSizeQuotaMB, [DefaultValue(ServiceConfiguration.GrpcDisabledPort), Description(RemoteGrpcPortDescription)] uint backingGrpcPort, [DefaultValue(null), Description("Name of scenario for backing CAS service")] string backingScenario, [DefaultValue("None"), Description("Ring Id. Used only for telemetry.")] string ringId, [DefaultValue("None"), Description("Stamp Id. Used only for telemetry.")] string stampId, [DefaultValue(null), Description("nLog configuration path. If empty, it is disabled")] string nLogConfigurationPath, [DefaultValue(null), Description("Whether to use Azure Blob logging or not")] string nLogToBlobStorageSecretName, [DefaultValue(null), Description("If using Azure Blob logging, where to temporarily store logs")] string nLogToBlobStorageWorkspacePath, [DefaultValue(false), Description("Enable metadata")] bool enableMetadata ) { Initialize(); if (stop) { IpcUtilities.SetShutdown(_scenario); return; } if (names == null || paths == null) { throw new CacheException("At least one cache name/path is required."); } if (names.Length != paths.Length) { throw new CacheException("Mismatching lengths of names/paths arguments."); } var serverDataRootPath = !string.IsNullOrWhiteSpace(dataRootPath) ? new AbsolutePath(dataRootPath) : new AbsolutePath(Environment.CurrentDirectory); var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken); if (_scenario != null) { _logger.Debug($"scenario=[{_scenario}]"); } using var exitEvent = GetExitEvent(); var localCasSettings = LocalCasSettings.Default(maxSizeQuotaMB, paths[0], names[0], grpcPort, grpcPortFileName); for (int i = 1; i < names.Length; i++) { localCasSettings.AddNamedCache(names[i], paths[i]); } localCasSettings.ServiceSettings.ScenarioName = _scenario; if (unusedSessionTimeoutSeconds != null) { localCasSettings.ServiceSettings.UnusedSessionTimeoutMinutes = TimeSpan.FromSeconds(unusedSessionTimeoutSeconds.Value).TotalMinutes; } if (unusedSessionHeartbeatTimeoutSeconds != null) { localCasSettings.ServiceSettings.UnusedSessionHeartbeatTimeoutMinutes = TimeSpan.FromSeconds(unusedSessionHeartbeatTimeoutSeconds.Value).TotalMinutes; } var distributedContentSettings = DistributedContentSettings.CreateDisabled(); if (backingGrpcPort != ServiceConfiguration.GrpcDisabledPort) { distributedContentSettings.BackingGrpcPort = (int)backingGrpcPort; distributedContentSettings.BackingScenario = backingScenario; } if (enableMetadata) { distributedContentSettings.EnableMetadataStore = true; } LoggingSettings loggingSettings = null; if (!string.IsNullOrEmpty(nLogConfigurationPath)) { loggingSettings = new LoggingSettings() { NLogConfigurationPath = nLogConfigurationPath, Configuration = new AzureBlobStorageLogPublicConfiguration() { SecretName = nLogToBlobStorageSecretName, WorkspaceFolderPath = nLogToBlobStorageWorkspacePath, } }; } var distributedCacheServiceConfiguration = new DistributedCacheServiceConfiguration(localCasSettings, distributedContentSettings, loggingSettings); // Ensure the computed keyspace is computed based on the hostInfo's StampId distributedCacheServiceConfiguration.UseStampBasedIsolation = false; var distributedCacheServiceArguments = new DistributedCacheServiceArguments( logger: _logger, copier: new DistributedCopier(), copyRequester: null, host: new EnvironmentVariableHost(), hostInfo: new HostInfo(null, null, new List <string>()), cancellation: cancellationTokenSource.Token, dataRootPath: serverDataRootPath.Path, configuration: distributedCacheServiceConfiguration, keyspace: null) { TelemetryFieldsProvider = new TelemetryFieldsProvider(ringId, stampId, serviceName: "Service"), }; var runTask = Task.Run(() => DistributedCacheServiceFacade.RunAsync(distributedCacheServiceArguments)); // Because the facade completes immediately and named wait handles don't exist in CORECLR, // completion here is gated on Control+C. In the future, this can be redone with another option, // such as a MemoryMappedFile or GRPC heartbeat. This is just intended to be functional. int completedIndex = WaitHandle.WaitAny(new WaitHandle[] { cancellationTokenSource.Token.WaitHandle, exitEvent }); var source = completedIndex == 0 ? "control-C" : "exit event"; _logger.Always($"Shutdown by {source}."); if (completedIndex == 1) { cancellationTokenSource.Cancel(); } runTask.GetAwaiter().GetResult(); }