private void HandleRedisExceptionAndResetMultiplexerIfNeeded(Context context, Exception exception) { if (IsRedisConnectionException(exception)) { // Using double-checked locking approach to reset the connection multiplexer only once. // Checking for greater then or equals because another thread can increment _connectionErrorCount. if (Interlocked.Increment(ref _connectionErrorCount) >= _configuration.RedisConnectionErrorLimit) { lock (_resetConnectionsLock) { // The second read of _connectionErrorCount is a non-interlocked read, but it should be fine because it is happening under the lock. var timeSinceLastReconnect = _clock.UtcNow.Subtract(_lastRedisReconnectDateTime); if (_connectionErrorCount >= _configuration.RedisConnectionErrorLimit && timeSinceLastReconnect >= _configuration.MinReconnectInterval) { // This means that there is no successful operations happening, and all the errors that we're seeing are redis connectivity issues. // This is, effectively, a work-around for the issue in StackExchange.Redis library (https://github.com/StackExchange/StackExchange.Redis/issues/559). _tracer.Warning(context, $"Reset redis connection to {DatabaseName} due to connectivity issues. ConnectionErrorCount={_connectionErrorCount}, RedisConnectionErrorLimit={_configuration.RedisConnectionErrorLimit}, ReconnectCount={_reconnectionCount}, LastReconnectDateTimeUtc={_lastRedisReconnectDateTime}."); _databaseFactory.ResetConnectionMultiplexer(); Interlocked.Exchange(ref _connectionErrorCount, 0); _lastRedisReconnectDateTime = _clock.UtcNow; Interlocked.Increment(ref _reconnectionCount); // In some cases the service can't connect to redis and the only option is to shut down the service. if (_reconnectionCount >= _configuration.RedisReconnectionLimitBeforeServiceRestart) { LifetimeManager.RequestTeardown(context, $"Requesting teardown because redis reconnection limit of {_configuration.RedisReconnectionLimitBeforeServiceRestart} is reached for {DatabaseName}."); } } } } } }
/// <summary> /// Loads a configuration object from preprocessed json and watches files for changes. /// When result config value changes, teardown will be requested. /// </summary> public static TResultConfig LoadAndWatchPreprocessedConfig <TConfig, TResultConfig>( OperationContext context, string configurationPath, HostParameters hostParameters, out string configHash, Func <TConfig, TResultConfig> extractConfig, Action <Context, string> requestTeardown = null, TimeSpan?pollingInterval = null) { requestTeardown ??= (context, reason) => LifetimeManager.RequestTeardown(context, reason); pollingInterval ??= TimeSpan.FromSeconds(5); var config = LoadPreprocessedConfig <TConfig>(configurationPath, out configHash, hostParameters); var resultConfig = extractConfig(config); var resultConfigString = JsonSerializer.Serialize(resultConfig); DeploymentUtilities.WatchFileAsync( configurationPath, context.Token, pollingInterval.Value, onChanged: () => { var newConfig = LoadPreprocessedConfig <TConfig>(configurationPath, out _, hostParameters); var newResultConfig = extractConfig(newConfig); var newResultConfigString = JsonSerializer.Serialize(resultConfig); if (newResultConfigString != resultConfigString) { resultConfigString = newResultConfigString; requestTeardown(context, "Configuration changed: " + configurationPath); } }, onError: ex => { requestTeardown(context, "Error: " + ex.ToString()); }); return(resultConfig); }