private void HandleRedisExceptionAndResetMultiplexerIfNeeded(Context context, Exception exception) { if (IsRedisConnectionException(exception)) { // Using double-checked locking approach to reset the connection multiplexer only once. // Checking for greater then or equals because another thread can increment _connectionErrorCount. if (Interlocked.Increment(ref _connectionErrorCount) >= _configuration.RedisConnectionErrorLimit) { lock (_resetConnectionsLock) { // The second read of _connectionErrorCount is a non-interlocked read, but it should be fine because it is happening under the lock. var timeSinceLastReconnect = _clock.UtcNow.Subtract(_lastRedisReconnectDateTime); if (_connectionErrorCount >= _configuration.RedisConnectionErrorLimit && timeSinceLastReconnect >= _configuration.MinReconnectInterval) { // This means that there is no successful operations happening, and all the errors that we're seeing are redis connectivity issues. // This is, effectively, a work-around for the issue in StackExchange.Redis library (https://github.com/StackExchange/StackExchange.Redis/issues/559). _tracer.Warning(context, $"Reset redis connection to {DatabaseName} due to connectivity issues. ConnectionErrorCount={_connectionErrorCount}, RedisConnectionErrorLimit={_configuration.RedisConnectionErrorLimit}, ReconnectCount={_reconnectionCount}, LastReconnectDateTimeUtc={_lastRedisReconnectDateTime}."); _databaseFactory.ResetConnectionMultiplexer(); Interlocked.Exchange(ref _connectionErrorCount, 0); _lastRedisReconnectDateTime = _clock.UtcNow; Interlocked.Increment(ref _reconnectionCount); // In some cases the service can't connect to redis and the only option is to shut down the service. if (_reconnectionCount >= _configuration.RedisReconnectionLimitBeforeServiceRestart) { LifetimeManager.RequestTeardown(context, $"Requesting teardown because redis reconnection limit of {_configuration.RedisReconnectionLimitBeforeServiceRestart} is reached for {DatabaseName}."); } } } } } }
private void HandleRedisExceptionAndResetMultiplexerIfNeeded(Context context, Exception exception) { if (IsRedisConnectionException(exception)) { // Using double-checked locking approach to reset the connection multiplexer only once. // Checking for greater then or equals because another thread can increment _connectionErrorCount. if (Interlocked.Increment(ref _connectionErrorCount) >= _configuration.RedisConnectionErrorLimit) { lock (_resetConnectionsLock) { // The second read of _connectionErrorCount is a non-interlocked read, but it should be fine because it is happening under the lock. if (_connectionErrorCount >= _configuration.RedisConnectionErrorLimit) { // This means that there is no successful operations happening, and all the errors that we're seeing are redis connectivity issues. // This is, effectively, a work-around for the issue in StackExchange.Redis library (https://github.com/StackExchange/StackExchange.Redis/issues/559). context.Warning($"Reset redis connection due to connectivity issues. ConnectionErrorCount={_connectionErrorCount}, RedisConnectionErrorLimit={_configuration.RedisConnectionErrorLimit}."); Interlocked.Exchange(ref _connectionErrorCount, 0); _databaseFactory.ResetConnectionMultiplexer(); } } } } }