private void HandleRedisExceptionAndResetMultiplexerIfNeeded(Context context, Exception exception)
{
if (IsRedisConnectionException(exception))
{
// Using double-checked locking approach to reset the connection multiplexer only once.
// Checking for greater then or equals because another thread can increment _connectionErrorCount.
if (Interlocked.Increment(ref _connectionErrorCount) >= _configuration.RedisConnectionErrorLimit)
{
lock (_resetConnectionsLock)
{
// The second read of _connectionErrorCount is a non-interlocked read, but it should be fine because it is happening under the lock.
var timeSinceLastReconnect = _clock.UtcNow.Subtract(_lastRedisReconnectDateTime);
if (_connectionErrorCount >= _configuration.RedisConnectionErrorLimit && timeSinceLastReconnect >= _configuration.MinReconnectInterval)
{
// This means that there is no successful operations happening, and all the errors that we're seeing are redis connectivity issues.
// This is, effectively, a work-around for the issue in StackExchange.Redis library (https://github.com/StackExchange/StackExchange.Redis/issues/559).
_tracer.Warning(context, $"Reset redis connection to {DatabaseName} due to connectivity issues. ConnectionErrorCount={_connectionErrorCount}, RedisConnectionErrorLimit={_configuration.RedisConnectionErrorLimit}, ReconnectCount={_reconnectionCount}, LastReconnectDateTimeUtc={_lastRedisReconnectDateTime}.");
_databaseFactory.ResetConnectionMultiplexer();
Interlocked.Exchange(ref _connectionErrorCount, 0);
_lastRedisReconnectDateTime = _clock.UtcNow;
Interlocked.Increment(ref _reconnectionCount);
// In some cases the service can't connect to redis and the only option is to shut down the service.
if (_reconnectionCount >= _configuration.RedisReconnectionLimitBeforeServiceRestart)
{
LifetimeManager.RequestTeardown(context, $"Requesting teardown because redis reconnection limit of {_configuration.RedisReconnectionLimitBeforeServiceRestart} is reached for {DatabaseName}.");
}
}
}
}
}
}
private void HandleRedisExceptionAndResetMultiplexerIfNeeded(Context context, Exception exception)
{
if (IsRedisConnectionException(exception))
{
// Using double-checked locking approach to reset the connection multiplexer only once.
// Checking for greater then or equals because another thread can increment _connectionErrorCount.
if (Interlocked.Increment(ref _connectionErrorCount) >= _configuration.RedisConnectionErrorLimit)
{
lock (_resetConnectionsLock)
{
// The second read of _connectionErrorCount is a non-interlocked read, but it should be fine because it is happening under the lock.
if (_connectionErrorCount >= _configuration.RedisConnectionErrorLimit)
{
// This means that there is no successful operations happening, and all the errors that we're seeing are redis connectivity issues.
// This is, effectively, a work-around for the issue in StackExchange.Redis library (https://github.com/StackExchange/StackExchange.Redis/issues/559).
context.Warning($"Reset redis connection due to connectivity issues. ConnectionErrorCount={_connectionErrorCount}, RedisConnectionErrorLimit={_configuration.RedisConnectionErrorLimit}.");
Interlocked.Exchange(ref _connectionErrorCount, 0);
_databaseFactory.ResetConnectionMultiplexer();
}
}
}
}
}
