private void HandleRedisExceptionAndResetMultiplexerIfNeeded(Context context, Exception exception)
{
if (IsRedisConnectionException(exception))
{
// Using double-checked locking approach to reset the connection multiplexer only once.
// Checking for greater then or equals because another thread can increment _connectionErrorCount.
if (Interlocked.Increment(ref _connectionErrorCount) >= _configuration.RedisConnectionErrorLimit)
{
lock (_resetConnectionsLock)
{
// The second read of _connectionErrorCount is a non-interlocked read, but it should be fine because it is happening under the lock.
var timeSinceLastReconnect = _clock.UtcNow.Subtract(_lastRedisReconnectDateTime);
if (_connectionErrorCount >= _configuration.RedisConnectionErrorLimit && timeSinceLastReconnect >= _configuration.MinReconnectInterval)
{
// This means that there is no successful operations happening, and all the errors that we're seeing are redis connectivity issues.
// This is, effectively, a work-around for the issue in StackExchange.Redis library (https://github.com/StackExchange/StackExchange.Redis/issues/559).
_tracer.Warning(context, $"Reset redis connection to {DatabaseName} due to connectivity issues. ConnectionErrorCount={_connectionErrorCount}, RedisConnectionErrorLimit={_configuration.RedisConnectionErrorLimit}, ReconnectCount={_reconnectionCount}, LastReconnectDateTimeUtc={_lastRedisReconnectDateTime}.");
_databaseFactory.ResetConnectionMultiplexer();
Interlocked.Exchange(ref _connectionErrorCount, 0);
_lastRedisReconnectDateTime = _clock.UtcNow;
Interlocked.Increment(ref _reconnectionCount);
// In some cases the service can't connect to redis and the only option is to shut down the service.
if (_reconnectionCount >= _configuration.RedisReconnectionLimitBeforeServiceRestart)
{
LifetimeManager.RequestTeardown(context, $"Requesting teardown because redis reconnection limit of {_configuration.RedisReconnectionLimitBeforeServiceRestart} is reached for {DatabaseName}.");
}
}
}
}
}
}
/// <summary>
/// Loads a configuration object from preprocessed json and watches files for changes.
/// When result config value changes, teardown will be requested.
/// </summary>
public static TResultConfig LoadAndWatchPreprocessedConfig <TConfig, TResultConfig>(
OperationContext context,
string configurationPath,
HostParameters hostParameters,
out string configHash,
Func <TConfig, TResultConfig> extractConfig,
Action <Context, string> requestTeardown = null,
TimeSpan?pollingInterval = null)
{
requestTeardown ??= (context, reason) => LifetimeManager.RequestTeardown(context, reason);
pollingInterval ??= TimeSpan.FromSeconds(5);
var config = LoadPreprocessedConfig <TConfig>(configurationPath, out configHash, hostParameters);
var resultConfig = extractConfig(config);
var resultConfigString = JsonSerializer.Serialize(resultConfig);
DeploymentUtilities.WatchFileAsync(
configurationPath,
context.Token,
pollingInterval.Value,
onChanged: () =>
{
var newConfig = LoadPreprocessedConfig <TConfig>(configurationPath, out _, hostParameters);
var newResultConfig = extractConfig(newConfig);
var newResultConfigString = JsonSerializer.Serialize(resultConfig);
if (newResultConfigString != resultConfigString)
{
resultConfigString = newResultConfigString;
requestTeardown(context, "Configuration changed: " + configurationPath);
}
},
onError: ex =>
{
requestTeardown(context, "Error: " + ex.ToString());
});
return(resultConfig);
}
