public async Task UseReplicatedHashAsyncShouldNotCrashWhenBothRedisInstancesAreFailing()
{
// bug: https://dev.azure.com/mseng/1ES/_workitems/edit/1656837
// It is important to throw non-redis error to not trigger retry strategy.
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = 1, ThrowRedisException = false
};
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb, throwConnectionExceptionOnGet: true);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace, retryCount: 1);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, null);
var context = new OperationContext(new Context(Logger));
var replicatedRedisHashKey = new ReplicatedRedisHashKey("key", new MockReplicatedKeyHost(), new MemoryClock(), raidedDatabaseAdapter);
var error = await replicatedRedisHashKey.UseReplicatedHashAsync(
context,
retryWindow : TimeSpan.FromMinutes(1),
RedisOperation.All,
(batch, key) =>
{
return(batch.StringGetAsync("first"));
}).ShouldBeError();
// The operation should fail gracefully, not with a critical error like contract violation.
error.IsCriticalFailure.Should().BeFalse();
}
public async Task ExecuteBatchOperationRetriesOnRedisExceptions()
{
// Setup test DB configured to fail 2nd query with Redis Exception
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = 2
};
// Setup Redis DB adapter
var testConn = MockRedisDatabaseFactory.CreateConnection(testDb);
var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace);
// Create a batch query
var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);
var first = redisBatch.StringGetAsync("first");
var second = redisBatch.StringGetAsync("second");
// Execute the batch
await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).ShouldBeSuccess();
// Adapter is expected to retry the entire batch if single call fails
Assert.True(testDb.BatchCalled);
Assert.Equal(4, testDb.Calls);
Assert.Null(first.Exception);
Assert.Null(second.Exception);
Assert.Equal("one", await first);
Assert.Equal("two", await second);
}
public async Task ExecuteBatchOperationNoRetryOnRandomExceptions()
{
// Setup test DB configured to fail 2nd query with normal Exception
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = 2,
ThrowRedisException = false,
};
// Setup Redis DB adapter
var testConn = MockRedisDatabaseFactory.CreateConnection(testDb);
var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace);
// Create a batch query
var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);
var first = redisBatch.StringGetAsync("first");
var second = redisBatch.StringGetAsync("second");
// Execute the batch
await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).IgnoreFailure();
// Adapter does not retry in case random exception is thrown
Assert.True(testDb.BatchCalled);
Assert.Equal(2, testDb.Calls);
Assert.NotNull(first.Exception);
Assert.NotNull(second.Exception);
}
public async Task BatchIsCanceledBeforeOperationStarts()
{
// The test checks that if the connection is lost and the new connection is established,
// all the pending operations are cancelled.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
var cts = new CancellationTokenSource();
var context = new OperationContext(new Context(TestGlobal.Logger), cts.Token);
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 1,
// No retries: should fail the operation immediately.
retryCount: 0,
cancelBatchWhenMultiplexerIsClosed: true);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);
// Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang".
var taskCompletionSource = new TaskCompletionSource <HashEntry[]>();
testDb.HashGetAllAsyncTask = taskCompletionSource.Task;
cts.Cancel();
// Using timeout to avoid hangs in the tests if something is wrong with the logic.
var result = await ExecuteGetCheckpointInfoAsync(context, dbAdapter).WithTimeoutAsync(TimeSpan.FromSeconds(1));
result.IsCancelled.Should().BeTrue();
}
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit()
{
// This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
int connectionCount = 0;
bool failWithRedisConnectionError = false;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
connectionCount++;
// Failing connection only when the local is true;
return(MockRedisDatabaseFactory.CreateConnection(
testDb,
testBatch: null,
throwConnectionExceptionOnGet: () => failWithRedisConnectionError));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 2,
retryCount: 1);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);
connectionCount.Should().Be(1);
// The first execution should fail with the connectivity issue.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
failWithRedisConnectionError = false;
// The second execution should recreate the connection.
await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess();
connectionCount.Should().Be(2);
// The connection was recently recreated.
// Introducing the connectivity issue again.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
// The previous call set the flag to reconnect, but the actual reconnect happening on the next call.
connectionCount.Should().Be(2);
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(3);
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(4);
}
public async Task TestRaidedRedisFailureRecovery()
{
// It is important to set ThrowRedisException to false, because redis exceptions are recoverable
// and we don't want to run this test for too long because of exponential back-off recovery algorithm
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = false
};
var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = false
};
// Setup Redis DB adapter
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace);
var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb);
var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter);
var context = new OperationContext(new Context(Logger));
var retryWindow = TimeSpan.FromSeconds(1);
// Running for the first time, both operation should be successful.
var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
r.primary.ShouldBeSuccess();
r.secondary.ShouldBeSuccess();
secondaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
r.primary.ShouldBeSuccess();
// The second redis should fail when we'll try to use it the second time.
r.secondary.ShouldBeError();
primaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
// Now all the instance should fail.
r.primary.ShouldBeError();
r.secondary.ShouldBeSuccess();
primaryDb.FailNextOperation();
secondaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
// Now all the instance should fail.
r.primary.ShouldBeError();
r.secondary.ShouldBeError();
}
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit()
{
// This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.
// Setup test DB configured to fail 2nd query with normal Exception
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
// No queries will fail, instead GetDatabase will throw with RedisConnectionException.
FailingQuery = -1,
};
int numberOfFactoryCalls = 0;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
numberOfFactoryCalls++;
// Failing connection error only from the first instance.
return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null, throwConnectionExceptionOnGet: numberOfFactoryCalls == 1));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var dbAdapter = new RedisDatabaseAdapter(
redisDatabaseFactory,
DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 2,
retryCount: 1);
// Create a batch query
var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);
// Execute the batch
var result = await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken));
// The first execute batch should fail with the connectivity issue.
result.ShouldBeError();
numberOfFactoryCalls.Should().Be(1);
var redisBatch2 = dbAdapter.CreateBatchOperation(RedisOperation.All);
// Then we should recreate the connection and the second one should be successful.
await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch2, default(CancellationToken)).ShouldBeSuccess();
numberOfFactoryCalls.Should().Be(2);
}
public async Task SlowOperationTimesOut()
{
// Setup test DB configured to fail 2nd query with Redis Exception
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1
};
// The second redis will throw RedisException, because we want to use retry strategy here and see the cancellation happening.
var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = true
};
// Setup Redis DB adapter
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace);
var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb);
var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter);
var context = new OperationContext(new Context(Logger));
var retryWindow = TimeSpan.FromSeconds(1);
// All the operations in the secondary instance will fail all the time.
secondaryDb.FailNextOperation(resetFailureAutomatically: false);
var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(
context,
(adapter, token) => ExecuteAsync(context, adapter, token),
retryWindow,
concurrent : true);
r.primary.ShouldBeSuccess();
// The secondary result is null is an indication that the operation was canceled.
r.secondary.Should().BeNull();
}
public async Task BatchIsCancelledOnReconnectForOtherOperation()
{
// The test checks that if the connection is lost and the new connection is established,
// all the pending operations are cancelled.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
var context = new OperationContext(new Context(TestGlobal.Logger));
int connectionCount = 0;
bool failWithRedisConnectionErrorOnce = false;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
connectionCount++;
// Failing connection only when the local is true;
return(MockRedisDatabaseFactory.CreateConnection(
testDb,
testBatch: null,
throwConnectionExceptionOnGet: () =>
{
var oldValue = failWithRedisConnectionErrorOnce;
failWithRedisConnectionErrorOnce = false;
return oldValue;
}));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 1,
// No retries: should fail the operation immediately.
retryCount: 0,
cancelBatchWhenMultiplexerIsClosed: true);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);
connectionCount.Should().Be(1);
// Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang".
var taskCompletionSource = new TaskCompletionSource <HashEntry[]>();
testDb.HashGetAllAsyncTask = taskCompletionSource.Task;
// Running two operations at the same time:
// The first one should get stuck on task completion source's task
// and the second one will fail with connectivity issue, will cause the restart of the multiplexer,
// and will cancel all the existing operations 9including the first one).
var task1 = ExecuteGetCheckpointInfoAsync(context, dbAdapter);
failWithRedisConnectionErrorOnce = true;
var task2 = ExecuteGetCheckpointInfoAsync(context, dbAdapter);
Output.WriteLine("Waiting for the redis operations to finish.");
await Task.WhenAll(task1, task2);
var results = new[] { task1.Result, task2.Result };
var errorCount = results.Count(r => !r.Succeeded && r.ErrorMessage?.Contains("RedisConnectionException") == true);
errorCount.Should().Be(1, $"Should have 1 error with RedisConnectionException. Results: {string.Join(Environment.NewLine, results.Select(r => r.ToString()))}");
var cancelledCount = results.Count(r => r.IsCancelled);
cancelledCount.Should().Be(1, $"Should have 1 cancellation. Results: {string.Join(Environment.NewLine, results.Select(r => r.ToString()))}");
}
public async Task DoNotReconnectTooFrequently()
{
var memoryClock = new MemoryClock();
memoryClock.UtcNow = DateTime.UtcNow;
// This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
int connectionCount = 0;
TimeSpan reconnectInterval = TimeSpan.FromSeconds(10);
bool failWithRedisConnectionError = false;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
connectionCount++;
// Failing connection only when the local is true;
return(MockRedisDatabaseFactory.CreateConnection(
testDb,
testBatch: null,
throwConnectionExceptionOnGet: () => failWithRedisConnectionError));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 2,
retryCount: 1,
minReconnectInterval: reconnectInterval);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration, memoryClock);
connectionCount.Should().Be(1);
// The first execution should fail with the connectivity issue.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
failWithRedisConnectionError = false;
// The second execution should recreate the connection.
await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess();
connectionCount.Should().Be(2);
// The connection was recently recreated.
// Introducing the connectivity issue first.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
// The next call should not trigger the reconnect
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(2);
// Moving the clock forward and the next call should cause a reconnect.
memoryClock.UtcNow += reconnectInterval.Multiply(2);
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
// The previous execution should set the flag to reconnect,
// but the reconnect count is still the same.
connectionCount.Should().Be(2);
// And only during the next call the multiplexer is actually recreated
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(3);
}
