public async Task UseReplicatedHashAsyncShouldNotCrashWhenBothRedisInstancesAreFailing()
{
// bug: https://dev.azure.com/mseng/1ES/_workitems/edit/1656837
// It is important to throw non-redis error to not trigger retry strategy.
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = 1, ThrowRedisException = false
};
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb, throwConnectionExceptionOnGet: () => true);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace, retryCount: 1);
var primaryAdapter = new RedisDatabaseAdapter(
await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection),
adapterConfiguration);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, null);
var context = new OperationContext(new Context(Logger));
var replicatedRedisHashKey = new ReplicatedRedisHashKey("key", new MockReplicatedKeyHost(), new MemoryClock(), raidedDatabaseAdapter);
var error = await replicatedRedisHashKey.UseNonConcurrentReplicatedHashAsync(
context,
retryWindow : TimeSpan.FromMinutes(1),
RedisOperation.All,
(batch, key) =>
{
return(batch.StringGetAsync("first"));
},
timeout : Timeout.InfiniteTimeSpan).ShouldBeError();
// The operation should fail gracefully, not with a critical error like contract violation.
error.IsCriticalFailure.Should().BeFalse();
}
public async Task BatchIsCanceledBeforeOperationStarts()
{
// The test checks that if the connection is lost and the new connection is established,
// all the pending operations are cancelled.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
var cts = new CancellationTokenSource();
var context = new OperationContext(new Context(TestGlobal.Logger), cts.Token);
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 1,
// No retries: should fail the operation immediately.
retryCount: 0,
cancelBatchWhenMultiplexerIsClosed: true);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);
// Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang".
var taskCompletionSource = new TaskCompletionSource <HashEntry[]>();
testDb.HashGetAllAsyncTask = taskCompletionSource.Task;
cts.Cancel();
// Using timeout to avoid hangs in the tests if something is wrong with the logic.
var result = await ExecuteGetCheckpointInfoAsync(context, dbAdapter).WithTimeoutAsync(TimeSpan.FromSeconds(1));
result.IsCancelled.Should().BeTrue();
}
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit()
{
// This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
int connectionCount = 0;
bool failWithRedisConnectionError = false;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
connectionCount++;
// Failing connection only when the local is true;
return(MockRedisDatabaseFactory.CreateConnection(
testDb,
testBatch: null,
throwConnectionExceptionOnGet: () => failWithRedisConnectionError));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 2,
retryCount: 1);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);
connectionCount.Should().Be(1);
// The first execution should fail with the connectivity issue.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
failWithRedisConnectionError = false;
// The second execution should recreate the connection.
await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess();
connectionCount.Should().Be(2);
// The connection was recently recreated.
// Introducing the connectivity issue again.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
// The previous call set the flag to reconnect, but the actual reconnect happening on the next call.
connectionCount.Should().Be(2);
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(3);
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(4);
}
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit()
{
// This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.
// Setup test DB configured to fail 2nd query with normal Exception
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
// No queries will fail, instead GetDatabase will throw with RedisConnectionException.
FailingQuery = -1,
};
int numberOfFactoryCalls = 0;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
numberOfFactoryCalls++;
// Failing connection error only from the first instance.
return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null, throwConnectionExceptionOnGet: numberOfFactoryCalls == 1));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 2,
retryCount: 1);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);
// Create a batch query
var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);
// Execute the batch
var result = await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken));
// The first execute batch should fail with the connectivity issue.
result.ShouldBeError();
numberOfFactoryCalls.Should().Be(1);
var redisBatch2 = dbAdapter.CreateBatchOperation(RedisOperation.All);
// Then we should recreate the connection and the second one should be successful.
await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch2, default(CancellationToken)).ShouldBeSuccess();
numberOfFactoryCalls.Should().Be(2);
}
public async Task BatchIsCancelledOnReconnectForOtherOperation()
{
// The test checks that if the connection is lost and the new connection is established,
// all the pending operations are cancelled.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
var context = new OperationContext(new Context(TestGlobal.Logger));
int connectionCount = 0;
bool failWithRedisConnectionErrorOnce = false;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
connectionCount++;
// Failing connection only when the local is true;
return(MockRedisDatabaseFactory.CreateConnection(
testDb,
testBatch: null,
throwConnectionExceptionOnGet: () =>
{
var oldValue = failWithRedisConnectionErrorOnce;
failWithRedisConnectionErrorOnce = false;
return oldValue;
}));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 1,
// No retries: should fail the operation immediately.
retryCount: 0,
cancelBatchWhenMultiplexerIsClosed: true);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);
connectionCount.Should().Be(1);
// Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang".
var taskCompletionSource = new TaskCompletionSource <HashEntry[]>();
testDb.HashGetAllAsyncTask = taskCompletionSource.Task;
// Running two operations at the same time:
// The first one should get stuck on task completion source's task
// and the second one will fail with connectivity issue, will cause the restart of the multiplexer,
// and will cancel all the existing operations 9including the first one).
var task1 = ExecuteGetCheckpointInfoAsync(context, dbAdapter);
failWithRedisConnectionErrorOnce = true;
var task2 = ExecuteGetCheckpointInfoAsync(context, dbAdapter);
Output.WriteLine("Waiting for the redis operations to finish.");
await Task.WhenAll(task1, task2);
var results = new[] { task1.Result, task2.Result };
var errorCount = results.Count(r => !r.Succeeded && r.ErrorMessage?.Contains("RedisConnectionException") == true);
errorCount.Should().Be(1, $"Should have 1 error with RedisConnectionException. Results: {string.Join(Environment.NewLine, results.Select(r => r.ToString()))}");
var cancelledCount = results.Count(r => r.IsCancelled);
cancelledCount.Should().Be(1, $"Should have 1 cancellation. Results: {string.Join(Environment.NewLine, results.Select(r => r.ToString()))}");
}
public async Task DoNotReconnectTooFrequently()
{
var memoryClock = new MemoryClock();
memoryClock.UtcNow = DateTime.UtcNow;
// This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
int connectionCount = 0;
TimeSpan reconnectInterval = TimeSpan.FromSeconds(10);
bool failWithRedisConnectionError = false;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
connectionCount++;
// Failing connection only when the local is true;
return(MockRedisDatabaseFactory.CreateConnection(
testDb,
testBatch: null,
throwConnectionExceptionOnGet: () => failWithRedisConnectionError));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 2,
retryCount: 1,
minReconnectInterval: reconnectInterval);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration, memoryClock);
connectionCount.Should().Be(1);
// The first execution should fail with the connectivity issue.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
failWithRedisConnectionError = false;
// The second execution should recreate the connection.
await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess();
connectionCount.Should().Be(2);
// The connection was recently recreated.
// Introducing the connectivity issue first.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
// The next call should not trigger the reconnect
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(2);
// Moving the clock forward and the next call should cause a reconnect.
memoryClock.UtcNow += reconnectInterval.Multiply(2);
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
// The previous execution should set the flag to reconnect,
// but the reconnect count is still the same.
connectionCount.Should().Be(2);
// And only during the next call the multiplexer is actually recreated
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(3);
}
