public async Task UseReplicatedHashAsyncShouldNotCrashWhenBothRedisInstancesAreFailing() { // bug: https://dev.azure.com/mseng/1ES/_workitems/edit/1656837 // It is important to throw non-redis error to not trigger retry strategy. var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = 1, ThrowRedisException = false }; var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb, throwConnectionExceptionOnGet: true); var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace, retryCount: 1); var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, null); var context = new OperationContext(new Context(Logger)); var replicatedRedisHashKey = new ReplicatedRedisHashKey("key", new MockReplicatedKeyHost(), new MemoryClock(), raidedDatabaseAdapter); var error = await replicatedRedisHashKey.UseReplicatedHashAsync( context, retryWindow : TimeSpan.FromMinutes(1), RedisOperation.All, (batch, key) => { return(batch.StringGetAsync("first")); }).ShouldBeError(); // The operation should fail gracefully, not with a critical error like contract violation. error.IsCriticalFailure.Should().BeFalse(); }
public async Task ExecuteBatchOperationRetriesOnRedisExceptions() { // Setup test DB configured to fail 2nd query with Redis Exception var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = 2 }; // Setup Redis DB adapter var testConn = MockRedisDatabaseFactory.CreateConnection(testDb); var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace); // Create a batch query var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All); var first = redisBatch.StringGetAsync("first"); var second = redisBatch.StringGetAsync("second"); // Execute the batch await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).ShouldBeSuccess(); // Adapter is expected to retry the entire batch if single call fails Assert.True(testDb.BatchCalled); Assert.Equal(4, testDb.Calls); Assert.Null(first.Exception); Assert.Null(second.Exception); Assert.Equal("one", await first); Assert.Equal("two", await second); }
public async Task ExecuteBatchOperationNoRetryOnRandomExceptions() { // Setup test DB configured to fail 2nd query with normal Exception var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = 2, ThrowRedisException = false, }; // Setup Redis DB adapter var testConn = MockRedisDatabaseFactory.CreateConnection(testDb); var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace); // Create a batch query var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All); var first = redisBatch.StringGetAsync("first"); var second = redisBatch.StringGetAsync("second"); // Execute the batch await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).IgnoreFailure(); // Adapter does not retry in case random exception is thrown Assert.True(testDb.BatchCalled); Assert.Equal(2, testDb.Calls); Assert.NotNull(first.Exception); Assert.NotNull(second.Exception); }
public async Task BatchIsCanceledBeforeOperationStarts() { // The test checks that if the connection is lost and the new connection is established, // all the pending operations are cancelled. var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData); var cts = new CancellationTokenSource(); var context = new OperationContext(new Context(TestGlobal.Logger), cts.Token); // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null)); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 1, // No retries: should fail the operation immediately. retryCount: 0, cancelBatchWhenMultiplexerIsClosed: true); var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration); // Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang". var taskCompletionSource = new TaskCompletionSource <HashEntry[]>(); testDb.HashGetAllAsyncTask = taskCompletionSource.Task; cts.Cancel(); // Using timeout to avoid hangs in the tests if something is wrong with the logic. var result = await ExecuteGetCheckpointInfoAsync(context, dbAdapter).WithTimeoutAsync(TimeSpan.FromSeconds(1)); result.IsCancelled.Should().BeTrue(); }
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit() { // This test checks that if the client fails to connect to redis, it'll successfully reconnect to it. var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData); int connectionCount = 0; bool failWithRedisConnectionError = false; // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { connectionCount++; // Failing connection only when the local is true; return(MockRedisDatabaseFactory.CreateConnection( testDb, testBatch: null, throwConnectionExceptionOnGet: () => failWithRedisConnectionError)); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 2, retryCount: 1); var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration); connectionCount.Should().Be(1); // The first execution should fail with the connectivity issue. failWithRedisConnectionError = true; await ExecuteBatchAsync(dbAdapter).ShouldBeError(); failWithRedisConnectionError = false; // The second execution should recreate the connection. await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess(); connectionCount.Should().Be(2); // The connection was recently recreated. // Introducing the connectivity issue again. failWithRedisConnectionError = true; await ExecuteBatchAsync(dbAdapter).ShouldBeError(); // The previous call set the flag to reconnect, but the actual reconnect happening on the next call. connectionCount.Should().Be(2); await ExecuteBatchAsync(dbAdapter).ShouldBeError(); connectionCount.Should().Be(3); await ExecuteBatchAsync(dbAdapter).ShouldBeError(); connectionCount.Should().Be(4); }
public async Task TestRaidedRedisFailureRecovery() { // It is important to set ThrowRedisException to false, because redis exceptions are recoverable // and we don't want to run this test for too long because of exponential back-off recovery algorithm var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = -1, ThrowRedisException = false }; var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = -1, ThrowRedisException = false }; // Setup Redis DB adapter var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb); var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace); var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb); var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace); var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter); var context = new OperationContext(new Context(Logger)); var retryWindow = TimeSpan.FromSeconds(1); // Running for the first time, both operation should be successful. var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); r.primary.ShouldBeSuccess(); r.secondary.ShouldBeSuccess(); secondaryDb.FailNextOperation(); r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); r.primary.ShouldBeSuccess(); // The second redis should fail when we'll try to use it the second time. r.secondary.ShouldBeError(); primaryDb.FailNextOperation(); r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); // Now all the instance should fail. r.primary.ShouldBeError(); r.secondary.ShouldBeSuccess(); primaryDb.FailNextOperation(); secondaryDb.FailNextOperation(); r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); // Now all the instance should fail. r.primary.ShouldBeError(); r.secondary.ShouldBeError(); }
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit() { // This test checks that if the client fails to connect to redis, it'll successfully reconnect to it. // Setup test DB configured to fail 2nd query with normal Exception var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { // No queries will fail, instead GetDatabase will throw with RedisConnectionException. FailingQuery = -1, }; int numberOfFactoryCalls = 0; // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { numberOfFactoryCalls++; // Failing connection error only from the first instance. return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null, throwConnectionExceptionOnGet: numberOfFactoryCalls == 1)); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var dbAdapter = new RedisDatabaseAdapter( redisDatabaseFactory, DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 2, retryCount: 1); // Create a batch query var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All); // Execute the batch var result = await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)); // The first execute batch should fail with the connectivity issue. result.ShouldBeError(); numberOfFactoryCalls.Should().Be(1); var redisBatch2 = dbAdapter.CreateBatchOperation(RedisOperation.All); // Then we should recreate the connection and the second one should be successful. await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch2, default(CancellationToken)).ShouldBeSuccess(); numberOfFactoryCalls.Should().Be(2); }
public async Task SlowOperationTimesOut() { // Setup test DB configured to fail 2nd query with Redis Exception var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = -1 }; // The second redis will throw RedisException, because we want to use retry strategy here and see the cancellation happening. var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = -1, ThrowRedisException = true }; // Setup Redis DB adapter var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb); var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace); var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb); var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace); var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter); var context = new OperationContext(new Context(Logger)); var retryWindow = TimeSpan.FromSeconds(1); // All the operations in the secondary instance will fail all the time. secondaryDb.FailNextOperation(resetFailureAutomatically: false); var r = await raidedDatabaseAdapter.ExecuteRaidedAsync( context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); r.primary.ShouldBeSuccess(); // The secondary result is null is an indication that the operation was canceled. r.secondary.Should().BeNull(); }
public async Task BatchIsCancelledOnReconnectForOtherOperation() { // The test checks that if the connection is lost and the new connection is established, // all the pending operations are cancelled. var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData); var context = new OperationContext(new Context(TestGlobal.Logger)); int connectionCount = 0; bool failWithRedisConnectionErrorOnce = false; // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { connectionCount++; // Failing connection only when the local is true; return(MockRedisDatabaseFactory.CreateConnection( testDb, testBatch: null, throwConnectionExceptionOnGet: () => { var oldValue = failWithRedisConnectionErrorOnce; failWithRedisConnectionErrorOnce = false; return oldValue; })); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 1, // No retries: should fail the operation immediately. retryCount: 0, cancelBatchWhenMultiplexerIsClosed: true); var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration); connectionCount.Should().Be(1); // Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang". var taskCompletionSource = new TaskCompletionSource <HashEntry[]>(); testDb.HashGetAllAsyncTask = taskCompletionSource.Task; // Running two operations at the same time: // The first one should get stuck on task completion source's task // and the second one will fail with connectivity issue, will cause the restart of the multiplexer, // and will cancel all the existing operations 9including the first one). var task1 = ExecuteGetCheckpointInfoAsync(context, dbAdapter); failWithRedisConnectionErrorOnce = true; var task2 = ExecuteGetCheckpointInfoAsync(context, dbAdapter); Output.WriteLine("Waiting for the redis operations to finish."); await Task.WhenAll(task1, task2); var results = new[] { task1.Result, task2.Result }; var errorCount = results.Count(r => !r.Succeeded && r.ErrorMessage?.Contains("RedisConnectionException") == true); errorCount.Should().Be(1, $"Should have 1 error with RedisConnectionException. Results: {string.Join(Environment.NewLine, results.Select(r => r.ToString()))}"); var cancelledCount = results.Count(r => r.IsCancelled); cancelledCount.Should().Be(1, $"Should have 1 cancellation. Results: {string.Join(Environment.NewLine, results.Select(r => r.ToString()))}"); }
public async Task DoNotReconnectTooFrequently() { var memoryClock = new MemoryClock(); memoryClock.UtcNow = DateTime.UtcNow; // This test checks that if the client fails to connect to redis, it'll successfully reconnect to it. var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData); int connectionCount = 0; TimeSpan reconnectInterval = TimeSpan.FromSeconds(10); bool failWithRedisConnectionError = false; // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { connectionCount++; // Failing connection only when the local is true; return(MockRedisDatabaseFactory.CreateConnection( testDb, testBatch: null, throwConnectionExceptionOnGet: () => failWithRedisConnectionError)); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 2, retryCount: 1, minReconnectInterval: reconnectInterval); var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration, memoryClock); connectionCount.Should().Be(1); // The first execution should fail with the connectivity issue. failWithRedisConnectionError = true; await ExecuteBatchAsync(dbAdapter).ShouldBeError(); failWithRedisConnectionError = false; // The second execution should recreate the connection. await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess(); connectionCount.Should().Be(2); // The connection was recently recreated. // Introducing the connectivity issue first. failWithRedisConnectionError = true; await ExecuteBatchAsync(dbAdapter).ShouldBeError(); // The next call should not trigger the reconnect await ExecuteBatchAsync(dbAdapter).ShouldBeError(); connectionCount.Should().Be(2); // Moving the clock forward and the next call should cause a reconnect. memoryClock.UtcNow += reconnectInterval.Multiply(2); await ExecuteBatchAsync(dbAdapter).ShouldBeError(); // The previous execution should set the flag to reconnect, // but the reconnect count is still the same. connectionCount.Should().Be(2); // And only during the next call the multiplexer is actually recreated await ExecuteBatchAsync(dbAdapter).ShouldBeError(); connectionCount.Should().Be(3); }