Example #1
0
        public async Task UseReplicatedHashAsyncShouldNotCrashWhenBothRedisInstancesAreFailing()
        {
            // bug: https://dev.azure.com/mseng/1ES/_workitems/edit/1656837

            // It is important to throw non-redis error to not trigger retry strategy.
            var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
            {
                FailingQuery = 1, ThrowRedisException = false
            };

            var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb, throwConnectionExceptionOnGet: true);
            var primaryAdapter    = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace, retryCount: 1);

            var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, null);
            var context = new OperationContext(new Context(Logger));

            var replicatedRedisHashKey = new ReplicatedRedisHashKey("key", new MockReplicatedKeyHost(), new MemoryClock(), raidedDatabaseAdapter);
            var error = await replicatedRedisHashKey.UseReplicatedHashAsync(
                context,
                retryWindow : TimeSpan.FromMinutes(1),
                RedisOperation.All,
                (batch, key) =>
            {
                return(batch.StringGetAsync("first"));
            }).ShouldBeError();

            // The operation should fail gracefully, not with a critical error like contract violation.
            error.IsCriticalFailure.Should().BeFalse();
        }
Example #2
0
        public async Task ExecuteBatchOperationRetriesOnRedisExceptions()
        {
            // Setup test DB configured to fail 2nd query with Redis Exception
            var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
            {
                FailingQuery = 2
            };

            // Setup Redis DB adapter
            var testConn  = MockRedisDatabaseFactory.CreateConnection(testDb);
            var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace);

            // Create a batch query
            var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);
            var first      = redisBatch.StringGetAsync("first");
            var second     = redisBatch.StringGetAsync("second");

            // Execute the batch
            await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).ShouldBeSuccess();

            // Adapter is expected to retry the entire batch if single call fails
            Assert.True(testDb.BatchCalled);
            Assert.Equal(4, testDb.Calls);
            Assert.Null(first.Exception);
            Assert.Null(second.Exception);
            Assert.Equal("one", await first);
            Assert.Equal("two", await second);
        }
Example #3
0
        public async Task ExecuteBatchOperationNoRetryOnRandomExceptions()
        {
            // Setup test DB configured to fail 2nd query with normal Exception
            var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
            {
                FailingQuery        = 2,
                ThrowRedisException = false,
            };

            // Setup Redis DB adapter
            var testConn  = MockRedisDatabaseFactory.CreateConnection(testDb);
            var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace);

            // Create a batch query
            var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);
            var first      = redisBatch.StringGetAsync("first");
            var second     = redisBatch.StringGetAsync("second");

            // Execute the batch
            await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).IgnoreFailure();

            // Adapter does not retry in case random exception is thrown
            Assert.True(testDb.BatchCalled);
            Assert.Equal(2, testDb.Calls);
            Assert.NotNull(first.Exception);
            Assert.NotNull(second.Exception);
        }
        public async Task BatchIsCanceledBeforeOperationStarts()
        {
            // The test checks that if the connection is lost and the new connection is established,
            // all the pending operations are cancelled.

            var testDb  = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
            var cts     = new CancellationTokenSource();
            var context = new OperationContext(new Context(TestGlobal.Logger), cts.Token);

            // Setup Redis DB adapter
            Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
            {
                return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null));
            };

            var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);

            var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
                                                                             // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
                                                                             redisConnectionErrorLimit: 1,
                                                                             // No retries: should fail the operation immediately.
                                                                             retryCount: 0,
                                                                             cancelBatchWhenMultiplexerIsClosed: true);
            var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);

            // Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang".
            var taskCompletionSource = new TaskCompletionSource <HashEntry[]>();

            testDb.HashGetAllAsyncTask = taskCompletionSource.Task;
            cts.Cancel();
            // Using timeout to avoid hangs in the tests if something is wrong with the logic.
            var result = await ExecuteGetCheckpointInfoAsync(context, dbAdapter).WithTimeoutAsync(TimeSpan.FromSeconds(1));

            result.IsCancelled.Should().BeTrue();
        }
        public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit()
        {
            // This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.

            var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);

            int connectionCount = 0;

            bool failWithRedisConnectionError = false;
            // Setup Redis DB adapter
            Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
            {
                connectionCount++;
                // Failing connection only when the local is true;
                return(MockRedisDatabaseFactory.CreateConnection(
                           testDb,
                           testBatch: null,
                           throwConnectionExceptionOnGet: () => failWithRedisConnectionError));
            };

            var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);

            var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
                                                                             // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
                                                                             redisConnectionErrorLimit: 2,
                                                                             retryCount: 1);
            var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);

            connectionCount.Should().Be(1);

            // The first execution should fail with the connectivity issue.
            failWithRedisConnectionError = true;
            await ExecuteBatchAsync(dbAdapter).ShouldBeError();

            failWithRedisConnectionError = false;

            // The second execution should recreate the connection.
            await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess();

            connectionCount.Should().Be(2);

            // The connection was recently recreated.
            // Introducing the connectivity issue again.
            failWithRedisConnectionError = true;
            await ExecuteBatchAsync(dbAdapter).ShouldBeError();

            // The previous call set the flag to reconnect, but the actual reconnect happening on the next call.
            connectionCount.Should().Be(2);

            await ExecuteBatchAsync(dbAdapter).ShouldBeError();

            connectionCount.Should().Be(3);

            await ExecuteBatchAsync(dbAdapter).ShouldBeError();

            connectionCount.Should().Be(4);
        }
        public async Task TestRaidedRedisFailureRecovery()
        {
            // It is important to set ThrowRedisException to false, because redis exceptions are recoverable
            // and we don't want to run this test for too long because of exponential back-off recovery algorithm
            var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
            {
                FailingQuery = -1, ThrowRedisException = false
            };

            var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
            {
                FailingQuery = -1, ThrowRedisException = false
            };

            // Setup Redis DB adapter
            var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb);
            var primaryAdapter    = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace);

            var secondaryConnection   = MockRedisDatabaseFactory.CreateConnection(secondaryDb);
            var secondaryAdapter      = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace);
            var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter);
            var context = new OperationContext(new Context(Logger));

            var retryWindow = TimeSpan.FromSeconds(1);

            // Running for the first time, both operation should be successful.
            var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);

            r.primary.ShouldBeSuccess();
            r.secondary.ShouldBeSuccess();

            secondaryDb.FailNextOperation();
            r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);

            r.primary.ShouldBeSuccess();
            // The second redis should fail when we'll try to use it the second time.
            r.secondary.ShouldBeError();

            primaryDb.FailNextOperation();
            r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);

            // Now all the instance should fail.
            r.primary.ShouldBeError();
            r.secondary.ShouldBeSuccess();

            primaryDb.FailNextOperation();
            secondaryDb.FailNextOperation();
            r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);

            // Now all the instance should fail.
            r.primary.ShouldBeError();
            r.secondary.ShouldBeError();
        }
Example #7
0
        public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit()
        {
            // This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.

            // Setup test DB configured to fail 2nd query with normal Exception
            var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
            {
                // No queries will fail, instead GetDatabase will throw with RedisConnectionException.
                FailingQuery = -1,
            };

            int numberOfFactoryCalls = 0;

            // Setup Redis DB adapter
            Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
            {
                numberOfFactoryCalls++;
                // Failing connection error only from the first instance.
                return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null, throwConnectionExceptionOnGet: numberOfFactoryCalls == 1));
            };

            var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);

            var dbAdapter = new RedisDatabaseAdapter(
                redisDatabaseFactory,
                DefaultKeySpace,
                // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
                redisConnectionErrorLimit: 2,
                retryCount: 1);

            // Create a batch query
            var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);

            // Execute the batch
            var result = await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken));

            // The first execute batch should fail with the connectivity issue.
            result.ShouldBeError();
            numberOfFactoryCalls.Should().Be(1);

            var redisBatch2 = dbAdapter.CreateBatchOperation(RedisOperation.All);
            // Then we should recreate the connection and the second one should be successful.
            await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch2, default(CancellationToken)).ShouldBeSuccess();

            numberOfFactoryCalls.Should().Be(2);
        }
        public async Task SlowOperationTimesOut()
        {
            // Setup test DB configured to fail 2nd query with Redis Exception
            var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
            {
                FailingQuery = -1
            };

            // The second redis will throw RedisException, because we want to use retry strategy here and see the cancellation happening.
            var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
            {
                FailingQuery = -1, ThrowRedisException = true
            };

            // Setup Redis DB adapter
            var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb);
            var primaryAdapter    = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace);

            var secondaryConnection   = MockRedisDatabaseFactory.CreateConnection(secondaryDb);
            var secondaryAdapter      = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace);
            var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter);
            var context = new OperationContext(new Context(Logger));

            var retryWindow = TimeSpan.FromSeconds(1);

            // All the operations in the secondary instance will fail all the time.
            secondaryDb.FailNextOperation(resetFailureAutomatically: false);

            var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(
                context,
                (adapter, token) => ExecuteAsync(context, adapter, token),
                retryWindow,
                concurrent : true);

            r.primary.ShouldBeSuccess();
            // The secondary result is null is an indication that the operation was canceled.
            r.secondary.Should().BeNull();
        }
        public async Task BatchIsCancelledOnReconnectForOtherOperation()
        {
            // The test checks that if the connection is lost and the new connection is established,
            // all the pending operations are cancelled.

            var testDb  = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
            var context = new OperationContext(new Context(TestGlobal.Logger));

            int connectionCount = 0;

            bool failWithRedisConnectionErrorOnce = false;

            // Setup Redis DB adapter
            Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
            {
                connectionCount++;
                // Failing connection only when the local is true;
                return(MockRedisDatabaseFactory.CreateConnection(
                           testDb,
                           testBatch: null,
                           throwConnectionExceptionOnGet: () =>
                {
                    var oldValue = failWithRedisConnectionErrorOnce;
                    failWithRedisConnectionErrorOnce = false;
                    return oldValue;
                }));
            };

            var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);

            var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
                                                                             // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
                                                                             redisConnectionErrorLimit: 1,
                                                                             // No retries: should fail the operation immediately.
                                                                             retryCount: 0,
                                                                             cancelBatchWhenMultiplexerIsClosed: true);
            var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);

            connectionCount.Should().Be(1);

            // Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang".
            var taskCompletionSource = new TaskCompletionSource <HashEntry[]>();

            testDb.HashGetAllAsyncTask = taskCompletionSource.Task;

            // Running two operations at the same time:
            // The first one should get stuck on task completion source's task
            // and the second one will fail with connectivity issue, will cause the restart of the multiplexer,
            // and will cancel all the existing operations 9including the first one).

            var task1 = ExecuteGetCheckpointInfoAsync(context, dbAdapter);

            failWithRedisConnectionErrorOnce = true;
            var task2 = ExecuteGetCheckpointInfoAsync(context, dbAdapter);

            Output.WriteLine("Waiting for the redis operations to finish.");
            await Task.WhenAll(task1, task2);

            var results    = new[] { task1.Result, task2.Result };
            var errorCount = results.Count(r => !r.Succeeded && r.ErrorMessage?.Contains("RedisConnectionException") == true);

            errorCount.Should().Be(1, $"Should have 1 error with RedisConnectionException. Results: {string.Join(Environment.NewLine, results.Select(r => r.ToString()))}");

            var cancelledCount = results.Count(r => r.IsCancelled);

            cancelledCount.Should().Be(1, $"Should have 1 cancellation. Results: {string.Join(Environment.NewLine, results.Select(r => r.ToString()))}");
        }
        public async Task DoNotReconnectTooFrequently()
        {
            var memoryClock = new MemoryClock();

            memoryClock.UtcNow = DateTime.UtcNow;

            // This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.

            var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);

            int      connectionCount              = 0;
            TimeSpan reconnectInterval            = TimeSpan.FromSeconds(10);
            bool     failWithRedisConnectionError = false;
            // Setup Redis DB adapter
            Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
            {
                connectionCount++;
                // Failing connection only when the local is true;
                return(MockRedisDatabaseFactory.CreateConnection(
                           testDb,
                           testBatch: null,
                           throwConnectionExceptionOnGet: () => failWithRedisConnectionError));
            };

            var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);

            var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
                                                                             // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
                                                                             redisConnectionErrorLimit: 2,
                                                                             retryCount: 1,
                                                                             minReconnectInterval: reconnectInterval);
            var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration, memoryClock);

            connectionCount.Should().Be(1);

            // The first execution should fail with the connectivity issue.
            failWithRedisConnectionError = true;
            await ExecuteBatchAsync(dbAdapter).ShouldBeError();

            failWithRedisConnectionError = false;

            // The second execution should recreate the connection.
            await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess();

            connectionCount.Should().Be(2);

            // The connection was recently recreated.
            // Introducing the connectivity issue first.
            failWithRedisConnectionError = true;
            await ExecuteBatchAsync(dbAdapter).ShouldBeError();

            // The next call should not trigger the reconnect
            await ExecuteBatchAsync(dbAdapter).ShouldBeError();

            connectionCount.Should().Be(2);

            // Moving the clock forward and the next call should cause a reconnect.
            memoryClock.UtcNow += reconnectInterval.Multiply(2);
            await ExecuteBatchAsync(dbAdapter).ShouldBeError();

            // The previous execution should set the flag to reconnect,
            // but the reconnect count is still the same.
            connectionCount.Should().Be(2);

            // And only during the next call the multiplexer is actually recreated
            await ExecuteBatchAsync(dbAdapter).ShouldBeError();

            connectionCount.Should().Be(3);
        }