public async Task ExecuteBatchOperationNoRetryOnRandomExceptions() { // Setup test DB configured to fail 2nd query with normal Exception var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = 2, ThrowRedisException = false, }; // Setup Redis DB adapter var testConn = MockRedisDatabaseFactory.CreateConnection(testDb); var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace); // Create a batch query var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All); var first = redisBatch.StringGetAsync("first"); var second = redisBatch.StringGetAsync("second"); // Execute the batch await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).IgnoreFailure(); // Adapter does not retry in case random exception is thrown Assert.True(testDb.BatchCalled); Assert.Equal(2, testDb.Calls); Assert.NotNull(first.Exception); Assert.NotNull(second.Exception); }
public async Task ExecuteBatchOperationRetriesOnRedisExceptions() { // Setup test DB configured to fail 2nd query with Redis Exception var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = 2 }; // Setup Redis DB adapter var testConn = MockRedisDatabaseFactory.CreateConnection(testDb); var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace); // Create a batch query var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All); var first = redisBatch.StringGetAsync("first"); var second = redisBatch.StringGetAsync("second"); // Execute the batch await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).ShouldBeSuccess(); // Adapter is expected to retry the entire batch if single call fails Assert.True(testDb.BatchCalled); Assert.Equal(4, testDb.Calls); Assert.Null(first.Exception); Assert.Null(second.Exception); Assert.Equal("one", await first); Assert.Equal("two", await second); }
protected override IMemoizationStore CreateStore(DisposableDirectory testDirectory) { var context = new Context(_logger); var keySpace = Guid.NewGuid().ToString(); var primaryRedisInstance = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock); _databasesToDispose.Add(primaryRedisInstance); var primaryFactory = RedisDatabaseFactory.CreateAsync( context, provider: new LiteralConnectionStringProvider(primaryRedisInstance.ConnectionString), logSeverity: Severity.Info, usePreventThreadTheft: false).GetAwaiter().GetResult(); var primaryRedisAdapter = new RedisDatabaseAdapter(primaryFactory, keySpace: keySpace); var secondaryRedisInstance = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock); _databasesToDispose.Add(secondaryRedisInstance); var secondaryFactory = RedisDatabaseFactory.CreateAsync( context, provider: new LiteralConnectionStringProvider(secondaryRedisInstance.ConnectionString), logSeverity: Severity.Info, usePreventThreadTheft: false).GetAwaiter().GetResult(); var secondaryRedisAdapter = new RedisDatabaseAdapter(secondaryFactory, keySpace: keySpace); var memoizationDb = new RedisMemoizationDatabase(primaryRedisAdapter, secondaryRedisAdapter, _clock, _memoizationExpiryTime, operationsTimeout: null, slowOperationRedisTimeout: null); return(new RedisMemoizationStore(_logger, memoizationDb)); }
public async Task BatchIsCanceledBeforeOperationStarts() { // The test checks that if the connection is lost and the new connection is established, // all the pending operations are cancelled. var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData); var cts = new CancellationTokenSource(); var context = new OperationContext(new Context(TestGlobal.Logger), cts.Token); // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null)); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 1, // No retries: should fail the operation immediately. retryCount: 0, cancelBatchWhenMultiplexerIsClosed: true); var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration); // Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang". var taskCompletionSource = new TaskCompletionSource <HashEntry[]>(); testDb.HashGetAllAsyncTask = taskCompletionSource.Task; cts.Cancel(); // Using timeout to avoid hangs in the tests if something is wrong with the logic. var result = await ExecuteGetCheckpointInfoAsync(context, dbAdapter).WithTimeoutAsync(TimeSpan.FromSeconds(1)); result.IsCancelled.Should().BeTrue(); }
private async Task RunTest(Func <OperationContext, RedisWriteAheadEventStorage, Task> runTestAsync, RedisVolatileEventStorageConfiguration configuration = null) { var tracingContext = new Context(Logger); var operationContext = new OperationContext(tracingContext); using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance); var primaryFactory = await RedisDatabaseFactory.CreateAsync( operationContext, new LiteralConnectionStringProvider(database.ConnectionString), new RedisConnectionMultiplexerConfiguration() { LoggingSeverity = Severity.Error }); var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace"); configuration ??= new RedisVolatileEventStorageConfiguration(); var instance = new RedisWriteAheadEventStorage(configuration, primaryDatabaseAdapter); await instance.StartupAsync(operationContext).ThrowIfFailure(); await runTestAsync(operationContext, instance); await instance.ShutdownAsync(operationContext).ThrowIfFailure(); }
public async Task UseReplicatedHashAsyncShouldNotCrashWhenBothRedisInstancesAreFailing() { // bug: https://dev.azure.com/mseng/1ES/_workitems/edit/1656837 // It is important to throw non-redis error to not trigger retry strategy. var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = 1, ThrowRedisException = false }; var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb, throwConnectionExceptionOnGet: true); var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace, retryCount: 1); var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, null); var context = new OperationContext(new Context(Logger)); var replicatedRedisHashKey = new ReplicatedRedisHashKey("key", new MockReplicatedKeyHost(), new MemoryClock(), raidedDatabaseAdapter); var error = await replicatedRedisHashKey.UseReplicatedHashAsync( context, retryWindow : TimeSpan.FromMinutes(1), RedisOperation.All, (batch, key) => { return(batch.StringGetAsync("first")); }).ShouldBeError(); // The operation should fail gracefully, not with a critical error like contract violation. error.IsCriticalFailure.Should().BeFalse(); }
/// <inheritdoc /> protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { _taskTracker = new BackgroundTaskTracker(nameof(RedisMetadataCache), context.CreateNested(nameof(RedisMetadataCache))); var redisDatabaseAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(context, ConnectionStringProvider, logSeverity: BuildXL.Cache.ContentStore.Interfaces.Logging.Severity.Unknown, usePreventThreadTheft: false), Keyspace); _dbAdapter = redisDatabaseAdapter; _stringDatabaseAdapter = redisDatabaseAdapter; return(BoolResult.Success); }
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit() { // This test checks that if the client fails to connect to redis, it'll successfully reconnect to it. var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData); int connectionCount = 0; bool failWithRedisConnectionError = false; // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { connectionCount++; // Failing connection only when the local is true; return(MockRedisDatabaseFactory.CreateConnection( testDb, testBatch: null, throwConnectionExceptionOnGet: () => failWithRedisConnectionError)); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 2, retryCount: 1); var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration); connectionCount.Should().Be(1); // The first execution should fail with the connectivity issue. failWithRedisConnectionError = true; await ExecuteBatchAsync(dbAdapter).ShouldBeError(); failWithRedisConnectionError = false; // The second execution should recreate the connection. await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess(); connectionCount.Should().Be(2); // The connection was recently recreated. // Introducing the connectivity issue again. failWithRedisConnectionError = true; await ExecuteBatchAsync(dbAdapter).ShouldBeError(); // The previous call set the flag to reconnect, but the actual reconnect happening on the next call. connectionCount.Should().Be(2); await ExecuteBatchAsync(dbAdapter).ShouldBeError(); connectionCount.Should().Be(3); await ExecuteBatchAsync(dbAdapter).ShouldBeError(); connectionCount.Should().Be(4); }
/// <inheritdoc /> protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { _taskTracker = new BackgroundTaskTracker(nameof(RedisMetadataCache), new Context(context)); var redisDatabaseAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(context, ConnectionStringProvider), Keyspace); _dbAdapter = redisDatabaseAdapter; _stringDatabaseAdapter = redisDatabaseAdapter; return(BoolResult.Success); }
public async Task TestRaidedRedisFailureRecovery() { // It is important to set ThrowRedisException to false, because redis exceptions are recoverable // and we don't want to run this test for too long because of exponential back-off recovery algorithm var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = -1, ThrowRedisException = false }; var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = -1, ThrowRedisException = false }; // Setup Redis DB adapter var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb); var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace); var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb); var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace); var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter); var context = new OperationContext(new Context(Logger)); var retryWindow = TimeSpan.FromSeconds(1); // Running for the first time, both operation should be successful. var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); r.primary.ShouldBeSuccess(); r.secondary.ShouldBeSuccess(); secondaryDb.FailNextOperation(); r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); r.primary.ShouldBeSuccess(); // The second redis should fail when we'll try to use it the second time. r.secondary.ShouldBeError(); primaryDb.FailNextOperation(); r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); // Now all the instance should fail. r.primary.ShouldBeError(); r.secondary.ShouldBeSuccess(); primaryDb.FailNextOperation(); secondaryDb.FailNextOperation(); r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); // Now all the instance should fail. r.primary.ShouldBeError(); r.secondary.ShouldBeError(); }
private async Task <IDatabase> GetKeyspacedDatabaseAsync() { if (_prefixedDatabase == null) { _prefixedDatabase = (await RedisDatabaseFactory .GetRedisDatabaseAsync(_connectionString) .ConfigureAwait(false)) .WithKeyPrefix(_keyPrefix); } return(_prefixedDatabase); }
private RedisDatabaseAdapter CreateDatabase(RedisDatabaseFactory factory, bool optional = false) { if (factory != null) { return(new RedisDatabaseAdapter(factory, KeySpace)); } else { Contract.Assert(optional); return(null); } }
public static RedisMemoizationStore Create( ILogger logger, IConnectionStringProvider connectionStringProvider, string keyspace, IClock clock, TimeSpan memoizationExpiryTime) { var context = new Context(logger); var redisFactory = RedisDatabaseFactory.CreateAsync(context, connectionStringProvider).GetAwaiter().GetResult(); var redisAdapter = new RedisDatabaseAdapter(redisFactory, keyspace); return(new RedisMemoizationStore(logger, clock, redisAdapter, memoizationExpiryTime)); }
/// <inheritdoc /> protected override RedisGlobalStore CreateRedisGlobalStore() { var primaryConnection = MockRedisDatabaseFactory.CreateConnection(_primaryRedisDatabase); var primaryDatabaseAdapter = new RedisDatabaseAdapter( RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection).GetAwaiter().GetResult(), DefaultKeySpace); var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(_secondaryRedisDatabase); var secondaryDatabaseAdapter = new RedisDatabaseAdapter( RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection).GetAwaiter().GetResult(), DefaultKeySpace); return(new RedisGlobalStore(Clock, Configuration, primaryDatabaseAdapter, secondaryDatabaseAdapter, primaryDatabaseAdapter, secondaryDatabaseAdapter)); }
protected override IMemoizationStore CreateStore(DisposableDirectory testDirectory) { var context = new Context(_logger); var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock); var connectionString = localDatabase.ConnectionString; _databasesToDispose.Add(localDatabase); var connectionStringProvider = new LiteralConnectionStringProvider(connectionString); var redisFactory = RedisDatabaseFactory.CreateAsync(context, connectionStringProvider).GetAwaiter().GetResult(); var redisAdapter = new RedisDatabaseAdapter(redisFactory, keySpace: Guid.NewGuid().ToString()); var memoizationDb = new RedisMemoizationDatabase(redisAdapter, _clock, _memoizationExpiryTime); return(new RedisMemoizationStore(_logger, memoizationDb)); }
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit() { // This test checks that if the client fails to connect to redis, it'll successfully reconnect to it. // Setup test DB configured to fail 2nd query with normal Exception var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { // No queries will fail, instead GetDatabase will throw with RedisConnectionException. FailingQuery = -1, }; int numberOfFactoryCalls = 0; // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { numberOfFactoryCalls++; // Failing connection error only from the first instance. return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null, throwConnectionExceptionOnGet: numberOfFactoryCalls == 1)); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var dbAdapter = new RedisDatabaseAdapter( redisDatabaseFactory, DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 2, retryCount: 1); // Create a batch query var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All); // Execute the batch var result = await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)); // The first execute batch should fail with the connectivity issue. result.ShouldBeError(); numberOfFactoryCalls.Should().Be(1); var redisBatch2 = dbAdapter.CreateBatchOperation(RedisOperation.All); // Then we should recreate the connection and the second one should be successful. await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch2, default(CancellationToken)).ShouldBeSuccess(); numberOfFactoryCalls.Should().Be(2); }
private async Task RunTest( Func <OperationContext, ContentMetadataEventStream, IFailureController, IFailureController, Task> runTestAsync, ContentMetadataEventStreamConfiguration contentMetadataEventStreamConfiguration = null, RedisVolatileEventStorageConfiguration redisVolatileEventLogConfiguration = null, FailureMode persistentStorageFailure = FailureMode.None, FailureMode volatileStorageFailure = FailureMode.None) { var tracingContext = new Context(Logger); var operationContext = new OperationContext(tracingContext); redisVolatileEventLogConfiguration ??= new RedisVolatileEventStorageConfiguration(); using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance); var primaryFactory = await RedisDatabaseFactory.CreateAsync( operationContext, new LiteralConnectionStringProvider(database.ConnectionString), new RedisConnectionMultiplexerConfiguration() { LoggingSeverity = Severity.Error }); var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace"); var redisVolatileEventStorage = new RedisWriteAheadEventStorage(redisVolatileEventLogConfiguration, primaryDatabaseAdapter); var mockPersistentEventStorage = new MockPersistentEventStorage(); var volatileEventStorage = new FailingVolatileEventStorage(volatileStorageFailure, redisVolatileEventStorage); var persistentEventStorage = new FailingPersistentEventStorage(persistentStorageFailure, mockPersistentEventStorage); contentMetadataEventStreamConfiguration ??= new ContentMetadataEventStreamConfiguration(); var contentMetadataEventStream = new ContentMetadataEventStream(contentMetadataEventStreamConfiguration, volatileEventStorage, persistentEventStorage); await contentMetadataEventStream.StartupAsync(operationContext).ThrowIfFailure(); await contentMetadataEventStream.CompleteOrChangeLogAsync(operationContext, CheckpointLogId.InitialLogId); contentMetadataEventStream.SetIsLogging(true); await runTestAsync(operationContext, contentMetadataEventStream, volatileEventStorage, persistentEventStorage); await contentMetadataEventStream.ShutdownAsync(operationContext).ThrowIfFailure(); }
public async Task SlowOperationTimesOut() { // Setup test DB configured to fail 2nd query with Redis Exception var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = -1 }; // The second redis will throw RedisException, because we want to use retry strategy here and see the cancellation happening. var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = -1, ThrowRedisException = true }; // Setup Redis DB adapter var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb); var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace); var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb); var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace); var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter); var context = new OperationContext(new Context(Logger)); var retryWindow = TimeSpan.FromSeconds(1); // All the operations in the secondary instance will fail all the time. secondaryDb.FailNextOperation(resetFailureAutomatically: false); var r = await raidedDatabaseAdapter.ExecuteRaidedAsync( context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); r.primary.ShouldBeSuccess(); // The secondary result is null is an indication that the operation was canceled. r.secondary.Should().BeNull(); }
private async Task RunTest( Func <OperationContext, ResilientGlobalCacheService, int, Task> runTestAsync, bool persistentStorageFailure = false, bool volatileStorageFailure = false, IClock?clock = null, int iterations = 1, Action <GlobalCacheServiceConfiguration>?modifyConfig = null) { var tracingContext = new Context(Logger); var operationContext = new OperationContext(tracingContext); clock ??= SystemClock.Instance; var contentMetadataServiceConfiguration = new GlobalCacheServiceConfiguration() { Checkpoint = new CheckpointManagerConfiguration(TestRootDirectoryPath / "CheckpointManager", default(MachineLocation)), EventStream = new ContentMetadataEventStreamConfiguration(), }; modifyConfig?.Invoke(contentMetadataServiceConfiguration); using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance); var primaryFactory = await RedisDatabaseFactory.CreateAsync( operationContext, new LiteralConnectionStringProvider(database.ConnectionString), new RedisConnectionMultiplexerConfiguration() { LoggingSeverity = Severity.Error }); var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace"); var centralStorage = new Dictionary <string, byte[]>(); for (var iteration = 0; iteration < iterations; iteration++) { Tracer.Info(operationContext, $"Running iteration {iteration}"); var redisVolatileEventStorage = new RedisWriteAheadEventStorage(new RedisVolatileEventStorageConfiguration(), primaryDatabaseAdapter, clock); IWriteAheadEventStorage volatileEventStorage = new FailingVolatileEventStorage(); if (!volatileStorageFailure) { volatileEventStorage = redisVolatileEventStorage; } IWriteBehindEventStorage persistentEventStorage = new FailingPersistentEventStorage(); if (!persistentStorageFailure) { persistentEventStorage = new MockPersistentEventStorage(); } var contentMetadataEventStream = new ContentMetadataEventStream( contentMetadataServiceConfiguration.EventStream, volatileEventStorage, persistentEventStorage); var rocksdbContentMetadataDatabaseConfiguration = new RocksDbContentMetadataDatabaseConfiguration(TestRootDirectoryPath / "ContentMetadataDatabase"); var rocksDbContentMetadataStore = new RocksDbContentMetadataStore(clock, new RocksDbContentMetadataStoreConfiguration() { Database = rocksdbContentMetadataDatabaseConfiguration, }); var storage = new MockCentralStorage(centralStorage); var checkpointManager = new CheckpointManager( rocksDbContentMetadataStore.Database, redisVolatileEventStorage, storage, contentMetadataServiceConfiguration.Checkpoint, new CounterCollection <ContentLocationStoreCounters>()); var resilientContentMetadataService = new ResilientGlobalCacheService( contentMetadataServiceConfiguration, checkpointManager, rocksDbContentMetadataStore, contentMetadataEventStream, storage, clock); await resilientContentMetadataService.StartupAsync(operationContext).ThrowIfFailure(); await runTestAsync(operationContext, resilientContentMetadataService, iteration); await resilientContentMetadataService.ShutdownAsync(operationContext).ThrowIfFailure(); } }
public async Task BatchIsCancelledOnReconnectForOtherOperation() { // The test checks that if the connection is lost and the new connection is established, // all the pending operations are cancelled. var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData); var context = new OperationContext(new Context(TestGlobal.Logger)); int connectionCount = 0; bool failWithRedisConnectionErrorOnce = false; // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { connectionCount++; // Failing connection only when the local is true; return(MockRedisDatabaseFactory.CreateConnection( testDb, testBatch: null, throwConnectionExceptionOnGet: () => { var oldValue = failWithRedisConnectionErrorOnce; failWithRedisConnectionErrorOnce = false; return oldValue; })); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 1, // No retries: should fail the operation immediately. retryCount: 0, cancelBatchWhenMultiplexerIsClosed: true); var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration); connectionCount.Should().Be(1); // Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang". var taskCompletionSource = new TaskCompletionSource <HashEntry[]>(); testDb.HashGetAllAsyncTask = taskCompletionSource.Task; // Running two operations at the same time: // The first one should get stuck on task completion source's task // and the second one will fail with connectivity issue, will cause the restart of the multiplexer, // and will cancel all the existing operations 9including the first one). var task1 = ExecuteGetCheckpointInfoAsync(context, dbAdapter); failWithRedisConnectionErrorOnce = true; var task2 = ExecuteGetCheckpointInfoAsync(context, dbAdapter); Output.WriteLine("Waiting for the redis operations to finish."); await Task.WhenAll(task1, task2); var results = new[] { task1.Result, task2.Result }; var errorCount = results.Count(r => !r.Succeeded && r.ErrorMessage?.Contains("RedisConnectionException") == true); errorCount.Should().Be(1, $"Should have 1 error with RedisConnectionException. Results: {string.Join(Environment.NewLine, results.Select(r => r.ToString()))}"); var cancelledCount = results.Count(r => r.IsCancelled); cancelledCount.Should().Be(1, $"Should have 1 cancellation. Results: {string.Join(Environment.NewLine, results.Select(r => r.ToString()))}"); }
public async Task DoNotReconnectTooFrequently() { var memoryClock = new MemoryClock(); memoryClock.UtcNow = DateTime.UtcNow; // This test checks that if the client fails to connect to redis, it'll successfully reconnect to it. var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData); int connectionCount = 0; TimeSpan reconnectInterval = TimeSpan.FromSeconds(10); bool failWithRedisConnectionError = false; // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { connectionCount++; // Failing connection only when the local is true; return(MockRedisDatabaseFactory.CreateConnection( testDb, testBatch: null, throwConnectionExceptionOnGet: () => failWithRedisConnectionError)); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 2, retryCount: 1, minReconnectInterval: reconnectInterval); var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration, memoryClock); connectionCount.Should().Be(1); // The first execution should fail with the connectivity issue. failWithRedisConnectionError = true; await ExecuteBatchAsync(dbAdapter).ShouldBeError(); failWithRedisConnectionError = false; // The second execution should recreate the connection. await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess(); connectionCount.Should().Be(2); // The connection was recently recreated. // Introducing the connectivity issue first. failWithRedisConnectionError = true; await ExecuteBatchAsync(dbAdapter).ShouldBeError(); // The next call should not trigger the reconnect await ExecuteBatchAsync(dbAdapter).ShouldBeError(); connectionCount.Should().Be(2); // Moving the clock forward and the next call should cause a reconnect. memoryClock.UtcNow += reconnectInterval.Multiply(2); await ExecuteBatchAsync(dbAdapter).ShouldBeError(); // The previous execution should set the flag to reconnect, // but the reconnect count is still the same. connectionCount.Should().Be(2); // And only during the next call the multiplexer is actually recreated await ExecuteBatchAsync(dbAdapter).ShouldBeError(); connectionCount.Should().Be(3); }
public async Task <IContentLocationStore> CreateAsync(MachineLocation machineLocation) { var connection = MockRedisDatabaseFactory.CreateConnection(RedisDatabase); RedisDatabaseAdapter = RedisDatabaseAdapter ?? new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), connection), RedisContentLocationStoreFactory.DefaultKeySpace); var machineLocationConnection = MockRedisDatabaseFactory.CreateConnection(MachineLocationRedisDatabase); MachineRedisDatabaseAdapter = MachineRedisDatabaseAdapter ?? new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), machineLocationConnection), RedisContentLocationStoreFactory.DefaultKeySpace); IContentLocationStore store = new RedisContentLocationStore( RedisDatabaseAdapter, MachineRedisDatabaseAdapter, _mockClock, BumpTime, machineLocation.Data, _configuration); var redisStore = (RedisContentLocationStore)store; redisStore.DisableReplica = true; return(store); }