public async Task ExecuteBatchOperationNoRetryOnRandomExceptions()
{
// Setup test DB configured to fail 2nd query with normal Exception
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = 2,
ThrowRedisException = false,
};
// Setup Redis DB adapter
var testConn = MockRedisDatabaseFactory.CreateConnection(testDb);
var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace);
// Create a batch query
var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);
var first = redisBatch.StringGetAsync("first");
var second = redisBatch.StringGetAsync("second");
// Execute the batch
await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).IgnoreFailure();
// Adapter does not retry in case random exception is thrown
Assert.True(testDb.BatchCalled);
Assert.Equal(2, testDb.Calls);
Assert.NotNull(first.Exception);
Assert.NotNull(second.Exception);
}
public async Task ExecuteBatchOperationRetriesOnRedisExceptions()
{
// Setup test DB configured to fail 2nd query with Redis Exception
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = 2
};
// Setup Redis DB adapter
var testConn = MockRedisDatabaseFactory.CreateConnection(testDb);
var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace);
// Create a batch query
var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);
var first = redisBatch.StringGetAsync("first");
var second = redisBatch.StringGetAsync("second");
// Execute the batch
await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).ShouldBeSuccess();
// Adapter is expected to retry the entire batch if single call fails
Assert.True(testDb.BatchCalled);
Assert.Equal(4, testDb.Calls);
Assert.Null(first.Exception);
Assert.Null(second.Exception);
Assert.Equal("one", await first);
Assert.Equal("two", await second);
}
protected override IMemoizationStore CreateStore(DisposableDirectory testDirectory)
{
var context = new Context(_logger);
var keySpace = Guid.NewGuid().ToString();
var primaryRedisInstance = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock);
_databasesToDispose.Add(primaryRedisInstance);
var primaryFactory = RedisDatabaseFactory.CreateAsync(
context,
provider: new LiteralConnectionStringProvider(primaryRedisInstance.ConnectionString),
logSeverity: Severity.Info,
usePreventThreadTheft: false).GetAwaiter().GetResult();
var primaryRedisAdapter = new RedisDatabaseAdapter(primaryFactory, keySpace: keySpace);
var secondaryRedisInstance = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock);
_databasesToDispose.Add(secondaryRedisInstance);
var secondaryFactory = RedisDatabaseFactory.CreateAsync(
context,
provider: new LiteralConnectionStringProvider(secondaryRedisInstance.ConnectionString),
logSeverity: Severity.Info,
usePreventThreadTheft: false).GetAwaiter().GetResult();
var secondaryRedisAdapter = new RedisDatabaseAdapter(secondaryFactory, keySpace: keySpace);
var memoizationDb = new RedisMemoizationDatabase(primaryRedisAdapter, secondaryRedisAdapter, _clock, _memoizationExpiryTime, operationsTimeout: null, slowOperationRedisTimeout: null);
return(new RedisMemoizationStore(_logger, memoizationDb));
}
public async Task BatchIsCanceledBeforeOperationStarts()
{
// The test checks that if the connection is lost and the new connection is established,
// all the pending operations are cancelled.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
var cts = new CancellationTokenSource();
var context = new OperationContext(new Context(TestGlobal.Logger), cts.Token);
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 1,
// No retries: should fail the operation immediately.
retryCount: 0,
cancelBatchWhenMultiplexerIsClosed: true);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);
// Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang".
var taskCompletionSource = new TaskCompletionSource <HashEntry[]>();
testDb.HashGetAllAsyncTask = taskCompletionSource.Task;
cts.Cancel();
// Using timeout to avoid hangs in the tests if something is wrong with the logic.
var result = await ExecuteGetCheckpointInfoAsync(context, dbAdapter).WithTimeoutAsync(TimeSpan.FromSeconds(1));
result.IsCancelled.Should().BeTrue();
}
private async Task RunTest(Func <OperationContext, RedisWriteAheadEventStorage, Task> runTestAsync, RedisVolatileEventStorageConfiguration configuration = null)
{
var tracingContext = new Context(Logger);
var operationContext = new OperationContext(tracingContext);
using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance);
var primaryFactory = await RedisDatabaseFactory.CreateAsync(
operationContext,
new LiteralConnectionStringProvider(database.ConnectionString),
new RedisConnectionMultiplexerConfiguration()
{
LoggingSeverity = Severity.Error
});
var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace");
configuration ??= new RedisVolatileEventStorageConfiguration();
var instance = new RedisWriteAheadEventStorage(configuration, primaryDatabaseAdapter);
await instance.StartupAsync(operationContext).ThrowIfFailure();
await runTestAsync(operationContext, instance);
await instance.ShutdownAsync(operationContext).ThrowIfFailure();
}
public async Task UseReplicatedHashAsyncShouldNotCrashWhenBothRedisInstancesAreFailing()
{
// bug: https://dev.azure.com/mseng/1ES/_workitems/edit/1656837
// It is important to throw non-redis error to not trigger retry strategy.
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = 1, ThrowRedisException = false
};
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb, throwConnectionExceptionOnGet: true);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace, retryCount: 1);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, null);
var context = new OperationContext(new Context(Logger));
var replicatedRedisHashKey = new ReplicatedRedisHashKey("key", new MockReplicatedKeyHost(), new MemoryClock(), raidedDatabaseAdapter);
var error = await replicatedRedisHashKey.UseReplicatedHashAsync(
context,
retryWindow : TimeSpan.FromMinutes(1),
RedisOperation.All,
(batch, key) =>
{
return(batch.StringGetAsync("first"));
}).ShouldBeError();
// The operation should fail gracefully, not with a critical error like contract violation.
error.IsCriticalFailure.Should().BeFalse();
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
_taskTracker = new BackgroundTaskTracker(nameof(RedisMetadataCache), context.CreateNested(nameof(RedisMetadataCache)));
var redisDatabaseAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(context, ConnectionStringProvider, logSeverity: BuildXL.Cache.ContentStore.Interfaces.Logging.Severity.Unknown, usePreventThreadTheft: false), Keyspace);
_dbAdapter = redisDatabaseAdapter;
_stringDatabaseAdapter = redisDatabaseAdapter;
return(BoolResult.Success);
}
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit()
{
// This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
int connectionCount = 0;
bool failWithRedisConnectionError = false;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
connectionCount++;
// Failing connection only when the local is true;
return(MockRedisDatabaseFactory.CreateConnection(
testDb,
testBatch: null,
throwConnectionExceptionOnGet: () => failWithRedisConnectionError));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 2,
retryCount: 1);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);
connectionCount.Should().Be(1);
// The first execution should fail with the connectivity issue.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
failWithRedisConnectionError = false;
// The second execution should recreate the connection.
await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess();
connectionCount.Should().Be(2);
// The connection was recently recreated.
// Introducing the connectivity issue again.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
// The previous call set the flag to reconnect, but the actual reconnect happening on the next call.
connectionCount.Should().Be(2);
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(3);
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(4);
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
_taskTracker = new BackgroundTaskTracker(nameof(RedisMetadataCache), new Context(context));
var redisDatabaseAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(context, ConnectionStringProvider), Keyspace);
_dbAdapter = redisDatabaseAdapter;
_stringDatabaseAdapter = redisDatabaseAdapter;
return(BoolResult.Success);
}
public async Task TestRaidedRedisFailureRecovery()
{
// It is important to set ThrowRedisException to false, because redis exceptions are recoverable
// and we don't want to run this test for too long because of exponential back-off recovery algorithm
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = false
};
var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = false
};
// Setup Redis DB adapter
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace);
var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb);
var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter);
var context = new OperationContext(new Context(Logger));
var retryWindow = TimeSpan.FromSeconds(1);
// Running for the first time, both operation should be successful.
var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
r.primary.ShouldBeSuccess();
r.secondary.ShouldBeSuccess();
secondaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
r.primary.ShouldBeSuccess();
// The second redis should fail when we'll try to use it the second time.
r.secondary.ShouldBeError();
primaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
// Now all the instance should fail.
r.primary.ShouldBeError();
r.secondary.ShouldBeSuccess();
primaryDb.FailNextOperation();
secondaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
// Now all the instance should fail.
r.primary.ShouldBeError();
r.secondary.ShouldBeError();
}
private async Task <IDatabase> GetKeyspacedDatabaseAsync()
{
if (_prefixedDatabase == null)
{
_prefixedDatabase = (await RedisDatabaseFactory
.GetRedisDatabaseAsync(_connectionString)
.ConfigureAwait(false))
.WithKeyPrefix(_keyPrefix);
}
return(_prefixedDatabase);
}
private RedisDatabaseAdapter CreateDatabase(RedisDatabaseFactory factory, bool optional = false)
{
if (factory != null)
{
return(new RedisDatabaseAdapter(factory, KeySpace));
}
else
{
Contract.Assert(optional);
return(null);
}
}
public static RedisMemoizationStore Create(
ILogger logger,
IConnectionStringProvider connectionStringProvider,
string keyspace,
IClock clock,
TimeSpan memoizationExpiryTime)
{
var context = new Context(logger);
var redisFactory = RedisDatabaseFactory.CreateAsync(context, connectionStringProvider).GetAwaiter().GetResult();
var redisAdapter = new RedisDatabaseAdapter(redisFactory, keyspace);
return(new RedisMemoizationStore(logger, clock, redisAdapter, memoizationExpiryTime));
}
/// <inheritdoc />
protected override RedisGlobalStore CreateRedisGlobalStore()
{
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(_primaryRedisDatabase);
var primaryDatabaseAdapter = new RedisDatabaseAdapter(
RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection).GetAwaiter().GetResult(),
DefaultKeySpace);
var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(_secondaryRedisDatabase);
var secondaryDatabaseAdapter = new RedisDatabaseAdapter(
RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection).GetAwaiter().GetResult(),
DefaultKeySpace);
return(new RedisGlobalStore(Clock, Configuration, primaryDatabaseAdapter, secondaryDatabaseAdapter, primaryDatabaseAdapter, secondaryDatabaseAdapter));
}
protected override IMemoizationStore CreateStore(DisposableDirectory testDirectory)
{
var context = new Context(_logger);
var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock);
var connectionString = localDatabase.ConnectionString;
_databasesToDispose.Add(localDatabase);
var connectionStringProvider = new LiteralConnectionStringProvider(connectionString);
var redisFactory = RedisDatabaseFactory.CreateAsync(context, connectionStringProvider).GetAwaiter().GetResult();
var redisAdapter = new RedisDatabaseAdapter(redisFactory, keySpace: Guid.NewGuid().ToString());
var memoizationDb = new RedisMemoizationDatabase(redisAdapter, _clock, _memoizationExpiryTime);
return(new RedisMemoizationStore(_logger, memoizationDb));
}
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit()
{
// This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.
// Setup test DB configured to fail 2nd query with normal Exception
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
// No queries will fail, instead GetDatabase will throw with RedisConnectionException.
FailingQuery = -1,
};
int numberOfFactoryCalls = 0;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
numberOfFactoryCalls++;
// Failing connection error only from the first instance.
return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null, throwConnectionExceptionOnGet: numberOfFactoryCalls == 1));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var dbAdapter = new RedisDatabaseAdapter(
redisDatabaseFactory,
DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 2,
retryCount: 1);
// Create a batch query
var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);
// Execute the batch
var result = await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken));
// The first execute batch should fail with the connectivity issue.
result.ShouldBeError();
numberOfFactoryCalls.Should().Be(1);
var redisBatch2 = dbAdapter.CreateBatchOperation(RedisOperation.All);
// Then we should recreate the connection and the second one should be successful.
await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch2, default(CancellationToken)).ShouldBeSuccess();
numberOfFactoryCalls.Should().Be(2);
}
private async Task RunTest(
Func <OperationContext, ContentMetadataEventStream, IFailureController, IFailureController, Task> runTestAsync,
ContentMetadataEventStreamConfiguration contentMetadataEventStreamConfiguration = null,
RedisVolatileEventStorageConfiguration redisVolatileEventLogConfiguration = null,
FailureMode persistentStorageFailure = FailureMode.None,
FailureMode volatileStorageFailure = FailureMode.None)
{
var tracingContext = new Context(Logger);
var operationContext = new OperationContext(tracingContext);
redisVolatileEventLogConfiguration ??= new RedisVolatileEventStorageConfiguration();
using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance);
var primaryFactory = await RedisDatabaseFactory.CreateAsync(
operationContext,
new LiteralConnectionStringProvider(database.ConnectionString),
new RedisConnectionMultiplexerConfiguration()
{
LoggingSeverity = Severity.Error
});
var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace");
var redisVolatileEventStorage = new RedisWriteAheadEventStorage(redisVolatileEventLogConfiguration, primaryDatabaseAdapter);
var mockPersistentEventStorage = new MockPersistentEventStorage();
var volatileEventStorage = new FailingVolatileEventStorage(volatileStorageFailure, redisVolatileEventStorage);
var persistentEventStorage = new FailingPersistentEventStorage(persistentStorageFailure, mockPersistentEventStorage);
contentMetadataEventStreamConfiguration ??= new ContentMetadataEventStreamConfiguration();
var contentMetadataEventStream = new ContentMetadataEventStream(contentMetadataEventStreamConfiguration, volatileEventStorage, persistentEventStorage);
await contentMetadataEventStream.StartupAsync(operationContext).ThrowIfFailure();
await contentMetadataEventStream.CompleteOrChangeLogAsync(operationContext, CheckpointLogId.InitialLogId);
contentMetadataEventStream.SetIsLogging(true);
await runTestAsync(operationContext, contentMetadataEventStream, volatileEventStorage, persistentEventStorage);
await contentMetadataEventStream.ShutdownAsync(operationContext).ThrowIfFailure();
}
public async Task SlowOperationTimesOut()
{
// Setup test DB configured to fail 2nd query with Redis Exception
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1
};
// The second redis will throw RedisException, because we want to use retry strategy here and see the cancellation happening.
var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = true
};
// Setup Redis DB adapter
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace);
var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb);
var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter);
var context = new OperationContext(new Context(Logger));
var retryWindow = TimeSpan.FromSeconds(1);
// All the operations in the secondary instance will fail all the time.
secondaryDb.FailNextOperation(resetFailureAutomatically: false);
var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(
context,
(adapter, token) => ExecuteAsync(context, adapter, token),
retryWindow,
concurrent : true);
r.primary.ShouldBeSuccess();
// The secondary result is null is an indication that the operation was canceled.
r.secondary.Should().BeNull();
}
private async Task RunTest(
Func <OperationContext, ResilientGlobalCacheService, int, Task> runTestAsync,
bool persistentStorageFailure = false,
bool volatileStorageFailure = false,
IClock?clock = null,
int iterations = 1,
Action <GlobalCacheServiceConfiguration>?modifyConfig = null)
{
var tracingContext = new Context(Logger);
var operationContext = new OperationContext(tracingContext);
clock ??= SystemClock.Instance;
var contentMetadataServiceConfiguration = new GlobalCacheServiceConfiguration()
{
Checkpoint = new CheckpointManagerConfiguration(TestRootDirectoryPath / "CheckpointManager", default(MachineLocation)),
EventStream = new ContentMetadataEventStreamConfiguration(),
};
modifyConfig?.Invoke(contentMetadataServiceConfiguration);
using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance);
var primaryFactory = await RedisDatabaseFactory.CreateAsync(
operationContext,
new LiteralConnectionStringProvider(database.ConnectionString),
new RedisConnectionMultiplexerConfiguration()
{
LoggingSeverity = Severity.Error
});
var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace");
var centralStorage = new Dictionary <string, byte[]>();
for (var iteration = 0; iteration < iterations; iteration++)
{
Tracer.Info(operationContext, $"Running iteration {iteration}");
var redisVolatileEventStorage = new RedisWriteAheadEventStorage(new RedisVolatileEventStorageConfiguration(), primaryDatabaseAdapter, clock);
IWriteAheadEventStorage volatileEventStorage = new FailingVolatileEventStorage();
if (!volatileStorageFailure)
{
volatileEventStorage = redisVolatileEventStorage;
}
IWriteBehindEventStorage persistentEventStorage = new FailingPersistentEventStorage();
if (!persistentStorageFailure)
{
persistentEventStorage = new MockPersistentEventStorage();
}
var contentMetadataEventStream = new ContentMetadataEventStream(
contentMetadataServiceConfiguration.EventStream,
volatileEventStorage,
persistentEventStorage);
var rocksdbContentMetadataDatabaseConfiguration = new RocksDbContentMetadataDatabaseConfiguration(TestRootDirectoryPath / "ContentMetadataDatabase");
var rocksDbContentMetadataStore = new RocksDbContentMetadataStore(clock, new RocksDbContentMetadataStoreConfiguration()
{
Database = rocksdbContentMetadataDatabaseConfiguration,
});
var storage = new MockCentralStorage(centralStorage);
var checkpointManager = new CheckpointManager(
rocksDbContentMetadataStore.Database,
redisVolatileEventStorage,
storage,
contentMetadataServiceConfiguration.Checkpoint,
new CounterCollection <ContentLocationStoreCounters>());
var resilientContentMetadataService = new ResilientGlobalCacheService(
contentMetadataServiceConfiguration,
checkpointManager,
rocksDbContentMetadataStore,
contentMetadataEventStream,
storage,
clock);
await resilientContentMetadataService.StartupAsync(operationContext).ThrowIfFailure();
await runTestAsync(operationContext, resilientContentMetadataService, iteration);
await resilientContentMetadataService.ShutdownAsync(operationContext).ThrowIfFailure();
}
}
public async Task BatchIsCancelledOnReconnectForOtherOperation()
{
// The test checks that if the connection is lost and the new connection is established,
// all the pending operations are cancelled.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
var context = new OperationContext(new Context(TestGlobal.Logger));
int connectionCount = 0;
bool failWithRedisConnectionErrorOnce = false;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
connectionCount++;
// Failing connection only when the local is true;
return(MockRedisDatabaseFactory.CreateConnection(
testDb,
testBatch: null,
throwConnectionExceptionOnGet: () =>
{
var oldValue = failWithRedisConnectionErrorOnce;
failWithRedisConnectionErrorOnce = false;
return oldValue;
}));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 1,
// No retries: should fail the operation immediately.
retryCount: 0,
cancelBatchWhenMultiplexerIsClosed: true);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);
connectionCount.Should().Be(1);
// Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang".
var taskCompletionSource = new TaskCompletionSource <HashEntry[]>();
testDb.HashGetAllAsyncTask = taskCompletionSource.Task;
// Running two operations at the same time:
// The first one should get stuck on task completion source's task
// and the second one will fail with connectivity issue, will cause the restart of the multiplexer,
// and will cancel all the existing operations 9including the first one).
var task1 = ExecuteGetCheckpointInfoAsync(context, dbAdapter);
failWithRedisConnectionErrorOnce = true;
var task2 = ExecuteGetCheckpointInfoAsync(context, dbAdapter);
Output.WriteLine("Waiting for the redis operations to finish.");
await Task.WhenAll(task1, task2);
var results = new[] { task1.Result, task2.Result };
var errorCount = results.Count(r => !r.Succeeded && r.ErrorMessage?.Contains("RedisConnectionException") == true);
errorCount.Should().Be(1, $"Should have 1 error with RedisConnectionException. Results: {string.Join(Environment.NewLine, results.Select(r => r.ToString()))}");
var cancelledCount = results.Count(r => r.IsCancelled);
cancelledCount.Should().Be(1, $"Should have 1 cancellation. Results: {string.Join(Environment.NewLine, results.Select(r => r.ToString()))}");
}
public async Task DoNotReconnectTooFrequently()
{
var memoryClock = new MemoryClock();
memoryClock.UtcNow = DateTime.UtcNow;
// This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
int connectionCount = 0;
TimeSpan reconnectInterval = TimeSpan.FromSeconds(10);
bool failWithRedisConnectionError = false;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
connectionCount++;
// Failing connection only when the local is true;
return(MockRedisDatabaseFactory.CreateConnection(
testDb,
testBatch: null,
throwConnectionExceptionOnGet: () => failWithRedisConnectionError));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 2,
retryCount: 1,
minReconnectInterval: reconnectInterval);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration, memoryClock);
connectionCount.Should().Be(1);
// The first execution should fail with the connectivity issue.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
failWithRedisConnectionError = false;
// The second execution should recreate the connection.
await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess();
connectionCount.Should().Be(2);
// The connection was recently recreated.
// Introducing the connectivity issue first.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
// The next call should not trigger the reconnect
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(2);
// Moving the clock forward and the next call should cause a reconnect.
memoryClock.UtcNow += reconnectInterval.Multiply(2);
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
// The previous execution should set the flag to reconnect,
// but the reconnect count is still the same.
connectionCount.Should().Be(2);
// And only during the next call the multiplexer is actually recreated
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(3);
}
public async Task <IContentLocationStore> CreateAsync(MachineLocation machineLocation)
{
var connection = MockRedisDatabaseFactory.CreateConnection(RedisDatabase);
RedisDatabaseAdapter = RedisDatabaseAdapter ?? new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), connection), RedisContentLocationStoreFactory.DefaultKeySpace);
var machineLocationConnection = MockRedisDatabaseFactory.CreateConnection(MachineLocationRedisDatabase);
MachineRedisDatabaseAdapter = MachineRedisDatabaseAdapter ?? new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), machineLocationConnection), RedisContentLocationStoreFactory.DefaultKeySpace);
IContentLocationStore store = new RedisContentLocationStore(
RedisDatabaseAdapter,
MachineRedisDatabaseAdapter,
_mockClock,
BumpTime,
machineLocation.Data,
_configuration);
var redisStore = (RedisContentLocationStore)store;
redisStore.DisableReplica = true;
return(store);
}
