public async Task UseReplicatedHashAsyncShouldNotCrashWhenBothRedisInstancesAreFailing()
{
// bug: https://dev.azure.com/mseng/1ES/_workitems/edit/1656837
// It is important to throw non-redis error to not trigger retry strategy.
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = 1, ThrowRedisException = false
};
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb, throwConnectionExceptionOnGet: () => true);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace, retryCount: 1);
var primaryAdapter = new RedisDatabaseAdapter(
await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection),
adapterConfiguration);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, null);
var context = new OperationContext(new Context(Logger));
var replicatedRedisHashKey = new ReplicatedRedisHashKey("key", new MockReplicatedKeyHost(), new MemoryClock(), raidedDatabaseAdapter);
var error = await replicatedRedisHashKey.UseReplicatedHashAsync(
context,
retryWindow : TimeSpan.FromMinutes(1),
RedisOperation.All,
(batch, key) =>
{
return(batch.StringGetAsync("first"));
}).ShouldBeError();
// The operation should fail gracefully, not with a critical error like contract violation.
error.IsCriticalFailure.Should().BeFalse();
}
/// <nodoc />
public RedisMemoizationDatabase(
RedisDatabaseAdapter primaryRedis,
RedisDatabaseAdapter secondaryRedis,
RedisMemoizationConfiguration configuration)
{
_redis = new RaidedRedisDatabase(Tracer, primaryRedis, secondaryRedis);
Configuration = configuration;
}
/// <nodoc />
public RedisMemoizationDatabase(
RedisDatabaseAdapter primaryRedis,
RedisDatabaseAdapter secondaryRedis,
IClock clock,
TimeSpan metadataExpiryTime,
TimeSpan?operationsTimeout,
TimeSpan?slowOperationRedisTimeout)
: this(null, clock, metadataExpiryTime, operationsTimeout, slowOperationRedisTimeout)
{
_redis = new RaidedRedisDatabase(Tracer, primaryRedis, secondaryRedis);
}
public async Task TestRaidedRedisFailureRecovery()
{
// It is important to set ThrowRedisException to false, because redis exceptions are recoverable
// and we don't want to run this test for too long because of exponential back-off recovery algorithm
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = false
};
var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = false
};
// Setup Redis DB adapter
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace);
var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb);
var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter);
var context = new OperationContext(new Context(Logger));
var retryWindow = TimeSpan.FromSeconds(1);
// Running for the first time, both operation should be successful.
var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
r.primary.ShouldBeSuccess();
r.secondary.ShouldBeSuccess();
secondaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
r.primary.ShouldBeSuccess();
// The second redis should fail when we'll try to use it the second time.
r.secondary.ShouldBeError();
primaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
// Now all the instance should fail.
r.primary.ShouldBeError();
r.secondary.ShouldBeSuccess();
primaryDb.FailNextOperation();
secondaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
// Now all the instance should fail.
r.primary.ShouldBeError();
r.secondary.ShouldBeError();
}
/// <nodoc />
public RedisMemoizationDatabase(
RaidedRedisDatabase redis,
IClock clock,
TimeSpan metadataExpiryTime,
TimeSpan?operationsTimeout,
TimeSpan?slowOperationCancellationTimeout)
: base(operationsTimeout)
{
_redis = redis;
_clock = clock;
_metadataExpiryTime = metadataExpiryTime;
_slowOperationCancellationTimeout = slowOperationCancellationTimeout;
}
public async Task SlowOperationTimesOut()
{
// Setup test DB configured to fail 2nd query with Redis Exception
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1
};
// The second redis will throw RedisException, because we want to use retry strategy here and see the cancellation happening.
var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = true
};
// Setup Redis DB adapter
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace);
var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb);
var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter);
var context = new OperationContext(new Context(Logger));
var retryWindow = TimeSpan.FromSeconds(1);
// All the operations in the secondary instance will fail all the time.
secondaryDb.FailNextOperation(resetFailureAutomatically: false);
var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(
context,
(adapter, token) => ExecuteAsync(context, adapter, token),
retryWindow,
concurrent : true);
r.primary.ShouldBeSuccess();
// The secondary result is null is an indication that the operation was canceled.
r.secondary.Should().BeNull();
}
