public async Task TestRaidedRedisFailureRecovery()
{
// It is important to set ThrowRedisException to false, because redis exceptions are recoverable
// and we don't want to run this test for too long because of exponential back-off recovery algorithm
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = false
};
var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = false
};
// Setup Redis DB adapter
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace);
var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb);
var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter);
var context = new OperationContext(new Context(Logger));
var retryWindow = TimeSpan.FromSeconds(1);
// Running for the first time, both operation should be successful.
var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
r.primary.ShouldBeSuccess();
r.secondary.ShouldBeSuccess();
secondaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
r.primary.ShouldBeSuccess();
// The second redis should fail when we'll try to use it the second time.
r.secondary.ShouldBeError();
primaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
// Now all the instance should fail.
r.primary.ShouldBeError();
r.secondary.ShouldBeSuccess();
primaryDb.FailNextOperation();
secondaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
// Now all the instance should fail.
r.primary.ShouldBeError();
r.secondary.ShouldBeError();
}
private async Task <Result <bool> > CompareExchangeInternalAsync(
OperationContext context,
StrongFingerprint strongFingerprint,
string expectedReplacementToken,
ContentHashListWithDeterminism replacement,
string newReplacementToken)
{
var key = GetKey(strongFingerprint.WeakFingerprint);
var replacementMetadata = new MetadataEntry(replacement, DateTime.UtcNow);
using var replacementBytes = SerializeMetadataEntry(replacementMetadata);
using var selectorBytes = SerializeSelector(strongFingerprint.Selector, isReplacementToken: false);
using var tokenFieldNameBytes = SerializeSelector(strongFingerprint.Selector, isReplacementToken: true);
var(primaryResult, secondaryResult) = await _redis.ExecuteRaidedAsync <bool>(
context,
async (redis, cancellationToken) =>
{
using var nestedContext = new CancellableOperationContext(context, cancellationToken);
return(await redis.ExecuteBatchAsync(
nestedContext,
batch =>
{
var task = batch.CompareExchangeAsync(
key,
(ReadOnlyMemory <byte>)selectorBytes,
(ReadOnlyMemory <byte>)tokenFieldNameBytes,
expectedReplacementToken,
(ReadOnlyMemory <byte>)replacementBytes,
newReplacementToken);
batch.KeyExpireAsync(key, Configuration.ExpiryTime).FireAndForget(nestedContext);
return task;
},
RedisOperation.CompareExchange));
},
retryWindow : Configuration.SlowOperationCancellationTimeout);
Contract.Assert(primaryResult != null || secondaryResult != null);
if (primaryResult?.Succeeded == true || secondaryResult?.Succeeded == true)
{
// One of the operations is successful.
return((primaryResult?.Value ?? false) || (secondaryResult?.Value ?? false));
}
// All operations failed, propagating the error back to the caller.
var failure = primaryResult ?? secondaryResult;
Contract.Assert(!failure.Succeeded);
return(new Result <bool>(failure));
}
public async Task SlowOperationTimesOut()
{
// Setup test DB configured to fail 2nd query with Redis Exception
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1
};
// The second redis will throw RedisException, because we want to use retry strategy here and see the cancellation happening.
var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = true
};
// Setup Redis DB adapter
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace);
var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb);
var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter);
var context = new OperationContext(new Context(Logger));
var retryWindow = TimeSpan.FromSeconds(1);
// All the operations in the secondary instance will fail all the time.
secondaryDb.FailNextOperation(resetFailureAutomatically: false);
var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(
context,
(adapter, token) => ExecuteAsync(context, adapter, token),
retryWindow,
concurrent : true);
r.primary.ShouldBeSuccess();
// The secondary result is null is an indication that the operation was canceled.
r.secondary.Should().BeNull();
}