/// <nodoc />
internal RedisGlobalStore(
IClock clock,
RedisContentLocationStoreConfiguration configuration,
RedisDatabaseAdapter primaryRedisDb,
RedisDatabaseAdapter secondaryRedisDb,
RedisDatabaseAdapter primaryRedisBlobDb,
RedisDatabaseAdapter secondaryRedisBlobDb,
IMasterElectionMechanism masterElectionMechanism)
{
Contract.Requires(configuration.CentralStore != null);
_clock = clock;
Configuration = configuration;
RaidedRedis = new RaidedRedisDatabase(Tracer, primaryRedisDb, secondaryRedisDb);
var checkpointKeyBase = configuration.CentralStore.CentralStateKeyBase;
_clusterStateKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".ClusterState", this, _clock, RaidedRedis);
MemoizationAdapter = new RedisMemoizationAdapter(RaidedRedis, configuration.Memoization);
PrimaryBlobAdapter = new RedisBlobAdapter(primaryRedisBlobDb, _clock, Configuration);
SecondaryBlobAdapter = new RedisBlobAdapter(secondaryRedisBlobDb, _clock, Configuration);
_masterElectionMechanism = masterElectionMechanism;
}
public async Task UseReplicatedHashAsyncShouldNotCrashWhenBothRedisInstancesAreFailing()
{
// bug: https://dev.azure.com/mseng/1ES/_workitems/edit/1656837
// It is important to throw non-redis error to not trigger retry strategy.
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = 1, ThrowRedisException = false
};
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb, throwConnectionExceptionOnGet: true);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace, retryCount: 1);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, null);
var context = new OperationContext(new Context(Logger));
var replicatedRedisHashKey = new ReplicatedRedisHashKey("key", new MockReplicatedKeyHost(), new MemoryClock(), raidedDatabaseAdapter);
var error = await replicatedRedisHashKey.UseReplicatedHashAsync(
context,
retryWindow : TimeSpan.FromMinutes(1),
RedisOperation.All,
(batch, key) =>
{
return(batch.StringGetAsync("first"));
}).ShouldBeError();
// The operation should fail gracefully, not with a critical error like contract violation.
error.IsCriticalFailure.Should().BeFalse();
}
private async Task RunTest(Func <OperationContext, RedisWriteAheadEventStorage, Task> runTestAsync, RedisVolatileEventStorageConfiguration configuration = null)
{
var tracingContext = new Context(Logger);
var operationContext = new OperationContext(tracingContext);
using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance);
var primaryFactory = await RedisDatabaseFactory.CreateAsync(
operationContext,
new LiteralConnectionStringProvider(database.ConnectionString),
new RedisConnectionMultiplexerConfiguration()
{
LoggingSeverity = Severity.Error
});
var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace");
configuration ??= new RedisVolatileEventStorageConfiguration();
var instance = new RedisWriteAheadEventStorage(configuration, primaryDatabaseAdapter);
await instance.StartupAsync(operationContext).ThrowIfFailure();
await runTestAsync(operationContext, instance);
await instance.ShutdownAsync(operationContext).ThrowIfFailure();
}
public async Task ExecuteBatchOperationNoRetryOnRandomExceptions()
{
// Setup test DB configured to fail 2nd query with normal Exception
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = 2,
ThrowRedisException = false,
};
// Setup Redis DB adapter
var testConn = MockRedisDatabaseFactory.CreateConnection(testDb);
var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace);
// Create a batch query
var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);
var first = redisBatch.StringGetAsync("first");
var second = redisBatch.StringGetAsync("second");
// Execute the batch
await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).IgnoreFailure();
// Adapter does not retry in case random exception is thrown
Assert.True(testDb.BatchCalled);
Assert.Equal(2, testDb.Calls);
Assert.NotNull(first.Exception);
Assert.NotNull(second.Exception);
}
public async Task BatchIsCanceledBeforeOperationStarts()
{
// The test checks that if the connection is lost and the new connection is established,
// all the pending operations are cancelled.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
var cts = new CancellationTokenSource();
var context = new OperationContext(new Context(TestGlobal.Logger), cts.Token);
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 1,
// No retries: should fail the operation immediately.
retryCount: 0,
cancelBatchWhenMultiplexerIsClosed: true);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);
// Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang".
var taskCompletionSource = new TaskCompletionSource <HashEntry[]>();
testDb.HashGetAllAsyncTask = taskCompletionSource.Task;
cts.Cancel();
// Using timeout to avoid hangs in the tests if something is wrong with the logic.
var result = await ExecuteGetCheckpointInfoAsync(context, dbAdapter).WithTimeoutAsync(TimeSpan.FromSeconds(1));
result.IsCancelled.Should().BeTrue();
}
protected override IMemoizationStore CreateStore(DisposableDirectory testDirectory)
{
var context = new Context(_logger);
var keySpace = Guid.NewGuid().ToString();
var primaryRedisInstance = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock);
_databasesToDispose.Add(primaryRedisInstance);
var primaryFactory = RedisDatabaseFactory.CreateAsync(
context,
provider: new LiteralConnectionStringProvider(primaryRedisInstance.ConnectionString),
logSeverity: Severity.Info,
usePreventThreadTheft: false).GetAwaiter().GetResult();
var primaryRedisAdapter = new RedisDatabaseAdapter(primaryFactory, keySpace: keySpace);
var secondaryRedisInstance = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock);
_databasesToDispose.Add(secondaryRedisInstance);
var secondaryFactory = RedisDatabaseFactory.CreateAsync(
context,
provider: new LiteralConnectionStringProvider(secondaryRedisInstance.ConnectionString),
logSeverity: Severity.Info,
usePreventThreadTheft: false).GetAwaiter().GetResult();
var secondaryRedisAdapter = new RedisDatabaseAdapter(secondaryFactory, keySpace: keySpace);
var memoizationDb = new RedisMemoizationDatabase(primaryRedisAdapter, secondaryRedisAdapter, _clock, _memoizationExpiryTime, operationsTimeout: null, slowOperationRedisTimeout: null);
return(new RedisMemoizationStore(_logger, memoizationDb));
}
public async Task ExecuteBatchOperationRetriesOnRedisExceptions()
{
// Setup test DB configured to fail 2nd query with Redis Exception
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = 2
};
// Setup Redis DB adapter
var testConn = MockRedisDatabaseFactory.CreateConnection(testDb);
var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace);
// Create a batch query
var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);
var first = redisBatch.StringGetAsync("first");
var second = redisBatch.StringGetAsync("second");
// Execute the batch
await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).ShouldBeSuccess();
// Adapter is expected to retry the entire batch if single call fails
Assert.True(testDb.BatchCalled);
Assert.Equal(4, testDb.Calls);
Assert.Null(first.Exception);
Assert.Null(second.Exception);
Assert.Equal("one", await first);
Assert.Equal("two", await second);
}
private async Task TryMirrorRedisHashDataAsync(OperationContext context, RedisDatabaseAdapter source, RedisDatabaseAdapter target, long?postMirrorSourceVersion = null)
{
await context.PerformOperationAsync(
_host.Tracer,
async() =>
{
var sourceDump = await source.ExecuteBatchAsync(context,
b => b.AddOperation(_key, b => b.KeyDumpAsync(_key)),
RedisOperation.HashGetKeys);
await target.ExecuteBatchAsync(context,
b =>
{
var deleteTask = b.AddOperation(_key, b => b.KeyDeleteAsync(_key));
var restoreTask = b.AddOperation(_key, b => b.KeyRestoreAsync(_key, sourceDump).WithResultAsync(Unit.Void));
return(Task.WhenAll(deleteTask, restoreTask).WithResultAsync(Unit.Void));
},
RedisOperation.HashDeleteAndRestore);
if (postMirrorSourceVersion.HasValue)
{
await source.ExecuteBatchAsync(context,
b => b.AddOperation(_key, b => b.HashSetAsync(_key, nameof(ReplicatedHashVersionNumber), postMirrorSourceVersion.Value)),
RedisOperation.HashSetValue);
}
return(Result.Success(sourceDump.Length));
},
extraStartMessage : $"({_redis.GetDbName(source)} -> {_redis.GetDbName(target)}) Key={_key}, PostMirrorSourceVersion={postMirrorSourceVersion ?? -1L}",
extraEndMessage : r => $"({_redis.GetDbName(source)} -> {_redis.GetDbName(target)}) Key={_key}, Length={r.GetValueOrDefault(-1)}").IgnoreFailure();
}
private static Task <BoolResult> ExecuteAsync(RedisDatabaseAdapter adapter, CancellationToken token)
{
var redisBatch = adapter.CreateBatchOperation(RedisOperation.All);
var first = redisBatch.StringGetAsync("first");
// Execute the batch
return(adapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, token));
}
/// <nodoc />
public RedisMemoizationDatabase(
RedisDatabaseAdapter primaryRedis,
RedisDatabaseAdapter secondaryRedis,
RedisMemoizationConfiguration configuration)
{
_redis = new RaidedRedisDatabase(Tracer, primaryRedis, secondaryRedis);
Configuration = configuration;
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
_taskTracker = new BackgroundTaskTracker(nameof(RedisMetadataCache), context.CreateNested(nameof(RedisMetadataCache)));
var redisDatabaseAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(context, ConnectionStringProvider, logSeverity: BuildXL.Cache.ContentStore.Interfaces.Logging.Severity.Unknown, usePreventThreadTheft: false), Keyspace);
_dbAdapter = redisDatabaseAdapter;
_stringDatabaseAdapter = redisDatabaseAdapter;
return(BoolResult.Success);
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
_taskTracker = new BackgroundTaskTracker(nameof(RedisMetadataCache), new Context(context));
var redisDatabaseAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(context, ConnectionStringProvider), Keyspace);
_dbAdapter = redisDatabaseAdapter;
_stringDatabaseAdapter = redisDatabaseAdapter;
return(BoolResult.Success);
}
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit()
{
// This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData);
int connectionCount = 0;
bool failWithRedisConnectionError = false;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
connectionCount++;
// Failing connection only when the local is true;
return(MockRedisDatabaseFactory.CreateConnection(
testDb,
testBatch: null,
throwConnectionExceptionOnGet: () => failWithRedisConnectionError));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 2,
retryCount: 1);
var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration);
connectionCount.Should().Be(1);
// The first execution should fail with the connectivity issue.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
failWithRedisConnectionError = false;
// The second execution should recreate the connection.
await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess();
connectionCount.Should().Be(2);
// The connection was recently recreated.
// Introducing the connectivity issue again.
failWithRedisConnectionError = true;
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
// The previous call set the flag to reconnect, but the actual reconnect happening on the next call.
connectionCount.Should().Be(2);
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(3);
await ExecuteBatchAsync(dbAdapter).ShouldBeError();
connectionCount.Should().Be(4);
}
/// <nodoc />
public RedisMemoizationDatabase(
RedisDatabaseAdapter redis,
IClock clock,
TimeSpan metadataExpiryTime
)
{
_redis = redis;
_clock = clock;
_metadataExpiryTime = metadataExpiryTime;
}
/// <nodoc />
public RedisMemoizationDatabase(
RedisDatabaseAdapter primaryRedis,
RedisDatabaseAdapter secondaryRedis,
IClock clock,
TimeSpan metadataExpiryTime,
TimeSpan?operationsTimeout,
TimeSpan?slowOperationRedisTimeout)
: this(null, clock, metadataExpiryTime, operationsTimeout, slowOperationRedisTimeout)
{
_redis = new RaidedRedisDatabase(Tracer, primaryRedis, secondaryRedis);
}
private async Task <TResult> ExecuteAndCaptureRedisErrorsAsync <TResult>(RedisDatabaseAdapter redisDb, Func <RedisDatabaseAdapter, Task <TResult> > executeAsync)
where TResult : ResultBase
{
try
{
return(await executeAsync(redisDb));
}
catch (RedisConnectionException ex)
{
return(new ErrorResult(ex).AsResult <TResult>());
}
}
public async Task TestRaidedRedisFailureRecovery()
{
// It is important to set ThrowRedisException to false, because redis exceptions are recoverable
// and we don't want to run this test for too long because of exponential back-off recovery algorithm
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = false
};
var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = false
};
// Setup Redis DB adapter
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace);
var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb);
var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter);
var context = new OperationContext(new Context(Logger));
var retryWindow = TimeSpan.FromSeconds(1);
// Running for the first time, both operation should be successful.
var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
r.primary.ShouldBeSuccess();
r.secondary.ShouldBeSuccess();
secondaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
r.primary.ShouldBeSuccess();
// The second redis should fail when we'll try to use it the second time.
r.secondary.ShouldBeError();
primaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
// Now all the instance should fail.
r.primary.ShouldBeError();
r.secondary.ShouldBeSuccess();
primaryDb.FailNextOperation();
secondaryDb.FailNextOperation();
r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true);
// Now all the instance should fail.
r.primary.ShouldBeError();
r.secondary.ShouldBeError();
}
public static RedisMemoizationStore Create(
ILogger logger,
IConnectionStringProvider connectionStringProvider,
string keyspace,
IClock clock,
TimeSpan memoizationExpiryTime)
{
var context = new Context(logger);
var redisFactory = RedisDatabaseFactory.CreateAsync(context, connectionStringProvider).GetAwaiter().GetResult();
var redisAdapter = new RedisDatabaseAdapter(redisFactory, keyspace);
return(new RedisMemoizationStore(logger, clock, redisAdapter, memoizationExpiryTime));
}
/// <inheritdoc />
protected override RedisGlobalStore CreateRedisGlobalStore()
{
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(_primaryRedisDatabase);
var primaryDatabaseAdapter = new RedisDatabaseAdapter(
RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection).GetAwaiter().GetResult(),
DefaultKeySpace);
var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(_secondaryRedisDatabase);
var secondaryDatabaseAdapter = new RedisDatabaseAdapter(
RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection).GetAwaiter().GetResult(),
DefaultKeySpace);
return(new RedisGlobalStore(Clock, Configuration, primaryDatabaseAdapter, secondaryDatabaseAdapter, primaryDatabaseAdapter, secondaryDatabaseAdapter));
}
protected override IMemoizationStore CreateStore(DisposableDirectory testDirectory)
{
var context = new Context(_logger);
var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock);
var connectionString = localDatabase.ConnectionString;
_databasesToDispose.Add(localDatabase);
var connectionStringProvider = new LiteralConnectionStringProvider(connectionString);
var redisFactory = RedisDatabaseFactory.CreateAsync(context, connectionStringProvider).GetAwaiter().GetResult();
var redisAdapter = new RedisDatabaseAdapter(redisFactory, keySpace: Guid.NewGuid().ToString());
var memoizationDb = new RedisMemoizationDatabase(redisAdapter, _clock, _memoizationExpiryTime);
return(new RedisMemoizationStore(_logger, memoizationDb));
}
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit()
{
// This test checks that if the client fails to connect to redis, it'll successfully reconnect to it.
// Setup test DB configured to fail 2nd query with normal Exception
var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
// No queries will fail, instead GetDatabase will throw with RedisConnectionException.
FailingQuery = -1,
};
int numberOfFactoryCalls = 0;
// Setup Redis DB adapter
Func <IConnectionMultiplexer> connectionMultiplexerFactory = () =>
{
numberOfFactoryCalls++;
// Failing connection error only from the first instance.
return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null, throwConnectionExceptionOnGet: numberOfFactoryCalls == 1));
};
var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask);
var dbAdapter = new RedisDatabaseAdapter(
redisDatabaseFactory,
DefaultKeySpace,
// If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one.
redisConnectionErrorLimit: 2,
retryCount: 1);
// Create a batch query
var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All);
// Execute the batch
var result = await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken));
// The first execute batch should fail with the connectivity issue.
result.ShouldBeError();
numberOfFactoryCalls.Should().Be(1);
var redisBatch2 = dbAdapter.CreateBatchOperation(RedisOperation.All);
// Then we should recreate the connection and the second one should be successful.
await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch2, default(CancellationToken)).ShouldBeSuccess();
numberOfFactoryCalls.Should().Be(2);
}
/// <nodoc />
public RedisGlobalStore(IClock clock, RedisContentLocationStoreConfiguration configuration, MachineLocation localMachineLocation, RedisDatabaseAdapter primaryRedisDb, RedisDatabaseAdapter secondaryRedisDb)
{
Contract.Requires(configuration.CentralStore != null);
_clock = clock;
_configuration = configuration;
_primaryRedisDb = primaryRedisDb;
_secondaryRedisDb = secondaryRedisDb;
var checkpointKeyBase = configuration.CentralStore.CentralStateKeyBase;
_checkpointsKey = configuration.GetCheckpointPrefix() + ".Checkpoints";
_masterLeaseKey = checkpointKeyBase + ".MasterLease";
_clusterStateKey = checkpointKeyBase + ".ClusterState";
LocalMachineLocation = localMachineLocation;
_blobAdapter = new RedisBlobAdapter(_primaryRedisDb, TimeSpan.FromMinutes(_configuration.BlobExpiryTimeMinutes), _configuration.MaxBlobCapacity, _clock, Tracer);
}
private static async Task IncrementWithExpiryValidate(
Context context,
RedisDatabaseAdapter adapter,
ITestRedisDatabase database,
string key,
uint comparisonValue,
TimeSpan specifiedExpiry,
int requestedIncrement,
long expectedReturnValue,
long?expectedIncrementedValue,
TimeSpan?expectedDifferentExpiry = null)
{
var batch = adapter.CreateBatchOperation(RedisOperation.All);
var redisKey = GetKey(key);
var incrementWithExpire = batch.TryStringIncrementBumpExpiryIfBelowOrEqualValueAsync(key, comparisonValue, timeToLive: specifiedExpiry, requestedIncrement: requestedIncrement);
await adapter.ExecuteBatchOperationAsync(context, batch, CancellationToken.None).IgnoreFailure();
var incrementedValue = await incrementWithExpire;
Assert.Equal(expectedReturnValue, incrementedValue.AppliedIncrement);
var keysWithExpiry = database.GetDbWithExpiry();
if (expectedIncrementedValue == null)
{
Assert.False(keysWithExpiry.ContainsKey(redisKey));
Assert.Equal(expectedReturnValue, incrementedValue.IncrementedValue);
return;
}
Assert.True(keysWithExpiry.ContainsKey(redisKey));
var expiry = keysWithExpiry[redisKey];
if (expectedDifferentExpiry != null)
{
Assert.False(expiry.Equals(new MockRedisValueWithExpiry(expectedIncrementedValue, DateTime.UtcNow + specifiedExpiry)));
Assert.True(expiry.Equals(new MockRedisValueWithExpiry(expectedIncrementedValue, DateTime.UtcNow + expectedDifferentExpiry.Value)));
}
else
{
Assert.True(expiry.Equals(new MockRedisValueWithExpiry(expectedIncrementedValue, DateTime.UtcNow + specifiedExpiry)));
}
}
/// <nodoc />
public RedisGlobalStore(
IClock clock,
RedisContentLocationStoreConfiguration configuration,
RedisDatabaseAdapter primaryRedisDb,
RedisDatabaseAdapter secondaryRedisDb)
{
Contract.Requires(configuration.CentralStore != null);
_clock = clock;
_configuration = configuration;
_raidedRedis = new RaidedRedisDatabase(Tracer, primaryRedisDb, secondaryRedisDb);
var checkpointKeyBase = configuration.CentralStore.CentralStateKeyBase;
_checkpointsKey = new ReplicatedRedisHashKey(configuration.GetCheckpointPrefix() + ".Checkpoints", this, _clock, _raidedRedis);
_masterLeaseKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".MasterLease", this, _clock, _raidedRedis);
_clusterStateKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".ClusterState", this, _clock, _raidedRedis);
_blobAdapter = new RedisBlobAdapter(_raidedRedis.PrimaryRedisDb, TimeSpan.FromMinutes(_configuration.BlobExpiryTimeMinutes), _configuration.MaxBlobCapacity, _clock);
}
private async Task RunTest(
Func <OperationContext, ContentMetadataEventStream, IFailureController, IFailureController, Task> runTestAsync,
ContentMetadataEventStreamConfiguration contentMetadataEventStreamConfiguration = null,
RedisVolatileEventStorageConfiguration redisVolatileEventLogConfiguration = null,
FailureMode persistentStorageFailure = FailureMode.None,
FailureMode volatileStorageFailure = FailureMode.None)
{
var tracingContext = new Context(Logger);
var operationContext = new OperationContext(tracingContext);
redisVolatileEventLogConfiguration ??= new RedisVolatileEventStorageConfiguration();
using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance);
var primaryFactory = await RedisDatabaseFactory.CreateAsync(
operationContext,
new LiteralConnectionStringProvider(database.ConnectionString),
new RedisConnectionMultiplexerConfiguration()
{
LoggingSeverity = Severity.Error
});
var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace");
var redisVolatileEventStorage = new RedisWriteAheadEventStorage(redisVolatileEventLogConfiguration, primaryDatabaseAdapter);
var mockPersistentEventStorage = new MockPersistentEventStorage();
var volatileEventStorage = new FailingVolatileEventStorage(volatileStorageFailure, redisVolatileEventStorage);
var persistentEventStorage = new FailingPersistentEventStorage(persistentStorageFailure, mockPersistentEventStorage);
contentMetadataEventStreamConfiguration ??= new ContentMetadataEventStreamConfiguration();
var contentMetadataEventStream = new ContentMetadataEventStream(contentMetadataEventStreamConfiguration, volatileEventStorage, persistentEventStorage);
await contentMetadataEventStream.StartupAsync(operationContext).ThrowIfFailure();
await contentMetadataEventStream.CompleteOrChangeLogAsync(operationContext, CheckpointLogId.InitialLogId);
contentMetadataEventStream.SetIsLogging(true);
await runTestAsync(operationContext, contentMetadataEventStream, volatileEventStorage, persistentEventStorage);
await contentMetadataEventStream.ShutdownAsync(operationContext).ThrowIfFailure();
}
/// <nodoc />
internal RedisGlobalStore(
IClock clock,
RedisContentLocationStoreConfiguration configuration,
RedisDatabaseAdapter primaryRedisDb,
RedisDatabaseAdapter secondaryRedisDb,
RedisDatabaseAdapter primaryRedisBlobDb,
RedisDatabaseAdapter secondaryRedisBlobDb)
{
Contract.Requires(configuration.CentralStore != null);
_clock = clock;
Configuration = configuration;
RaidedRedis = new RaidedRedisDatabase(Tracer, primaryRedisDb, secondaryRedisDb);
var checkpointKeyBase = configuration.CentralStore.CentralStateKeyBase;
_checkpointsKey = new ReplicatedRedisHashKey(configuration.GetCheckpointPrefix() + ".Checkpoints", this, _clock, RaidedRedis);
_masterLeaseKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".MasterLease", this, _clock, RaidedRedis);
_clusterStateKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".ClusterState", this, _clock, RaidedRedis);
PrimaryBlobAdapter = new RedisBlobAdapter(primaryRedisBlobDb, _clock, Configuration);
SecondaryBlobAdapter = new RedisBlobAdapter(secondaryRedisBlobDb, _clock, Configuration);
}
public async Task SlowOperationTimesOut()
{
// Setup test DB configured to fail 2nd query with Redis Exception
var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1
};
// The second redis will throw RedisException, because we want to use retry strategy here and see the cancellation happening.
var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData)
{
FailingQuery = -1, ThrowRedisException = true
};
// Setup Redis DB adapter
var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb);
var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace);
var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb);
var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace);
var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter);
var context = new OperationContext(new Context(Logger));
var retryWindow = TimeSpan.FromSeconds(1);
// All the operations in the secondary instance will fail all the time.
secondaryDb.FailNextOperation(resetFailureAutomatically: false);
var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(
context,
(adapter, token) => ExecuteAsync(context, adapter, token),
retryWindow,
concurrent : true);
r.primary.ShouldBeSuccess();
// The secondary result is null is an indication that the operation was canceled.
r.secondary.Should().BeNull();
}
private Task <Result <HashEntry[]> > UpdateLocalClusterStateAsync(OperationContext context, ClusterState clusterState, RedisDatabaseAdapter redisDb)
{
return(redisDb.ExecuteBatchAsync(context, async batch =>
{
var heartbeatResultTask = CallHeartbeatAsync(context, batch, MachineState.Active);
var getUnknownMachinesTask = batch.GetUnknownMachinesAsync(
_clusterStateKey,
clusterState.MaxMachineId);
// Only master should mirror cluster state
bool shouldMirrorClusterState = _role == Role.Master &&
HasSecondary &&
_configuration.MirrorClusterState
// Only mirror after a long interval, but not long enough to allow machines to appear expired
&& !_lastClusterStateMirrorTime.IsRecent(_clock.UtcNow, _configuration.ClusterStateMirrorInterval)
// Only mirror from primary to secondary, so no need to dump cluster state if this is the secondary
&& IsPrimary(redisDb);
Task <HashEntry[]> dumpClusterStateBlobTask = shouldMirrorClusterState
? batch.AddOperation(_clusterStateKey, b => b.HashGetAllAsync(_clusterStateKey))
: _emptyClusterStateDump;
await Task.WhenAll(heartbeatResultTask, getUnknownMachinesTask, dumpClusterStateBlobTask);
var clusterStateBlob = await dumpClusterStateBlobTask ?? CollectionUtilities.EmptyArray <HashEntry>();
var heartbeatResult = await heartbeatResultTask;
var getUnknownMachinesResult = await getUnknownMachinesTask;
if (shouldMirrorClusterState)
{
_lastClusterStateMirrorTime = _clock.UtcNow;
}
if (getUnknownMachinesResult.maxMachineId < LocalMachineId.Index)
{
return Result.FromErrorMessage <HashEntry[]>($"Invalid {GetDbName(redisDb)} redis cluster state on machine {LocalMachineId} (max machine id={getUnknownMachinesResult.maxMachineId})");
}
if (heartbeatResult.priorState == MachineState.Unavailable || heartbeatResult.priorState == MachineState.Expired)
{
clusterState.LastInactiveTime = _clock.UtcNow;
}
if (getUnknownMachinesResult.maxMachineId != clusterState.MaxMachineId)
{
Tracer.Debug(context, $"Retrieved unknown machines from ({clusterState.MaxMachineId}, {getUnknownMachinesResult.maxMachineId}]");
foreach (var item in getUnknownMachinesResult.unknownMachines)
{
context.LogMachineMapping(Tracer, item.Key, item.Value);
}
}
clusterState.AddUnknownMachines(getUnknownMachinesResult.maxMachineId, getUnknownMachinesResult.unknownMachines);
clusterState.SetInactiveMachines(heartbeatResult.inactiveMachineIdSet);
Tracer.Debug(context, $"Inactive machines: Count={heartbeatResult.inactiveMachineIdSet.Count}, [{string.Join(", ", heartbeatResult.inactiveMachineIdSet)}]");
Tracer.TrackMetric(context, "InactiveMachineCount", heartbeatResult.inactiveMachineIdSet.Count);
return Result.Success(await dumpClusterStateBlobTask ?? CollectionUtilities.EmptyArray <HashEntry>());
},
RedisOperation.UpdateClusterState));
}
private bool IsPrimary(RedisDatabaseAdapter redisDb)
{
return(redisDb == _primaryRedisDb);
}
private string GetDbName(RedisDatabaseAdapter redisDb)
{
return(redisDb == _primaryRedisDb ? "primary" : "secondary");
}
