/// <nodoc /> internal RedisGlobalStore( IClock clock, RedisContentLocationStoreConfiguration configuration, RedisDatabaseAdapter primaryRedisDb, RedisDatabaseAdapter secondaryRedisDb, RedisDatabaseAdapter primaryRedisBlobDb, RedisDatabaseAdapter secondaryRedisBlobDb, IMasterElectionMechanism masterElectionMechanism) { Contract.Requires(configuration.CentralStore != null); _clock = clock; Configuration = configuration; RaidedRedis = new RaidedRedisDatabase(Tracer, primaryRedisDb, secondaryRedisDb); var checkpointKeyBase = configuration.CentralStore.CentralStateKeyBase; _clusterStateKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".ClusterState", this, _clock, RaidedRedis); MemoizationAdapter = new RedisMemoizationAdapter(RaidedRedis, configuration.Memoization); PrimaryBlobAdapter = new RedisBlobAdapter(primaryRedisBlobDb, _clock, Configuration); SecondaryBlobAdapter = new RedisBlobAdapter(secondaryRedisBlobDb, _clock, Configuration); _masterElectionMechanism = masterElectionMechanism; }
public async Task UseReplicatedHashAsyncShouldNotCrashWhenBothRedisInstancesAreFailing() { // bug: https://dev.azure.com/mseng/1ES/_workitems/edit/1656837 // It is important to throw non-redis error to not trigger retry strategy. var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = 1, ThrowRedisException = false }; var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb, throwConnectionExceptionOnGet: true); var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace, retryCount: 1); var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, null); var context = new OperationContext(new Context(Logger)); var replicatedRedisHashKey = new ReplicatedRedisHashKey("key", new MockReplicatedKeyHost(), new MemoryClock(), raidedDatabaseAdapter); var error = await replicatedRedisHashKey.UseReplicatedHashAsync( context, retryWindow : TimeSpan.FromMinutes(1), RedisOperation.All, (batch, key) => { return(batch.StringGetAsync("first")); }).ShouldBeError(); // The operation should fail gracefully, not with a critical error like contract violation. error.IsCriticalFailure.Should().BeFalse(); }
private async Task RunTest(Func <OperationContext, RedisWriteAheadEventStorage, Task> runTestAsync, RedisVolatileEventStorageConfiguration configuration = null) { var tracingContext = new Context(Logger); var operationContext = new OperationContext(tracingContext); using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance); var primaryFactory = await RedisDatabaseFactory.CreateAsync( operationContext, new LiteralConnectionStringProvider(database.ConnectionString), new RedisConnectionMultiplexerConfiguration() { LoggingSeverity = Severity.Error }); var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace"); configuration ??= new RedisVolatileEventStorageConfiguration(); var instance = new RedisWriteAheadEventStorage(configuration, primaryDatabaseAdapter); await instance.StartupAsync(operationContext).ThrowIfFailure(); await runTestAsync(operationContext, instance); await instance.ShutdownAsync(operationContext).ThrowIfFailure(); }
public async Task ExecuteBatchOperationNoRetryOnRandomExceptions() { // Setup test DB configured to fail 2nd query with normal Exception var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = 2, ThrowRedisException = false, }; // Setup Redis DB adapter var testConn = MockRedisDatabaseFactory.CreateConnection(testDb); var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace); // Create a batch query var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All); var first = redisBatch.StringGetAsync("first"); var second = redisBatch.StringGetAsync("second"); // Execute the batch await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).IgnoreFailure(); // Adapter does not retry in case random exception is thrown Assert.True(testDb.BatchCalled); Assert.Equal(2, testDb.Calls); Assert.NotNull(first.Exception); Assert.NotNull(second.Exception); }
public async Task BatchIsCanceledBeforeOperationStarts() { // The test checks that if the connection is lost and the new connection is established, // all the pending operations are cancelled. var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData); var cts = new CancellationTokenSource(); var context = new OperationContext(new Context(TestGlobal.Logger), cts.Token); // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null)); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 1, // No retries: should fail the operation immediately. retryCount: 0, cancelBatchWhenMultiplexerIsClosed: true); var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration); // Causing HashGetAllAsync operation to hang that will cause ExecuteGetCheckpointInfoAsync operation to "hang". var taskCompletionSource = new TaskCompletionSource <HashEntry[]>(); testDb.HashGetAllAsyncTask = taskCompletionSource.Task; cts.Cancel(); // Using timeout to avoid hangs in the tests if something is wrong with the logic. var result = await ExecuteGetCheckpointInfoAsync(context, dbAdapter).WithTimeoutAsync(TimeSpan.FromSeconds(1)); result.IsCancelled.Should().BeTrue(); }
protected override IMemoizationStore CreateStore(DisposableDirectory testDirectory) { var context = new Context(_logger); var keySpace = Guid.NewGuid().ToString(); var primaryRedisInstance = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock); _databasesToDispose.Add(primaryRedisInstance); var primaryFactory = RedisDatabaseFactory.CreateAsync( context, provider: new LiteralConnectionStringProvider(primaryRedisInstance.ConnectionString), logSeverity: Severity.Info, usePreventThreadTheft: false).GetAwaiter().GetResult(); var primaryRedisAdapter = new RedisDatabaseAdapter(primaryFactory, keySpace: keySpace); var secondaryRedisInstance = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock); _databasesToDispose.Add(secondaryRedisInstance); var secondaryFactory = RedisDatabaseFactory.CreateAsync( context, provider: new LiteralConnectionStringProvider(secondaryRedisInstance.ConnectionString), logSeverity: Severity.Info, usePreventThreadTheft: false).GetAwaiter().GetResult(); var secondaryRedisAdapter = new RedisDatabaseAdapter(secondaryFactory, keySpace: keySpace); var memoizationDb = new RedisMemoizationDatabase(primaryRedisAdapter, secondaryRedisAdapter, _clock, _memoizationExpiryTime, operationsTimeout: null, slowOperationRedisTimeout: null); return(new RedisMemoizationStore(_logger, memoizationDb)); }
public async Task ExecuteBatchOperationRetriesOnRedisExceptions() { // Setup test DB configured to fail 2nd query with Redis Exception var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = 2 }; // Setup Redis DB adapter var testConn = MockRedisDatabaseFactory.CreateConnection(testDb); var dbAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), testConn), DefaultKeySpace); // Create a batch query var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All); var first = redisBatch.StringGetAsync("first"); var second = redisBatch.StringGetAsync("second"); // Execute the batch await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)).ShouldBeSuccess(); // Adapter is expected to retry the entire batch if single call fails Assert.True(testDb.BatchCalled); Assert.Equal(4, testDb.Calls); Assert.Null(first.Exception); Assert.Null(second.Exception); Assert.Equal("one", await first); Assert.Equal("two", await second); }
private async Task TryMirrorRedisHashDataAsync(OperationContext context, RedisDatabaseAdapter source, RedisDatabaseAdapter target, long?postMirrorSourceVersion = null) { await context.PerformOperationAsync( _host.Tracer, async() => { var sourceDump = await source.ExecuteBatchAsync(context, b => b.AddOperation(_key, b => b.KeyDumpAsync(_key)), RedisOperation.HashGetKeys); await target.ExecuteBatchAsync(context, b => { var deleteTask = b.AddOperation(_key, b => b.KeyDeleteAsync(_key)); var restoreTask = b.AddOperation(_key, b => b.KeyRestoreAsync(_key, sourceDump).WithResultAsync(Unit.Void)); return(Task.WhenAll(deleteTask, restoreTask).WithResultAsync(Unit.Void)); }, RedisOperation.HashDeleteAndRestore); if (postMirrorSourceVersion.HasValue) { await source.ExecuteBatchAsync(context, b => b.AddOperation(_key, b => b.HashSetAsync(_key, nameof(ReplicatedHashVersionNumber), postMirrorSourceVersion.Value)), RedisOperation.HashSetValue); } return(Result.Success(sourceDump.Length)); }, extraStartMessage : $"({_redis.GetDbName(source)} -> {_redis.GetDbName(target)}) Key={_key}, PostMirrorSourceVersion={postMirrorSourceVersion ?? -1L}", extraEndMessage : r => $"({_redis.GetDbName(source)} -> {_redis.GetDbName(target)}) Key={_key}, Length={r.GetValueOrDefault(-1)}").IgnoreFailure(); }
private static Task <BoolResult> ExecuteAsync(RedisDatabaseAdapter adapter, CancellationToken token) { var redisBatch = adapter.CreateBatchOperation(RedisOperation.All); var first = redisBatch.StringGetAsync("first"); // Execute the batch return(adapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, token)); }
/// <nodoc /> public RedisMemoizationDatabase( RedisDatabaseAdapter primaryRedis, RedisDatabaseAdapter secondaryRedis, RedisMemoizationConfiguration configuration) { _redis = new RaidedRedisDatabase(Tracer, primaryRedis, secondaryRedis); Configuration = configuration; }
/// <inheritdoc /> protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { _taskTracker = new BackgroundTaskTracker(nameof(RedisMetadataCache), context.CreateNested(nameof(RedisMetadataCache))); var redisDatabaseAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(context, ConnectionStringProvider, logSeverity: BuildXL.Cache.ContentStore.Interfaces.Logging.Severity.Unknown, usePreventThreadTheft: false), Keyspace); _dbAdapter = redisDatabaseAdapter; _stringDatabaseAdapter = redisDatabaseAdapter; return(BoolResult.Success); }
/// <inheritdoc /> protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { _taskTracker = new BackgroundTaskTracker(nameof(RedisMetadataCache), new Context(context)); var redisDatabaseAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(context, ConnectionStringProvider), Keyspace); _dbAdapter = redisDatabaseAdapter; _stringDatabaseAdapter = redisDatabaseAdapter; return(BoolResult.Success); }
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit() { // This test checks that if the client fails to connect to redis, it'll successfully reconnect to it. var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData); int connectionCount = 0; bool failWithRedisConnectionError = false; // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { connectionCount++; // Failing connection only when the local is true; return(MockRedisDatabaseFactory.CreateConnection( testDb, testBatch: null, throwConnectionExceptionOnGet: () => failWithRedisConnectionError)); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var adapterConfiguration = new RedisDatabaseAdapterConfiguration(DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 2, retryCount: 1); var dbAdapter = new RedisDatabaseAdapter(redisDatabaseFactory, adapterConfiguration); connectionCount.Should().Be(1); // The first execution should fail with the connectivity issue. failWithRedisConnectionError = true; await ExecuteBatchAsync(dbAdapter).ShouldBeError(); failWithRedisConnectionError = false; // The second execution should recreate the connection. await ExecuteBatchAsync(dbAdapter).ShouldBeSuccess(); connectionCount.Should().Be(2); // The connection was recently recreated. // Introducing the connectivity issue again. failWithRedisConnectionError = true; await ExecuteBatchAsync(dbAdapter).ShouldBeError(); // The previous call set the flag to reconnect, but the actual reconnect happening on the next call. connectionCount.Should().Be(2); await ExecuteBatchAsync(dbAdapter).ShouldBeError(); connectionCount.Should().Be(3); await ExecuteBatchAsync(dbAdapter).ShouldBeError(); connectionCount.Should().Be(4); }
/// <nodoc /> public RedisMemoizationDatabase( RedisDatabaseAdapter redis, IClock clock, TimeSpan metadataExpiryTime ) { _redis = redis; _clock = clock; _metadataExpiryTime = metadataExpiryTime; }
/// <nodoc /> public RedisMemoizationDatabase( RedisDatabaseAdapter primaryRedis, RedisDatabaseAdapter secondaryRedis, IClock clock, TimeSpan metadataExpiryTime, TimeSpan?operationsTimeout, TimeSpan?slowOperationRedisTimeout) : this(null, clock, metadataExpiryTime, operationsTimeout, slowOperationRedisTimeout) { _redis = new RaidedRedisDatabase(Tracer, primaryRedis, secondaryRedis); }
private async Task <TResult> ExecuteAndCaptureRedisErrorsAsync <TResult>(RedisDatabaseAdapter redisDb, Func <RedisDatabaseAdapter, Task <TResult> > executeAsync) where TResult : ResultBase { try { return(await executeAsync(redisDb)); } catch (RedisConnectionException ex) { return(new ErrorResult(ex).AsResult <TResult>()); } }
public async Task TestRaidedRedisFailureRecovery() { // It is important to set ThrowRedisException to false, because redis exceptions are recoverable // and we don't want to run this test for too long because of exponential back-off recovery algorithm var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = -1, ThrowRedisException = false }; var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = -1, ThrowRedisException = false }; // Setup Redis DB adapter var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb); var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace); var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb); var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace); var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter); var context = new OperationContext(new Context(Logger)); var retryWindow = TimeSpan.FromSeconds(1); // Running for the first time, both operation should be successful. var r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); r.primary.ShouldBeSuccess(); r.secondary.ShouldBeSuccess(); secondaryDb.FailNextOperation(); r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); r.primary.ShouldBeSuccess(); // The second redis should fail when we'll try to use it the second time. r.secondary.ShouldBeError(); primaryDb.FailNextOperation(); r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); // Now all the instance should fail. r.primary.ShouldBeError(); r.secondary.ShouldBeSuccess(); primaryDb.FailNextOperation(); secondaryDb.FailNextOperation(); r = await raidedDatabaseAdapter.ExecuteRaidedAsync(context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); // Now all the instance should fail. r.primary.ShouldBeError(); r.secondary.ShouldBeError(); }
public static RedisMemoizationStore Create( ILogger logger, IConnectionStringProvider connectionStringProvider, string keyspace, IClock clock, TimeSpan memoizationExpiryTime) { var context = new Context(logger); var redisFactory = RedisDatabaseFactory.CreateAsync(context, connectionStringProvider).GetAwaiter().GetResult(); var redisAdapter = new RedisDatabaseAdapter(redisFactory, keyspace); return(new RedisMemoizationStore(logger, clock, redisAdapter, memoizationExpiryTime)); }
/// <inheritdoc /> protected override RedisGlobalStore CreateRedisGlobalStore() { var primaryConnection = MockRedisDatabaseFactory.CreateConnection(_primaryRedisDatabase); var primaryDatabaseAdapter = new RedisDatabaseAdapter( RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection).GetAwaiter().GetResult(), DefaultKeySpace); var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(_secondaryRedisDatabase); var secondaryDatabaseAdapter = new RedisDatabaseAdapter( RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection).GetAwaiter().GetResult(), DefaultKeySpace); return(new RedisGlobalStore(Clock, Configuration, primaryDatabaseAdapter, secondaryDatabaseAdapter, primaryDatabaseAdapter, secondaryDatabaseAdapter)); }
protected override IMemoizationStore CreateStore(DisposableDirectory testDirectory) { var context = new Context(_logger); var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock); var connectionString = localDatabase.ConnectionString; _databasesToDispose.Add(localDatabase); var connectionStringProvider = new LiteralConnectionStringProvider(connectionString); var redisFactory = RedisDatabaseFactory.CreateAsync(context, connectionStringProvider).GetAwaiter().GetResult(); var redisAdapter = new RedisDatabaseAdapter(redisFactory, keySpace: Guid.NewGuid().ToString()); var memoizationDb = new RedisMemoizationDatabase(redisAdapter, _clock, _memoizationExpiryTime); return(new RedisMemoizationStore(_logger, memoizationDb)); }
public async Task TheClientReconnectsWhenTheNumberOfConnectionIssuesExceedsTheLimit() { // This test checks that if the client fails to connect to redis, it'll successfully reconnect to it. // Setup test DB configured to fail 2nd query with normal Exception var testDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { // No queries will fail, instead GetDatabase will throw with RedisConnectionException. FailingQuery = -1, }; int numberOfFactoryCalls = 0; // Setup Redis DB adapter Func <IConnectionMultiplexer> connectionMultiplexerFactory = () => { numberOfFactoryCalls++; // Failing connection error only from the first instance. return(MockRedisDatabaseFactory.CreateConnection(testDb, testBatch: null, throwConnectionExceptionOnGet: numberOfFactoryCalls == 1)); }; var redisDatabaseFactory = await RedisDatabaseFactory.CreateAsync(connectionMultiplexerFactory, connectionMultiplexer => BoolResult.SuccessTask); var dbAdapter = new RedisDatabaseAdapter( redisDatabaseFactory, DefaultKeySpace, // If the operation fails we'll retry once and after that we should reset the connection multiplexer so the next operation should create a new one. redisConnectionErrorLimit: 2, retryCount: 1); // Create a batch query var redisBatch = dbAdapter.CreateBatchOperation(RedisOperation.All); // Execute the batch var result = await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch, default(CancellationToken)); // The first execute batch should fail with the connectivity issue. result.ShouldBeError(); numberOfFactoryCalls.Should().Be(1); var redisBatch2 = dbAdapter.CreateBatchOperation(RedisOperation.All); // Then we should recreate the connection and the second one should be successful. await dbAdapter.ExecuteBatchOperationAsync(new Context(TestGlobal.Logger), redisBatch2, default(CancellationToken)).ShouldBeSuccess(); numberOfFactoryCalls.Should().Be(2); }
/// <nodoc /> public RedisGlobalStore(IClock clock, RedisContentLocationStoreConfiguration configuration, MachineLocation localMachineLocation, RedisDatabaseAdapter primaryRedisDb, RedisDatabaseAdapter secondaryRedisDb) { Contract.Requires(configuration.CentralStore != null); _clock = clock; _configuration = configuration; _primaryRedisDb = primaryRedisDb; _secondaryRedisDb = secondaryRedisDb; var checkpointKeyBase = configuration.CentralStore.CentralStateKeyBase; _checkpointsKey = configuration.GetCheckpointPrefix() + ".Checkpoints"; _masterLeaseKey = checkpointKeyBase + ".MasterLease"; _clusterStateKey = checkpointKeyBase + ".ClusterState"; LocalMachineLocation = localMachineLocation; _blobAdapter = new RedisBlobAdapter(_primaryRedisDb, TimeSpan.FromMinutes(_configuration.BlobExpiryTimeMinutes), _configuration.MaxBlobCapacity, _clock, Tracer); }
private static async Task IncrementWithExpiryValidate( Context context, RedisDatabaseAdapter adapter, ITestRedisDatabase database, string key, uint comparisonValue, TimeSpan specifiedExpiry, int requestedIncrement, long expectedReturnValue, long?expectedIncrementedValue, TimeSpan?expectedDifferentExpiry = null) { var batch = adapter.CreateBatchOperation(RedisOperation.All); var redisKey = GetKey(key); var incrementWithExpire = batch.TryStringIncrementBumpExpiryIfBelowOrEqualValueAsync(key, comparisonValue, timeToLive: specifiedExpiry, requestedIncrement: requestedIncrement); await adapter.ExecuteBatchOperationAsync(context, batch, CancellationToken.None).IgnoreFailure(); var incrementedValue = await incrementWithExpire; Assert.Equal(expectedReturnValue, incrementedValue.AppliedIncrement); var keysWithExpiry = database.GetDbWithExpiry(); if (expectedIncrementedValue == null) { Assert.False(keysWithExpiry.ContainsKey(redisKey)); Assert.Equal(expectedReturnValue, incrementedValue.IncrementedValue); return; } Assert.True(keysWithExpiry.ContainsKey(redisKey)); var expiry = keysWithExpiry[redisKey]; if (expectedDifferentExpiry != null) { Assert.False(expiry.Equals(new MockRedisValueWithExpiry(expectedIncrementedValue, DateTime.UtcNow + specifiedExpiry))); Assert.True(expiry.Equals(new MockRedisValueWithExpiry(expectedIncrementedValue, DateTime.UtcNow + expectedDifferentExpiry.Value))); } else { Assert.True(expiry.Equals(new MockRedisValueWithExpiry(expectedIncrementedValue, DateTime.UtcNow + specifiedExpiry))); } }
/// <nodoc /> public RedisGlobalStore( IClock clock, RedisContentLocationStoreConfiguration configuration, RedisDatabaseAdapter primaryRedisDb, RedisDatabaseAdapter secondaryRedisDb) { Contract.Requires(configuration.CentralStore != null); _clock = clock; _configuration = configuration; _raidedRedis = new RaidedRedisDatabase(Tracer, primaryRedisDb, secondaryRedisDb); var checkpointKeyBase = configuration.CentralStore.CentralStateKeyBase; _checkpointsKey = new ReplicatedRedisHashKey(configuration.GetCheckpointPrefix() + ".Checkpoints", this, _clock, _raidedRedis); _masterLeaseKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".MasterLease", this, _clock, _raidedRedis); _clusterStateKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".ClusterState", this, _clock, _raidedRedis); _blobAdapter = new RedisBlobAdapter(_raidedRedis.PrimaryRedisDb, TimeSpan.FromMinutes(_configuration.BlobExpiryTimeMinutes), _configuration.MaxBlobCapacity, _clock); }
private async Task RunTest( Func <OperationContext, ContentMetadataEventStream, IFailureController, IFailureController, Task> runTestAsync, ContentMetadataEventStreamConfiguration contentMetadataEventStreamConfiguration = null, RedisVolatileEventStorageConfiguration redisVolatileEventLogConfiguration = null, FailureMode persistentStorageFailure = FailureMode.None, FailureMode volatileStorageFailure = FailureMode.None) { var tracingContext = new Context(Logger); var operationContext = new OperationContext(tracingContext); redisVolatileEventLogConfiguration ??= new RedisVolatileEventStorageConfiguration(); using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance); var primaryFactory = await RedisDatabaseFactory.CreateAsync( operationContext, new LiteralConnectionStringProvider(database.ConnectionString), new RedisConnectionMultiplexerConfiguration() { LoggingSeverity = Severity.Error }); var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace"); var redisVolatileEventStorage = new RedisWriteAheadEventStorage(redisVolatileEventLogConfiguration, primaryDatabaseAdapter); var mockPersistentEventStorage = new MockPersistentEventStorage(); var volatileEventStorage = new FailingVolatileEventStorage(volatileStorageFailure, redisVolatileEventStorage); var persistentEventStorage = new FailingPersistentEventStorage(persistentStorageFailure, mockPersistentEventStorage); contentMetadataEventStreamConfiguration ??= new ContentMetadataEventStreamConfiguration(); var contentMetadataEventStream = new ContentMetadataEventStream(contentMetadataEventStreamConfiguration, volatileEventStorage, persistentEventStorage); await contentMetadataEventStream.StartupAsync(operationContext).ThrowIfFailure(); await contentMetadataEventStream.CompleteOrChangeLogAsync(operationContext, CheckpointLogId.InitialLogId); contentMetadataEventStream.SetIsLogging(true); await runTestAsync(operationContext, contentMetadataEventStream, volatileEventStorage, persistentEventStorage); await contentMetadataEventStream.ShutdownAsync(operationContext).ThrowIfFailure(); }
/// <nodoc /> internal RedisGlobalStore( IClock clock, RedisContentLocationStoreConfiguration configuration, RedisDatabaseAdapter primaryRedisDb, RedisDatabaseAdapter secondaryRedisDb, RedisDatabaseAdapter primaryRedisBlobDb, RedisDatabaseAdapter secondaryRedisBlobDb) { Contract.Requires(configuration.CentralStore != null); _clock = clock; Configuration = configuration; RaidedRedis = new RaidedRedisDatabase(Tracer, primaryRedisDb, secondaryRedisDb); var checkpointKeyBase = configuration.CentralStore.CentralStateKeyBase; _checkpointsKey = new ReplicatedRedisHashKey(configuration.GetCheckpointPrefix() + ".Checkpoints", this, _clock, RaidedRedis); _masterLeaseKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".MasterLease", this, _clock, RaidedRedis); _clusterStateKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".ClusterState", this, _clock, RaidedRedis); PrimaryBlobAdapter = new RedisBlobAdapter(primaryRedisBlobDb, _clock, Configuration); SecondaryBlobAdapter = new RedisBlobAdapter(secondaryRedisBlobDb, _clock, Configuration); }
public async Task SlowOperationTimesOut() { // Setup test DB configured to fail 2nd query with Redis Exception var primaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = -1 }; // The second redis will throw RedisException, because we want to use retry strategy here and see the cancellation happening. var secondaryDb = new FailureInjectingRedisDatabase(SystemClock.Instance, InitialTestData) { FailingQuery = -1, ThrowRedisException = true }; // Setup Redis DB adapter var primaryConnection = MockRedisDatabaseFactory.CreateConnection(primaryDb); var primaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), primaryConnection), DefaultKeySpace); var secondaryConnection = MockRedisDatabaseFactory.CreateConnection(secondaryDb); var secondaryAdapter = new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), secondaryConnection), DefaultKeySpace); var raidedDatabaseAdapter = new RaidedRedisDatabase(new Tracer("Test"), primaryAdapter, secondaryAdapter); var context = new OperationContext(new Context(Logger)); var retryWindow = TimeSpan.FromSeconds(1); // All the operations in the secondary instance will fail all the time. secondaryDb.FailNextOperation(resetFailureAutomatically: false); var r = await raidedDatabaseAdapter.ExecuteRaidedAsync( context, (adapter, token) => ExecuteAsync(context, adapter, token), retryWindow, concurrent : true); r.primary.ShouldBeSuccess(); // The secondary result is null is an indication that the operation was canceled. r.secondary.Should().BeNull(); }
private Task <Result <HashEntry[]> > UpdateLocalClusterStateAsync(OperationContext context, ClusterState clusterState, RedisDatabaseAdapter redisDb) { return(redisDb.ExecuteBatchAsync(context, async batch => { var heartbeatResultTask = CallHeartbeatAsync(context, batch, MachineState.Active); var getUnknownMachinesTask = batch.GetUnknownMachinesAsync( _clusterStateKey, clusterState.MaxMachineId); // Only master should mirror cluster state bool shouldMirrorClusterState = _role == Role.Master && HasSecondary && _configuration.MirrorClusterState // Only mirror after a long interval, but not long enough to allow machines to appear expired && !_lastClusterStateMirrorTime.IsRecent(_clock.UtcNow, _configuration.ClusterStateMirrorInterval) // Only mirror from primary to secondary, so no need to dump cluster state if this is the secondary && IsPrimary(redisDb); Task <HashEntry[]> dumpClusterStateBlobTask = shouldMirrorClusterState ? batch.AddOperation(_clusterStateKey, b => b.HashGetAllAsync(_clusterStateKey)) : _emptyClusterStateDump; await Task.WhenAll(heartbeatResultTask, getUnknownMachinesTask, dumpClusterStateBlobTask); var clusterStateBlob = await dumpClusterStateBlobTask ?? CollectionUtilities.EmptyArray <HashEntry>(); var heartbeatResult = await heartbeatResultTask; var getUnknownMachinesResult = await getUnknownMachinesTask; if (shouldMirrorClusterState) { _lastClusterStateMirrorTime = _clock.UtcNow; } if (getUnknownMachinesResult.maxMachineId < LocalMachineId.Index) { return Result.FromErrorMessage <HashEntry[]>($"Invalid {GetDbName(redisDb)} redis cluster state on machine {LocalMachineId} (max machine id={getUnknownMachinesResult.maxMachineId})"); } if (heartbeatResult.priorState == MachineState.Unavailable || heartbeatResult.priorState == MachineState.Expired) { clusterState.LastInactiveTime = _clock.UtcNow; } if (getUnknownMachinesResult.maxMachineId != clusterState.MaxMachineId) { Tracer.Debug(context, $"Retrieved unknown machines from ({clusterState.MaxMachineId}, {getUnknownMachinesResult.maxMachineId}]"); foreach (var item in getUnknownMachinesResult.unknownMachines) { context.LogMachineMapping(Tracer, item.Key, item.Value); } } clusterState.AddUnknownMachines(getUnknownMachinesResult.maxMachineId, getUnknownMachinesResult.unknownMachines); clusterState.SetInactiveMachines(heartbeatResult.inactiveMachineIdSet); Tracer.Debug(context, $"Inactive machines: Count={heartbeatResult.inactiveMachineIdSet.Count}, [{string.Join(", ", heartbeatResult.inactiveMachineIdSet)}]"); Tracer.TrackMetric(context, "InactiveMachineCount", heartbeatResult.inactiveMachineIdSet.Count); return Result.Success(await dumpClusterStateBlobTask ?? CollectionUtilities.EmptyArray <HashEntry>()); }, RedisOperation.UpdateClusterState)); }
private bool IsPrimary(RedisDatabaseAdapter redisDb) { return(redisDb == _primaryRedisDb); }
private string GetDbName(RedisDatabaseAdapter redisDb) { return(redisDb == _primaryRedisDb ? "primary" : "secondary"); }