private async Task TryMirrorRedisHashDataAsync(OperationContext context, RedisDatabaseAdapter source, RedisDatabaseAdapter target, long?postMirrorSourceVersion = null) { await context.PerformOperationAsync( _host.Tracer, async() => { var sourceDump = await source.ExecuteBatchAsync(context, b => b.AddOperation(_key, b => b.KeyDumpAsync(_key)), RedisOperation.HashGetKeys); await target.ExecuteBatchAsync(context, b => { var deleteTask = b.AddOperation(_key, b => b.KeyDeleteAsync(_key)); var restoreTask = b.AddOperation(_key, b => b.KeyRestoreAsync(_key, sourceDump).WithResultAsync(Unit.Void)); return(Task.WhenAll(deleteTask, restoreTask).WithResultAsync(Unit.Void)); }, RedisOperation.HashDeleteAndRestore); if (postMirrorSourceVersion.HasValue) { await source.ExecuteBatchAsync(context, b => b.AddOperation(_key, b => b.HashSetAsync(_key, nameof(ReplicatedHashVersionNumber), postMirrorSourceVersion.Value)), RedisOperation.HashSetValue); } return(Result.Success(sourceDump.Length)); }, extraStartMessage : $"({_redis.GetDbName(source)} -> {_redis.GetDbName(target)}) Key={_key}, PostMirrorSourceVersion={postMirrorSourceVersion ?? -1L}", extraEndMessage : r => $"({_redis.GetDbName(source)} -> {_redis.GetDbName(target)}) Key={_key}, Length={r.GetValueOrDefault(-1)}").IgnoreFailure(); }
private Task <bool> CompareExchangeInternalAsync(OperationContext context, StrongFingerprint strongFingerprint, string expectedReplacementToken, ContentHashListWithDeterminism expected, ContentHashListWithDeterminism replacement, string newReplacementToken) { var key = GetKey(strongFingerprint.WeakFingerprint); var replacementMetadata = new MetadataEntry(replacement, _clock.UtcNow); var replacementBytes = SerializeMetadataEntry(replacementMetadata); byte[] selectorBytes = SerializeSelector(strongFingerprint.Selector, isReplacementToken: false); byte[] tokenFieldNameBytes = SerializeSelector(strongFingerprint.Selector, isReplacementToken: true); return(_redis.ExecuteBatchAsync(context, batch => { var task = batch.CompareExchangeAsync(key, selectorBytes, tokenFieldNameBytes, expectedReplacementToken, replacementBytes, newReplacementToken); batch.KeyExpireAsync(key, _metadataExpiryTime).FireAndForget(context); return task; }, RedisOperation.CompareExchange)); }
private async Task <Result <Role> > UpdateRoleAsync(OperationContext context, bool release) { return(await context.PerformOperationAsync <Result <Role> >( Tracer, async() => { // This mutex ensure that Release of master role during shutdown and Heartbeat role acquisition are synchronized. // Ensuring that a heartbeat during shutdown doesn't trigger the released master role to be acquired again. using (await _roleMutex.AcquireAsync()) { if (ShutdownStarted) { // Don't acquire a role during shutdown return Role.Worker; } var configuredRole = _configuration.Checkpoint?.Role; if (configuredRole != null) { return configuredRole.Value; } var localMachineName = LocalMachineLocation.ToString(); var masterAcquisitonResult = await _primaryRedisDb.ExecuteBatchAsync(context, batch => batch.AcquireMasterRoleAsync( masterRoleRegistryKey: _masterLeaseKey, machineName: localMachineName, currentTime: _clock.UtcNow, leaseExpiryTime: _configuration.Checkpoint.MasterLeaseExpiryTime, // 1 master only is allowed. This should be changed if more than one master becomes a possible configuration slotCount: 1, release: release ), RedisOperation.UpdateRole); if (release) { Tracer.Debug(context, $"'{localMachineName}' released master role."); return Role.Worker; } if (masterAcquisitonResult != null) { var priorMachineName = masterAcquisitonResult.Value.PriorMasterMachineName; if (priorMachineName != localMachineName || masterAcquisitonResult.Value.PriorMachineStatus != SlotStatus.Acquired) { Tracer.Debug(context, $"'{localMachineName}' acquired master role from '{priorMachineName}', Status: '{masterAcquisitonResult?.PriorMachineStatus}', LastHeartbeat: '{masterAcquisitonResult?.PriorMasterLastHeartbeat}'"); } return Role.Master; } else { return Role.Worker; } } }, Counters[GlobalStoreCounters.UpdateRole]).FireAndForgetAndReturnTask(context)); }
/// <inheritdoc /> public Task <BoolResult> UpdateClusterStateAsync(OperationContext context, ClusterState clusterState) { return(context.PerformOperationAsync( Tracer, async() => { HashEntry[] clusterStateDump = await ExecuteRedisFallbackAsync(context, redisDb => UpdateLocalClusterStateAsync(context, clusterState, redisDb)).ThrowIfFailureAsync(); if (clusterStateDump.Length != 0 && HasSecondary && _configuration.MirrorClusterState) { Tracer.Debug(context, $"Mirroring cluster state with '{clusterStateDump.Length}' entries to secondary"); await _secondaryRedisDb.ExecuteBatchAsync(context, batch => batch.AddOperation(_clusterStateKey, async b => { await b.HashSetAsync(_clusterStateKey, clusterStateDump); return Unit.Void; }), RedisOperation.MirrorClusterState).FireAndForgetErrorsAsync(context); } return BoolResult.Success; }, Counters[GlobalStoreCounters.UpdateClusterState])); }
private Task <Result <HashEntry[]> > UpdateLocalClusterStateAsync(OperationContext context, ClusterState clusterState, RedisDatabaseAdapter redisDb) { return(redisDb.ExecuteBatchAsync(context, async batch => { var heartbeatResultTask = CallHeartbeatAsync(context, batch, MachineState.Active); var getUnknownMachinesTask = batch.GetUnknownMachinesAsync( _clusterStateKey, clusterState.MaxMachineId); // Only master should mirror cluster state bool shouldMirrorClusterState = _role == Role.Master && HasSecondary && _configuration.MirrorClusterState // Only mirror after a long interval, but not long enough to allow machines to appear expired && !_lastClusterStateMirrorTime.IsRecent(_clock.UtcNow, _configuration.ClusterStateMirrorInterval) // Only mirror from primary to secondary, so no need to dump cluster state if this is the secondary && IsPrimary(redisDb); Task <HashEntry[]> dumpClusterStateBlobTask = shouldMirrorClusterState ? batch.AddOperation(_clusterStateKey, b => b.HashGetAllAsync(_clusterStateKey)) : _emptyClusterStateDump; await Task.WhenAll(heartbeatResultTask, getUnknownMachinesTask, dumpClusterStateBlobTask); var clusterStateBlob = await dumpClusterStateBlobTask ?? CollectionUtilities.EmptyArray <HashEntry>(); var heartbeatResult = await heartbeatResultTask; var getUnknownMachinesResult = await getUnknownMachinesTask; if (shouldMirrorClusterState) { _lastClusterStateMirrorTime = _clock.UtcNow; } if (getUnknownMachinesResult.maxMachineId < LocalMachineId.Index) { return Result.FromErrorMessage <HashEntry[]>($"Invalid {GetDbName(redisDb)} redis cluster state on machine {LocalMachineId} (max machine id={getUnknownMachinesResult.maxMachineId})"); } if (heartbeatResult.priorState == MachineState.Unavailable || heartbeatResult.priorState == MachineState.Expired) { clusterState.LastInactiveTime = _clock.UtcNow; } if (getUnknownMachinesResult.maxMachineId != clusterState.MaxMachineId) { Tracer.Debug(context, $"Retrieved unknown machines from ({clusterState.MaxMachineId}, {getUnknownMachinesResult.maxMachineId}]"); foreach (var item in getUnknownMachinesResult.unknownMachines) { context.LogMachineMapping(Tracer, item.Key, item.Value); } } clusterState.AddUnknownMachines(getUnknownMachinesResult.maxMachineId, getUnknownMachinesResult.unknownMachines); clusterState.SetInactiveMachines(heartbeatResult.inactiveMachineIdSet); Tracer.Debug(context, $"Inactive machines: Count={heartbeatResult.inactiveMachineIdSet.Count}, [{string.Join(", ", heartbeatResult.inactiveMachineIdSet)}]"); Tracer.TrackMetric(context, "InactiveMachineCount", heartbeatResult.inactiveMachineIdSet.Count); return Result.Success(await dumpClusterStateBlobTask ?? CollectionUtilities.EmptyArray <HashEntry>()); }, RedisOperation.UpdateClusterState)); }