/// <summary> /// 持续观察Leader状态,并在合适的时机发起选举 /// </summary> /// <param name="leaderElectCompletedHandler">leader选举完毕处理方法</param> /// <param name="cancellationToken"></param> public void Watch(Action <LeaderElectionResult> leaderElectCompletedHandler, CancellationToken cancellationToken = default) { // 启动一个线程进行处理,不阻塞当前线程 _watchThread = new Thread(new ThreadStart(() => { // 上来就先选举一次,以获取要监控的状态 _electState = Elect(leaderElectCompletedHandler, cancellationToken); do { cancellationToken.ThrowIfCancellationRequested(); try { Console.WriteLine("start watch state..."); _election.WatchState(_electState, newState => { Console.WriteLine("start process latest state..."); ProcessState(leaderElectCompletedHandler, newState, cancellationToken); }, cancellationToken); } catch (Exception ex) { Console.WriteLine(ex); cancellationToken.ThrowIfCancellationRequested(); Thread.Sleep(3000); } }while (true); })) { IsBackground = true }; _watchThread.Start(); }
private void ProcessElectionFlagCreated(Action <LeaderElectionState> processLatestState) { Console.WriteLine("start process election flag created..."); var flag = _zkElectionClient.GetData(_electionFlag); var newState = new LeaderElectionState() { IsLeaderOnline = flag != null ? true : false, CurrentLeaderId = flag, Data = flag }; processLatestState?.Invoke(newState); }
private void ProcessReconnectWithElectionFlag(Action <LeaderElectionState> processLatestState) { Console.WriteLine("start process reconnect..."); var stableLeader = _zkElectionClient.GetData(_electionFlag); var newState = new LeaderElectionState() { IsLeaderOnline = stableLeader != null ? true : false, CurrentLeaderId = stableLeader, Data = stableLeader }; processLatestState?.Invoke(newState); }
private void ProcessElectionFlagDeleted(Action <LeaderElectionState> processLatestState) { Console.WriteLine("start process election flag deleted..."); // flag deleted, read stable leader info var stableLeader = _zkElectionClient.GetData(_electionLeader); var newState = new LeaderElectionState() { IsLeaderOnline = false, CurrentLeaderId = stableLeader, Data = stableLeader }; processLatestState?.Invoke(newState); }
/// <summary> /// 观察选举状态,直到状态变化或者超时 /// </summary> /// <param name="state">上一次获取的状态,作为实时状态的对比参照,如果两个状态不同则说明选举状态发生了变化</param> /// <param name="processLatestState">收到最新状态后的处理方法,可能是状态变化或者超时触发此操作</param> /// <param name="cancellationToken"></param> public void WatchState(LeaderElectionState state, Action <LeaderElectionState> processLatestState, CancellationToken cancellationToken = default) { if (_electionWatcher == null) { _electionWatcher = new ZkElectionPathWatcher((leaderPath, eventType) => { if (eventType == EventType.NodeDeleted) { ProcessElectionFlagDeleted(processLatestState); } }, _electionDataChangedEvent); } // zk不可用或连接不上 if (_isDisconnected) { var newState = new LeaderElectionState() { IsDisconnected = _isDisconnected }; processLatestState?.Invoke(newState); return; } var flagExists = _zkElectionClient.Exists(_electionFlag, _electionWatcher); if (flagExists != null) { // zk又连接上了 if (state != null && state.IsDisconnected && !_isDisconnected) { ProcessReconnectWithElectionFlag(processLatestState); return; } Console.WriteLine("wait election flag change"); _electionDataChangedEvent.WaitOne(); } else { // 非leader节点将确认X次,每次确认需要等待一段时间 Thread.Sleep(_leaderOfflineConfirmInterval); ProcessElectionFlag(processLatestState); } }
/// <summary> /// 观察选举状态,直到状态变化或者超时 /// </summary> /// <param name="state">上一次获取的状态,作为实时状态的对比参照,如果两个状态不同则说明选举状态发生了变化</param> /// <param name="processLatestState">收到最新状态后的处理方法,可能是状态变化或者超时触发此操作</param> /// <param name="cancellationToken"></param> public void WatchState(LeaderElectionState state, Action <LeaderElectionState> processLatestState, CancellationToken cancellationToken = default) { KVPair consulKv = state != null ? (KVPair)state.Data : null; ulong waitIndex = consulKv == null ? 0 : consulKv.ModifyIndex++; TimeSpan waitTime = state.IsLeaderOnline ? _defaultWatchInterval : _leaderOfflineConfirmInterval; KVPair newConsulKv = null; bool isDisconnected = false; try { if (waitTime.TotalSeconds <= 0) { newConsulKv = _consulElectionClient.Get(_electionKey, cancellationToken); } else { newConsulKv = _consulElectionClient.BlockGet(_electionKey, waitTime, waitIndex, cancellationToken); } } catch { // consul不可用或连接不上 isDisconnected = true; } LeaderElectionState newState = new LeaderElectionState() { IsDisconnected = isDisconnected }; // 有选举数据 if (newConsulKv != null) { newState.Data = newConsulKv; newState.CurrentLeaderId = Encoding.UTF8.GetString(newConsulKv.Value); newState.IsLeaderOnline = !string.IsNullOrWhiteSpace(newConsulKv.Session); } processLatestState?.Invoke(newState); }
/// <summary> /// 处理选举状态 /// </summary> /// <param name="leaderElectCompletedHandler"></param> /// <param name="newState"></param> /// <param name="cancellationToken"></param> private void ProcessState(Action <LeaderElectionResult> leaderElectCompletedHandler, LeaderElectionState newState, CancellationToken cancellationToken) { Console.WriteLine("newState: " + JsonConvert.SerializeObject(newState)); var lastIsDisconnected = _electState.IsDisconnected; _electState = newState; // 本地下线,远程在线,还是leader的处理 if (lastIsDisconnected && newState.CurrentLeaderId == _currentServiceId && newState.IsLeaderOnline) { Console.WriteLine("leader back online"); leaderElectCompletedHandler?.Invoke(new LeaderElectionResult() { IsSuccess = true, State = newState }); return; } // 断开连接了,自动选举失败 if (newState.IsDisconnected) { leaderElectCompletedHandler?.Invoke(new LeaderElectionResult() { IsSuccess = false, State = newState }); return; } // 为空代表全局选举状态不存在或者被移除,则马上启动选举 if (string.IsNullOrWhiteSpace(newState.CurrentLeaderId)) { _electState = Elect(leaderElectCompletedHandler, cancellationToken); return; } // Leader下线处理 if (!newState.IsLeaderOnline) { // 下线的Leader有优先选举权 if (newState.CurrentLeaderId == _currentServiceId) { _electState = Elect(leaderElectCompletedHandler, cancellationToken); return; } // 其它节点需要确认Leader真的下线了才能发起选举 // 真的下线通过连续多次监听到的状态为下线进行认定 if (_offlineConfirmNumber == 0) { _electState = Elect(leaderElectCompletedHandler, cancellationToken); // 有选举出新的Leader才需要重新确认 if (_electState != null && _electState.IsLeaderOnline) { _offlineConfirmNumber = _defaultOfflineConfirmNumber; } return; } Console.WriteLine("leader offline, need confirm: " + _offlineConfirmNumber); _offlineConfirmNumber--; return; } _offlineConfirmNumber = _defaultOfflineConfirmNumber; }