/// <summary>
/// 持续观察Leader状态，并在合适的时机发起选举
/// </summary>
/// <param name="leaderElectCompletedHandler">leader选举完毕处理方法</param>
/// <param name="cancellationToken"></param>
public void Watch(Action <LeaderElectionResult> leaderElectCompletedHandler, CancellationToken cancellationToken = default)
{
// 启动一个线程进行处理，不阻塞当前线程
_watchThread = new Thread(new ThreadStart(() =>
{
// 上来就先选举一次，以获取要监控的状态
_electState = Elect(leaderElectCompletedHandler, cancellationToken);
do
{
cancellationToken.ThrowIfCancellationRequested();
try
{
Console.WriteLine("start watch state...");
_election.WatchState(_electState, newState =>
{
Console.WriteLine("start process latest state...");
ProcessState(leaderElectCompletedHandler, newState, cancellationToken);
}, cancellationToken);
}
catch (Exception ex)
{
Console.WriteLine(ex);
cancellationToken.ThrowIfCancellationRequested();
Thread.Sleep(3000);
}
}while (true);
}))
{
IsBackground = true
};
_watchThread.Start();
}
private void ProcessElectionFlagCreated(Action <LeaderElectionState> processLatestState)
{
Console.WriteLine("start process election flag created...");
var flag = _zkElectionClient.GetData(_electionFlag);
var newState = new LeaderElectionState()
{
IsLeaderOnline = flag != null ? true : false,
CurrentLeaderId = flag,
Data = flag
};
processLatestState?.Invoke(newState);
}
private void ProcessReconnectWithElectionFlag(Action <LeaderElectionState> processLatestState)
{
Console.WriteLine("start process reconnect...");
var stableLeader = _zkElectionClient.GetData(_electionFlag);
var newState = new LeaderElectionState()
{
IsLeaderOnline = stableLeader != null ? true : false,
CurrentLeaderId = stableLeader,
Data = stableLeader
};
processLatestState?.Invoke(newState);
}
private void ProcessElectionFlagDeleted(Action <LeaderElectionState> processLatestState)
{
Console.WriteLine("start process election flag deleted...");
// flag deleted, read stable leader info
var stableLeader = _zkElectionClient.GetData(_electionLeader);
var newState = new LeaderElectionState()
{
IsLeaderOnline = false,
CurrentLeaderId = stableLeader,
Data = stableLeader
};
processLatestState?.Invoke(newState);
}
/// <summary>
/// 观察选举状态，直到状态变化或者超时
/// </summary>
/// <param name="state">上一次获取的状态，作为实时状态的对比参照，如果两个状态不同则说明选举状态发生了变化</param>
/// <param name="processLatestState">收到最新状态后的处理方法，可能是状态变化或者超时触发此操作</param>
/// <param name="cancellationToken"></param>
public void WatchState(LeaderElectionState state, Action <LeaderElectionState> processLatestState, CancellationToken cancellationToken = default)
{
if (_electionWatcher == null)
{
_electionWatcher = new ZkElectionPathWatcher((leaderPath, eventType) =>
{
if (eventType == EventType.NodeDeleted)
{
ProcessElectionFlagDeleted(processLatestState);
}
}, _electionDataChangedEvent);
}
// zk不可用或连接不上
if (_isDisconnected)
{
var newState = new LeaderElectionState()
{
IsDisconnected = _isDisconnected
};
processLatestState?.Invoke(newState);
return;
}
var flagExists = _zkElectionClient.Exists(_electionFlag, _electionWatcher);
if (flagExists != null)
{
// zk又连接上了
if (state != null && state.IsDisconnected && !_isDisconnected)
{
ProcessReconnectWithElectionFlag(processLatestState);
return;
}
Console.WriteLine("wait election flag change");
_electionDataChangedEvent.WaitOne();
}
else
{
// 非leader节点将确认X次，每次确认需要等待一段时间
Thread.Sleep(_leaderOfflineConfirmInterval);
ProcessElectionFlag(processLatestState);
}
}
/// <summary>
/// 观察选举状态，直到状态变化或者超时
/// </summary>
/// <param name="state">上一次获取的状态，作为实时状态的对比参照，如果两个状态不同则说明选举状态发生了变化</param>
/// <param name="processLatestState">收到最新状态后的处理方法，可能是状态变化或者超时触发此操作</param>
/// <param name="cancellationToken"></param>
public void WatchState(LeaderElectionState state, Action <LeaderElectionState> processLatestState, CancellationToken cancellationToken = default)
{
KVPair consulKv = state != null ? (KVPair)state.Data : null;
ulong waitIndex = consulKv == null ? 0 : consulKv.ModifyIndex++;
TimeSpan waitTime = state.IsLeaderOnline ? _defaultWatchInterval : _leaderOfflineConfirmInterval;
KVPair newConsulKv = null;
bool isDisconnected = false;
try
{
if (waitTime.TotalSeconds <= 0)
{
newConsulKv = _consulElectionClient.Get(_electionKey, cancellationToken);
}
else
{
newConsulKv = _consulElectionClient.BlockGet(_electionKey, waitTime, waitIndex, cancellationToken);
}
}
catch
{
// consul不可用或连接不上
isDisconnected = true;
}
LeaderElectionState newState = new LeaderElectionState()
{
IsDisconnected = isDisconnected
};
// 有选举数据
if (newConsulKv != null)
{
newState.Data = newConsulKv;
newState.CurrentLeaderId = Encoding.UTF8.GetString(newConsulKv.Value);
newState.IsLeaderOnline = !string.IsNullOrWhiteSpace(newConsulKv.Session);
}
processLatestState?.Invoke(newState);
}
/// <summary>
/// 处理选举状态
/// </summary>
/// <param name="leaderElectCompletedHandler"></param>
/// <param name="newState"></param>
/// <param name="cancellationToken"></param>
private void ProcessState(Action <LeaderElectionResult> leaderElectCompletedHandler, LeaderElectionState newState, CancellationToken cancellationToken)
{
Console.WriteLine("newState: " + JsonConvert.SerializeObject(newState));
var lastIsDisconnected = _electState.IsDisconnected;
_electState = newState;
// 本地下线，远程在线，还是leader的处理
if (lastIsDisconnected &&
newState.CurrentLeaderId == _currentServiceId &&
newState.IsLeaderOnline)
{
Console.WriteLine("leader back online");
leaderElectCompletedHandler?.Invoke(new LeaderElectionResult()
{
IsSuccess = true,
State = newState
});
return;
}
// 断开连接了，自动选举失败
if (newState.IsDisconnected)
{
leaderElectCompletedHandler?.Invoke(new LeaderElectionResult()
{
IsSuccess = false,
State = newState
});
return;
}
// 为空代表全局选举状态不存在或者被移除，则马上启动选举
if (string.IsNullOrWhiteSpace(newState.CurrentLeaderId))
{
_electState = Elect(leaderElectCompletedHandler, cancellationToken);
return;
}
// Leader下线处理
if (!newState.IsLeaderOnline)
{
// 下线的Leader有优先选举权
if (newState.CurrentLeaderId == _currentServiceId)
{
_electState = Elect(leaderElectCompletedHandler, cancellationToken);
return;
}
// 其它节点需要确认Leader真的下线了才能发起选举
// 真的下线通过连续多次监听到的状态为下线进行认定
if (_offlineConfirmNumber == 0)
{
_electState = Elect(leaderElectCompletedHandler, cancellationToken);
// 有选举出新的Leader才需要重新确认
if (_electState != null && _electState.IsLeaderOnline)
{
_offlineConfirmNumber = _defaultOfflineConfirmNumber;
}
return;
}
Console.WriteLine("leader offline, need confirm: " + _offlineConfirmNumber);
_offlineConfirmNumber--;
return;
}
_offlineConfirmNumber = _defaultOfflineConfirmNumber;
}
