public virtual void OnMismatch(long lastRemoteAppendIndex, LeaderContext leaderContext)
{
lock (this)
{
switch (_mode)
{
case Org.Neo4j.causalclustering.core.consensus.shipping.RaftLogShipper.Mode.Mismatch:
long logIndex = max(min(_lastSentIndex - 1, lastRemoteAppendIndex), MIN_INDEX);
SendEmpty(logIndex, leaderContext);
break;
case Org.Neo4j.causalclustering.core.consensus.shipping.RaftLogShipper.Mode.Pipeline:
case Org.Neo4j.causalclustering.core.consensus.shipping.RaftLogShipper.Mode.Catchup:
_log.info("%s: mismatch in mode %s from follower %s, moving to MISMATCH mode", StatusAsString(), _mode, _follower);
_mode = Mode.Mismatch;
SendEmpty(_lastSentIndex, leaderContext);
break;
default:
throw new System.InvalidOperationException("Unknown mode: " + _mode);
}
_lastLeaderContext = leaderContext;
}
}
public void Can_JoinElectionByUseTheSameZKConnection()
{
IZooKeeper zookeeper = new ZooKeeper("127.0.0.1:2181", new TimeSpan(0, 60, 0), null);
Server server1 = new Server()
{
ShardName = "shard1",
Host = "127.0.0.1",
Port = 6369,
paramStr = "allowAdmin=true",
Path = ZkPath.ShardsPath(ZkPath.CollectionName)
};
Server server2 = new Server()
{
ShardName = "shard1",
Host = "127.0.0.1",
Port = 6379,
paramStr = "allowAdmin=true",
Path = ZkPath.ShardsPath(ZkPath.CollectionName)
};
ServerInstances si = new ServerInstances();
si.Add(new ServerInst(server1.Host, server1.Port, "allowAdmin=true"));
si.Add(new ServerInst(server2.Host, server2.Port, "allowAdmin=true"));
LeaderContext context1 = new LeaderContext(zookeeper, server1);
context1.JoinElection();
LeaderContext context2 = new LeaderContext(zookeeper, server2);
context2.JoinElection();
}
public override void ApplyTo(RaftLogShipper raftLogShipper, LeaderContext leaderContext)
{
if (raftLogShipper.Identity().Equals(Target))
{
raftLogShipper.OnMismatch(LastRemoteAppendIndex, leaderContext);
}
}
public override void ApplyTo(RaftLogShipper raftLogShipper, LeaderContext leaderContext)
{
if (raftLogShipper.Identity().Equals(Target))
{
raftLogShipper.OnMatch(NewMatchIndex, leaderContext);
}
}
public virtual void OnMatch(long newMatchIndex, LeaderContext leaderContext)
{
lock (this)
{
bool progress = newMatchIndex > _matchIndex;
if (newMatchIndex > _matchIndex)
{
_matchIndex = newMatchIndex;
}
else
{
_log.warn("%s: match index not progressing. This should be transient.", StatusAsString());
}
switch (_mode)
{
case Org.Neo4j.causalclustering.core.consensus.shipping.RaftLogShipper.Mode.Mismatch:
if (SendNextBatchAfterMatch(leaderContext))
{
_log.info("%s: caught up after mismatch, moving to PIPELINE mode", StatusAsString());
_mode = PIPELINE;
}
else
{
_log.info("%s: starting catch up after mismatch, moving to CATCHUP mode", StatusAsString());
_mode = Mode.Catchup;
}
break;
case Org.Neo4j.causalclustering.core.consensus.shipping.RaftLogShipper.Mode.Catchup:
if (_matchIndex >= _lastSentIndex)
{
if (SendNextBatchAfterMatch(leaderContext))
{
_log.info("%s: caught up, moving to PIPELINE mode", StatusAsString());
_mode = PIPELINE;
}
}
break;
case Org.Neo4j.causalclustering.core.consensus.shipping.RaftLogShipper.Mode.Pipeline:
if (_matchIndex == _lastSentIndex)
{
AbortTimeout();
}
else if (progress)
{
ScheduleTimeout(_retryTimeMillis);
}
break;
default:
throw new System.InvalidOperationException("Unknown mode: " + _mode);
}
_lastLeaderContext = leaderContext;
}
}
private void SendCommitUpdate(LeaderContext leaderContext)
{
/*
* This is a commit update. That means that we just received enough success responses to an append
* request to allow us to send a commit. By Raft invariants, this means that the term for the committed
* entry is the current term.
*/
Org.Neo4j.causalclustering.core.consensus.RaftMessages_Heartbeat appendRequest = new Org.Neo4j.causalclustering.core.consensus.RaftMessages_Heartbeat(_leader, leaderContext.Term, leaderContext.CommitIndex, leaderContext.Term);
_outbound.send(_follower, appendRequest);
}
public virtual void OnCommitUpdate(LeaderContext leaderContext)
{
lock (this)
{
if (_mode == Mode.Pipeline)
{
SendCommitUpdate(leaderContext);
}
_lastLeaderContext = leaderContext;
}
}
private void EnsureLogShipperRunning(MemberId member, LeaderContext leaderContext)
{
RaftLogShipper logShipper = _logShippers[member];
if (logShipper == null && !member.Equals(_myself))
{
logShipper = new RaftLogShipper(_outbound, _logProvider, _raftLog, _clock, _timerService, _myself, member, leaderContext.Term, leaderContext.CommitIndex, _retryTimeMillis, _catchupBatchSize, _maxAllowedShippingLag, _inFlightCache);
_logShippers[member] = logShipper;
logShipper.Start();
}
}
public void UnRegistry(Guid id)
{
log.Info("unregistry on Zk,id=" + id);
ServerInst inst = serverInstances.GetById(id);
if (inst != null)
{
log.Info(inst);
//多次对每个server初始化有没有问题？
LeaderContext leaderContext = new LeaderContext(zookeeper, inst.ServerInfo);
leaderContext.CancelElection();
}
}
internal RaftLogShipper(Outbound <MemberId, Org.Neo4j.causalclustering.core.consensus.RaftMessages_RaftMessage> outbound, LogProvider logProvider, ReadableRaftLog raftLog, Clock clock, TimerService timerService, MemberId leader, MemberId follower, long leaderTerm, long leaderCommit, long retryTimeMillis, int catchupBatchSize, int maxAllowedShippingLag, InFlightCache inFlightCache)
{
this._outbound = outbound;
this._timerService = timerService;
this._catchupBatchSize = catchupBatchSize;
this._maxAllowedShippingLag = maxAllowedShippingLag;
this._log = logProvider.getLog(this.GetType());
this._raftLog = raftLog;
this._clock = clock;
this._follower = follower;
this._leader = leader;
this._retryTimeMillis = retryTimeMillis;
this._lastLeaderContext = new LeaderContext(leaderTerm, leaderCommit);
this._inFlightCache = inFlightCache;
}
public void RegistryZk(Guid id)
{
log.Info("registry on Zk,id=" + id);
ServerInst inst = serverInstances.GetById(id);
if (inst != null)
{
log.Info(inst);
ShardBuilder builder = new ShardBuilder();
Shard shard = builder.AddServer(inst.ServerInfo).Build();
//多次对每个server初始化有没有问题？
LeaderContext leaderContext = new LeaderContext(zookeeper, inst.ServerInfo);
leaderContext.JoinElection();
}
}
public virtual void HandleCommands(IEnumerable <ShipCommand> shipCommands, LeaderContext leaderContext)
{
lock (this)
{
foreach (ShipCommand shipCommand in shipCommands)
{
foreach (RaftLogShipper logShipper in _logShippers.Values)
{
shipCommand.ApplyTo(logShipper, leaderContext);
}
}
_lastLeaderContext = leaderContext;
}
}
/// <summary>
/// Returns true if this sent the last batch.
/// </summary>
private bool SendNextBatchAfterMatch(LeaderContext leaderContext)
{
long lastIndex = _raftLog.appendIndex();
if (lastIndex > _matchIndex)
{
long endIndex = min(lastIndex, _matchIndex + _catchupBatchSize);
ScheduleTimeout(_retryTimeMillis);
SendRange(_matchIndex + 1, endIndex, leaderContext);
return(endIndex == lastIndex);
}
else
{
return(true);
}
}
/// <summary>
/// Resumed when becoming leader.
/// </summary>
public virtual void Resume(LeaderContext initialLeaderContext)
{
lock (this)
{
if (_stopped)
{
return;
}
_running = true;
foreach (MemberId member in _membership.replicationMembers())
{
EnsureLogShipperRunning(member, initialLeaderContext);
}
_lastLeaderContext = initialLeaderContext;
}
}
public override bool Equals(object o)
{
if (this == o)
{
return(true);
}
if (o == null || this.GetType() != o.GetType())
{
return(false);
}
LeaderContext that = ( LeaderContext )o;
if (Term != that.Term)
{
return(false);
}
return(CommitIndex == that.CommitIndex);
}
private void SendRange(long startIndex, long endIndex, LeaderContext leaderContext)
{
if (startIndex > endIndex)
{
return;
}
_lastSentIndex = endIndex;
try
{
int batchSize = ( int )(endIndex - startIndex + 1);
RaftLogEntry[] entries = new RaftLogEntry[batchSize];
long prevLogIndex = startIndex - 1;
long prevLogTerm = _raftLog.readEntryTerm(prevLogIndex);
if (prevLogTerm > leaderContext.Term)
{
_log.warn("%s aborting send. Not leader anymore? %s, prevLogTerm=%d", StatusAsString(), leaderContext, prevLogTerm);
return;
}
bool entryMissing = false;
using (InFlightLogEntryReader logEntrySupplier = new InFlightLogEntryReader(_raftLog, _inFlightCache, false))
{
for (int offset = 0; offset < batchSize; offset++)
{
entries[offset] = logEntrySupplier.Get(startIndex + offset);
if (entries[offset] == null)
{
entryMissing = true;
break;
}
if (entries[offset].Term() > leaderContext.Term)
{
_log.warn("%s aborting send. Not leader anymore? %s, entryTerm=%d", StatusAsString(), leaderContext, entries[offset].Term());
return;
}
}
}
if (entryMissing || DoesNotExistInLog(prevLogIndex, prevLogTerm))
{
if (_raftLog.prevIndex() >= prevLogIndex)
{
SendLogCompactionInfo(leaderContext);
}
else
{
_log.error("%s: Could not send compaction info and entries were missing, but log is not behind.", StatusAsString());
}
}
else
{
Org.Neo4j.causalclustering.core.consensus.RaftMessages_AppendEntries_Request appendRequest = new Org.Neo4j.causalclustering.core.consensus.RaftMessages_AppendEntries_Request(_leader, leaderContext.Term, prevLogIndex, prevLogTerm, entries, leaderContext.CommitIndex);
_outbound.send(_follower, appendRequest);
}
}
catch (IOException e)
{
_log.warn(StatusAsString() + " exception during batch send", e);
}
}
private void SendLogCompactionInfo(LeaderContext leaderContext)
{
_log.warn("Sending log compaction info. Log pruned? Status=%s, LeaderContext=%s", StatusAsString(), leaderContext);
_outbound.send(_follower, new Org.Neo4j.causalclustering.core.consensus.RaftMessages_LogCompactionInfo(_leader, leaderContext.Term, _raftLog.prevIndex()));
}
private void SendNewEntries(long prevLogIndex, long prevLogTerm, RaftLogEntry[] newEntries, LeaderContext leaderContext)
{
ScheduleTimeout(_retryTimeMillis);
_lastSentIndex = prevLogIndex + 1;
Org.Neo4j.causalclustering.core.consensus.RaftMessages_AppendEntries_Request appendRequest = new Org.Neo4j.causalclustering.core.consensus.RaftMessages_AppendEntries_Request(_leader, leaderContext.Term, prevLogIndex, prevLogTerm, newEntries, leaderContext.CommitIndex);
_outbound.send(_follower, appendRequest);
}
public override void ApplyTo(RaftLogShipper raftLogShipper, LeaderContext leaderContext)
{
raftLogShipper.OnNewEntries(PrevLogIndex, PrevLogTerm, NewLogEntries, leaderContext);
}
public override void ApplyTo(RaftLogShipper raftLogShipper, LeaderContext leaderContext)
{
raftLogShipper.OnCommitUpdate(leaderContext);
}
public abstract void ApplyTo(RaftLogShipper raftLogShipper, LeaderContext leaderContext);
public virtual void OnNewEntries(long prevLogIndex, long prevLogTerm, RaftLogEntry[] newLogEntries, LeaderContext leaderContext)
{
lock (this)
{
if (_mode == Mode.Pipeline)
{
while (_lastSentIndex <= prevLogIndex)
{
if (prevLogIndex - _matchIndex <= _maxAllowedShippingLag)
{
// all sending functions update lastSentIndex
SendNewEntries(prevLogIndex, prevLogTerm, newLogEntries, leaderContext);
}
else
{
/* The timer is still set at this point. Either we will send the next batch
* as soon as the follower has caught up with the last pipelined entry,
* or when we timeout and resend. */
_log.info("%s: follower has fallen behind (target prevLogIndex was %d, maxAllowedShippingLag " + "is %d), moving to CATCHUP mode", StatusAsString(), prevLogIndex, _maxAllowedShippingLag);
_mode = Mode.Catchup;
break;
}
}
}
_lastLeaderContext = leaderContext;
}
}
