public virtual void OnMismatch(long lastRemoteAppendIndex, LeaderContext leaderContext) { lock (this) { switch (_mode) { case Org.Neo4j.causalclustering.core.consensus.shipping.RaftLogShipper.Mode.Mismatch: long logIndex = max(min(_lastSentIndex - 1, lastRemoteAppendIndex), MIN_INDEX); SendEmpty(logIndex, leaderContext); break; case Org.Neo4j.causalclustering.core.consensus.shipping.RaftLogShipper.Mode.Pipeline: case Org.Neo4j.causalclustering.core.consensus.shipping.RaftLogShipper.Mode.Catchup: _log.info("%s: mismatch in mode %s from follower %s, moving to MISMATCH mode", StatusAsString(), _mode, _follower); _mode = Mode.Mismatch; SendEmpty(_lastSentIndex, leaderContext); break; default: throw new System.InvalidOperationException("Unknown mode: " + _mode); } _lastLeaderContext = leaderContext; } }
public void Can_JoinElectionByUseTheSameZKConnection() { IZooKeeper zookeeper = new ZooKeeper("127.0.0.1:2181", new TimeSpan(0, 60, 0), null); Server server1 = new Server() { ShardName = "shard1", Host = "127.0.0.1", Port = 6369, paramStr = "allowAdmin=true", Path = ZkPath.ShardsPath(ZkPath.CollectionName) }; Server server2 = new Server() { ShardName = "shard1", Host = "127.0.0.1", Port = 6379, paramStr = "allowAdmin=true", Path = ZkPath.ShardsPath(ZkPath.CollectionName) }; ServerInstances si = new ServerInstances(); si.Add(new ServerInst(server1.Host, server1.Port, "allowAdmin=true")); si.Add(new ServerInst(server2.Host, server2.Port, "allowAdmin=true")); LeaderContext context1 = new LeaderContext(zookeeper, server1); context1.JoinElection(); LeaderContext context2 = new LeaderContext(zookeeper, server2); context2.JoinElection(); }
public override void ApplyTo(RaftLogShipper raftLogShipper, LeaderContext leaderContext) { if (raftLogShipper.Identity().Equals(Target)) { raftLogShipper.OnMismatch(LastRemoteAppendIndex, leaderContext); } }
public override void ApplyTo(RaftLogShipper raftLogShipper, LeaderContext leaderContext) { if (raftLogShipper.Identity().Equals(Target)) { raftLogShipper.OnMatch(NewMatchIndex, leaderContext); } }
public virtual void OnMatch(long newMatchIndex, LeaderContext leaderContext) { lock (this) { bool progress = newMatchIndex > _matchIndex; if (newMatchIndex > _matchIndex) { _matchIndex = newMatchIndex; } else { _log.warn("%s: match index not progressing. This should be transient.", StatusAsString()); } switch (_mode) { case Org.Neo4j.causalclustering.core.consensus.shipping.RaftLogShipper.Mode.Mismatch: if (SendNextBatchAfterMatch(leaderContext)) { _log.info("%s: caught up after mismatch, moving to PIPELINE mode", StatusAsString()); _mode = PIPELINE; } else { _log.info("%s: starting catch up after mismatch, moving to CATCHUP mode", StatusAsString()); _mode = Mode.Catchup; } break; case Org.Neo4j.causalclustering.core.consensus.shipping.RaftLogShipper.Mode.Catchup: if (_matchIndex >= _lastSentIndex) { if (SendNextBatchAfterMatch(leaderContext)) { _log.info("%s: caught up, moving to PIPELINE mode", StatusAsString()); _mode = PIPELINE; } } break; case Org.Neo4j.causalclustering.core.consensus.shipping.RaftLogShipper.Mode.Pipeline: if (_matchIndex == _lastSentIndex) { AbortTimeout(); } else if (progress) { ScheduleTimeout(_retryTimeMillis); } break; default: throw new System.InvalidOperationException("Unknown mode: " + _mode); } _lastLeaderContext = leaderContext; } }
private void SendCommitUpdate(LeaderContext leaderContext) { /* * This is a commit update. That means that we just received enough success responses to an append * request to allow us to send a commit. By Raft invariants, this means that the term for the committed * entry is the current term. */ Org.Neo4j.causalclustering.core.consensus.RaftMessages_Heartbeat appendRequest = new Org.Neo4j.causalclustering.core.consensus.RaftMessages_Heartbeat(_leader, leaderContext.Term, leaderContext.CommitIndex, leaderContext.Term); _outbound.send(_follower, appendRequest); }
public virtual void OnCommitUpdate(LeaderContext leaderContext) { lock (this) { if (_mode == Mode.Pipeline) { SendCommitUpdate(leaderContext); } _lastLeaderContext = leaderContext; } }
private void EnsureLogShipperRunning(MemberId member, LeaderContext leaderContext) { RaftLogShipper logShipper = _logShippers[member]; if (logShipper == null && !member.Equals(_myself)) { logShipper = new RaftLogShipper(_outbound, _logProvider, _raftLog, _clock, _timerService, _myself, member, leaderContext.Term, leaderContext.CommitIndex, _retryTimeMillis, _catchupBatchSize, _maxAllowedShippingLag, _inFlightCache); _logShippers[member] = logShipper; logShipper.Start(); } }
public void UnRegistry(Guid id) { log.Info("unregistry on Zk,id=" + id); ServerInst inst = serverInstances.GetById(id); if (inst != null) { log.Info(inst); //多次对每个server初始化有没有问题? LeaderContext leaderContext = new LeaderContext(zookeeper, inst.ServerInfo); leaderContext.CancelElection(); } }
internal RaftLogShipper(Outbound <MemberId, Org.Neo4j.causalclustering.core.consensus.RaftMessages_RaftMessage> outbound, LogProvider logProvider, ReadableRaftLog raftLog, Clock clock, TimerService timerService, MemberId leader, MemberId follower, long leaderTerm, long leaderCommit, long retryTimeMillis, int catchupBatchSize, int maxAllowedShippingLag, InFlightCache inFlightCache) { this._outbound = outbound; this._timerService = timerService; this._catchupBatchSize = catchupBatchSize; this._maxAllowedShippingLag = maxAllowedShippingLag; this._log = logProvider.getLog(this.GetType()); this._raftLog = raftLog; this._clock = clock; this._follower = follower; this._leader = leader; this._retryTimeMillis = retryTimeMillis; this._lastLeaderContext = new LeaderContext(leaderTerm, leaderCommit); this._inFlightCache = inFlightCache; }
public void RegistryZk(Guid id) { log.Info("registry on Zk,id=" + id); ServerInst inst = serverInstances.GetById(id); if (inst != null) { log.Info(inst); ShardBuilder builder = new ShardBuilder(); Shard shard = builder.AddServer(inst.ServerInfo).Build(); //多次对每个server初始化有没有问题? LeaderContext leaderContext = new LeaderContext(zookeeper, inst.ServerInfo); leaderContext.JoinElection(); } }
public virtual void HandleCommands(IEnumerable <ShipCommand> shipCommands, LeaderContext leaderContext) { lock (this) { foreach (ShipCommand shipCommand in shipCommands) { foreach (RaftLogShipper logShipper in _logShippers.Values) { shipCommand.ApplyTo(logShipper, leaderContext); } } _lastLeaderContext = leaderContext; } }
/// <summary> /// Returns true if this sent the last batch. /// </summary> private bool SendNextBatchAfterMatch(LeaderContext leaderContext) { long lastIndex = _raftLog.appendIndex(); if (lastIndex > _matchIndex) { long endIndex = min(lastIndex, _matchIndex + _catchupBatchSize); ScheduleTimeout(_retryTimeMillis); SendRange(_matchIndex + 1, endIndex, leaderContext); return(endIndex == lastIndex); } else { return(true); } }
/// <summary> /// Resumed when becoming leader. /// </summary> public virtual void Resume(LeaderContext initialLeaderContext) { lock (this) { if (_stopped) { return; } _running = true; foreach (MemberId member in _membership.replicationMembers()) { EnsureLogShipperRunning(member, initialLeaderContext); } _lastLeaderContext = initialLeaderContext; } }
public override bool Equals(object o) { if (this == o) { return(true); } if (o == null || this.GetType() != o.GetType()) { return(false); } LeaderContext that = ( LeaderContext )o; if (Term != that.Term) { return(false); } return(CommitIndex == that.CommitIndex); }
private void SendRange(long startIndex, long endIndex, LeaderContext leaderContext) { if (startIndex > endIndex) { return; } _lastSentIndex = endIndex; try { int batchSize = ( int )(endIndex - startIndex + 1); RaftLogEntry[] entries = new RaftLogEntry[batchSize]; long prevLogIndex = startIndex - 1; long prevLogTerm = _raftLog.readEntryTerm(prevLogIndex); if (prevLogTerm > leaderContext.Term) { _log.warn("%s aborting send. Not leader anymore? %s, prevLogTerm=%d", StatusAsString(), leaderContext, prevLogTerm); return; } bool entryMissing = false; using (InFlightLogEntryReader logEntrySupplier = new InFlightLogEntryReader(_raftLog, _inFlightCache, false)) { for (int offset = 0; offset < batchSize; offset++) { entries[offset] = logEntrySupplier.Get(startIndex + offset); if (entries[offset] == null) { entryMissing = true; break; } if (entries[offset].Term() > leaderContext.Term) { _log.warn("%s aborting send. Not leader anymore? %s, entryTerm=%d", StatusAsString(), leaderContext, entries[offset].Term()); return; } } } if (entryMissing || DoesNotExistInLog(prevLogIndex, prevLogTerm)) { if (_raftLog.prevIndex() >= prevLogIndex) { SendLogCompactionInfo(leaderContext); } else { _log.error("%s: Could not send compaction info and entries were missing, but log is not behind.", StatusAsString()); } } else { Org.Neo4j.causalclustering.core.consensus.RaftMessages_AppendEntries_Request appendRequest = new Org.Neo4j.causalclustering.core.consensus.RaftMessages_AppendEntries_Request(_leader, leaderContext.Term, prevLogIndex, prevLogTerm, entries, leaderContext.CommitIndex); _outbound.send(_follower, appendRequest); } } catch (IOException e) { _log.warn(StatusAsString() + " exception during batch send", e); } }
private void SendLogCompactionInfo(LeaderContext leaderContext) { _log.warn("Sending log compaction info. Log pruned? Status=%s, LeaderContext=%s", StatusAsString(), leaderContext); _outbound.send(_follower, new Org.Neo4j.causalclustering.core.consensus.RaftMessages_LogCompactionInfo(_leader, leaderContext.Term, _raftLog.prevIndex())); }
private void SendNewEntries(long prevLogIndex, long prevLogTerm, RaftLogEntry[] newEntries, LeaderContext leaderContext) { ScheduleTimeout(_retryTimeMillis); _lastSentIndex = prevLogIndex + 1; Org.Neo4j.causalclustering.core.consensus.RaftMessages_AppendEntries_Request appendRequest = new Org.Neo4j.causalclustering.core.consensus.RaftMessages_AppendEntries_Request(_leader, leaderContext.Term, prevLogIndex, prevLogTerm, newEntries, leaderContext.CommitIndex); _outbound.send(_follower, appendRequest); }
public override void ApplyTo(RaftLogShipper raftLogShipper, LeaderContext leaderContext) { raftLogShipper.OnNewEntries(PrevLogIndex, PrevLogTerm, NewLogEntries, leaderContext); }
public override void ApplyTo(RaftLogShipper raftLogShipper, LeaderContext leaderContext) { raftLogShipper.OnCommitUpdate(leaderContext); }
public abstract void ApplyTo(RaftLogShipper raftLogShipper, LeaderContext leaderContext);
public virtual void OnNewEntries(long prevLogIndex, long prevLogTerm, RaftLogEntry[] newLogEntries, LeaderContext leaderContext) { lock (this) { if (_mode == Mode.Pipeline) { while (_lastSentIndex <= prevLogIndex) { if (prevLogIndex - _matchIndex <= _maxAllowedShippingLag) { // all sending functions update lastSentIndex SendNewEntries(prevLogIndex, prevLogTerm, newLogEntries, leaderContext); } else { /* The timer is still set at this point. Either we will send the next batch * as soon as the follower has caught up with the last pipelined entry, * or when we timeout and resend. */ _log.info("%s: follower has fallen behind (target prevLogIndex was %d, maxAllowedShippingLag " + "is %d), moving to CATCHUP mode", StatusAsString(), prevLogIndex, _maxAllowedShippingLag); _mode = Mode.Catchup; break; } } } _lastLeaderContext = leaderContext; } }