private int ProcessAppendEntriesResult(Server.ConnectorEx connector, Protocol p)
{
if (false == Raft.IsLeader)
{
return(Procedure.Success); // maybe close
}
var r = p as AppendEntries;
if (r.IsTimeout)
{
TrySendAppendEntries(connector, r); //resend
return(Procedure.Success);
}
lock (Raft)
{
if (Raft.LogSequence.TrySetTerm(r.Result.Term))
{
Raft.LeaderId = string.Empty; // 此时不知道谁是Leader。
// new term found.
Raft.ConvertStateTo(Raft.RaftState.Follower);
// 发现新的 Term，已经不是Leader，不能继续处理了。
// 直接返回。
connector.Pending = null;
return(Procedure.Success);
}
if (Raft.State != Raft.RaftState.Leader)
{
connector.Pending = null;
return(Procedure.Success);
}
}
if (r.Result.Success)
{
lock (Raft)
{
TryCommit(r, connector);
}
// TryCommit 推进了NextIndex，
// 可能日志没有复制完或者有新的AppendLog。
// 尝试继续复制日志。
// see TrySendAppendEntries 内的
// “限制一次发送的日志数量”
TrySendAppendEntries(connector, r);
return(Procedure.Success);
}
lock (Raft)
{
// TODO raft.pdf 提到一个优化
connector.NextIndex--;
TrySendAppendEntries(connector, r); //resend. use new NextIndex。
return(Procedure.Success);
}
}
public void StopRaft()
{
lock (this)
{
logger.Debug("Raft {0} Stop ...", RaftName);
Raft?.Server.Stop();
// 在同一个进程中，没法模拟进程退出，
// 此时RocksDb应该需要关闭，否则重启回失败吧。
Raft?.Close();
Raft = null;
}
}
public void StartRaft(bool resetLog = false)
{
lock (this)
{
if (null != Raft)
{
return;
}
logger.Debug("Raft {0} Start ...", RaftName);
StateMachine = new TestStateMachine();
var raftConfig = RaftConfig.Load(RaftConfigFileName);
raftConfig.AppendEntriesTimeout = 1000;
raftConfig.LeaderHeartbeatTimer = 1500;
raftConfig.LeaderLostTimeout = 2000;
raftConfig.DbHome = Path.Combine(".", RaftName.Replace(':', '_'));
if (resetLog)
{
if (Directory.Exists(raftConfig.DbHome))
{
Directory.Delete(raftConfig.DbHome, true);
}
}
Directory.CreateDirectory(raftConfig.DbHome);
Raft = new Raft(StateMachine, RaftName, raftConfig);
Raft.Server.AddFactoryHandle(
new AddCount().TypeId,
new Net.Service.ProtocolFactoryHandle()
{
Factory = () => new AddCount(),
Handle = ProcessAddCount,
});
Raft.Server.AddFactoryHandle(
new GetCount().TypeId,
new Net.Service.ProtocolFactoryHandle()
{
Factory = () => new GetCount(),
Handle = ProcessGetCount,
});
Raft.Server.Start();
}
}
private int ProcessRequest(long appInstance, Protocol p, ProtocolFactoryHandle factoryHandle)
{
return(Util.Task.Call(() =>
{
if (Raft.WaitLeaderReady())
{
if (Raft.LogSequence.LastAppliedAppRpcSessionId.TryGetValue(appInstance, out var exist))
{
if (p.UniqueRequestId <= exist)
{
p.SendResultCode(Procedure.DuplicateRequest);
return Procedure.DuplicateRequest;
}
}
return factoryHandle.Handle(p);
}
TrySendLeaderIs(p.Sender);
return Procedure.LogicError;
},
p,
(p, code) => p.SendResultCode(code)
));
}
public void AddCountAndWait(long requestId)
{
Raft.AppendLog(new AddCount(requestId));
}
private void InstallSnapshot(string path, Server.ConnectorEx connector)
{
// 整个安装成功结束时设置。中间Break(return)不设置。
// 后面 finally 里面使用这个标志
bool InstallSuccess = false;
logger.Debug("{0} InstallSnapshot Start... Path={1} ToConnector={2}",
Raft.Name, path, connector.Name);
try
{
var snapshotFile = new FileStream(path, FileMode.Open);
long offset = 0;
var buffer = new byte[32 * 1024];
var FirstLog = ReadLog(FirstIndex);
var trunkArg = new InstallSnapshotArgument();
trunkArg.Term = Term;
trunkArg.LeaderId = Raft.LeaderId;
trunkArg.LastIncludedIndex = FirstLog.Index;
trunkArg.LastIncludedTerm = FirstLog.Term;
while (!trunkArg.Done && Raft.IsLeader)
{
int rc = snapshotFile.Read(buffer);
trunkArg.Offset = offset;
trunkArg.Data = new Binary(buffer, 0, rc);
trunkArg.Done = rc < buffer.Length;
offset += rc;
if (trunkArg.Done)
{
trunkArg.LastIncludedLog = new Binary(FirstLog.Encode());
}
while (Raft.IsLeader)
{
connector.HandshakeDoneEvent.WaitOne();
var future = new TaskCompletionSource <int>();
var r = new InstallSnapshot()
{
Argument = trunkArg
};
if (!r.Send(connector.Socket,
(_) =>
{
future.SetResult(0);
return(Procedure.Success);
}))
{
continue;
}
future.Task.Wait();
if (r.IsTimeout)
{
continue;
}
lock (Raft)
{
if (this.TrySetTerm(r.Result.Term))
{
// new term found.
Raft.ConvertStateTo(Raft.RaftState.Follower);
return;
}
}
switch (r.ResultCode)
{
case global::Zeze.Raft.InstallSnapshot.ResultCodeNewOffset:
break;
default:
logger.Warn($"InstallSnapshot Break ResultCode={r.ResultCode}");
return;
}
if (r.Result.Offset >= 0)
{
if (r.Result.Offset > snapshotFile.Length)
{
logger.Error($"InstallSnapshot.Result.Offset Too Big.{r.Result.Offset}/{snapshotFile.Length}");
return; // 中断安装。
}
offset = r.Result.Offset;
snapshotFile.Seek(offset, SeekOrigin.Begin);
}
break;
}
}
InstallSuccess = Raft.IsLeader;
logger.Debug("{0} InstallSnapshot [SUCCESS] Path={1} ToConnector={2}",
Raft.Name, path, connector.Name);
}
finally
{
lock (Raft)
{
connector.InstallSnapshotting = false;
InstallSnapshotting.Remove(connector.Name);
if (InstallSuccess)
{
// 安装完成，重新初始化，使得以后的AppendEnties能继续工作。
// = FirstIndex + 1，防止Index跳着分配，使用ReadLogStart。
var next = ReadLogStart(FirstIndex + 1);
connector.NextIndex = next == null ? FirstIndex + 1 : next.Index;
}
}
}
}
public LogSequence(Raft raft)
{
Raft = raft;
var options = new DbOptions().SetCreateIfMissing(true);
Rafts = RocksDb.Open(options, Path.Combine(Raft.RaftConfig.DbHome, "rafts"));
{
// Read Term
var termKey = ByteBuffer.Allocate();
termKey.WriteInt(0);
RaftsTermKey = termKey.Copy();
var termValue = Rafts.Get(RaftsTermKey);
if (null != termValue)
{
var bb = ByteBuffer.Wrap(termValue);
Term = bb.ReadLong();
}
else
{
Term = 0;
}
// Read VoteFor
var voteForKey = ByteBuffer.Allocate();
voteForKey.WriteInt(1);
RaftsVoteForKey = voteForKey.Copy();
var voteForvalue = Rafts.Get(RaftsVoteForKey);
if (null != voteForvalue)
{
var bb = ByteBuffer.Wrap(voteForvalue);
VoteFor = bb.ReadString();
}
else
{
VoteFor = string.Empty;
}
}
Logs = RocksDb.Open(options, Path.Combine(Raft.RaftConfig.DbHome, "logs"));
{
// Read Last Log Index
using var itLast = Logs.NewIterator();
itLast.SeekToLast();
if (itLast.Valid())
{
LastIndex = RaftLog.Decode(
new Binary(itLast.Value()),
Raft.StateMachine.LogFactory
).Index;
}
else
{
// empty. add one for prev.
SaveLog(new RaftLog(Term, 0, new HeartbeatLog()));
LastIndex = 0;
}
using var itFirst = Logs.NewIterator();
itFirst.SeekToFirst();
FirstIndex = RaftLog.Decode(
new Binary(itFirst.Value()),
Raft.StateMachine.LogFactory
).Index;
// 【注意】snapshot 以后 FirstIndex 会推进，不再是从0开始。
LastApplied = FirstIndex;
CommitIndex = FirstIndex;
}
}
// 多个Raft实例才需要自定义配置名字，否则使用默认名字就可以了。
public Server(Raft raft, string name, Zeze.Config config) : base(name, config)
{
Raft = raft;
}