public async Task <bool> AddLogEntryAsync(byte[] data, string entityName = "default", int timeoutMs = 20000)
{
if (System.Threading.Interlocked.Read(ref disposed) == 1)
{
return(false);
}
RaftNode rn = null;
if (this.raftNodes.TryGetValue(entityName, out rn))
{
//Generating externalId
var msgId = AsyncResponseHandler.GetMessageId();
var msgIdStr = msgId.ToBytesString();
var resp = new ResponseCrate();
resp.TimeoutsMs = timeoutMs; //enable for amre
//resp.TimeoutsMs = Int32.MaxValue; //using timeout of the wait handle (not the timer), enable for mre
//resp.Init_MRE();
resp.Init_AMRE();
AsyncResponseHandler.df[msgIdStr] = resp;
var aler = rn.AddLogEntry(data, msgId);
switch (aler.AddResult)
{
case AddLogEntryResult.eAddLogEntryResult.LOG_ENTRY_IS_CACHED:
case AddLogEntryResult.eAddLogEntryResult.NODE_NOT_A_LEADER:
//async waiting
await resp.amre.WaitAsync(); //enable for amre
resp.Dispose_MRE();
if (AsyncResponseHandler.df.TryRemove(msgIdStr, out resp))
{
if (resp.IsRespOk)
{
return(true);
}
}
break;
default:
//case AddLogEntryResult.eAddLogEntryResult.ERROR_OCCURED:
//case AddLogEntryResult.eAddLogEntryResult.NO_LEADER_YET:
resp.Dispose_MRE();
AsyncResponseHandler.df.TryRemove(msgIdStr, out resp);
return(false);
}
}
//return new AddLogEntryResult { AddResult = AddLogEntryResult.eAddLogEntryResult.NODE_NOT_FOUND_BY_NAME };
//return new Tuple<bool, byte[]>(false, null);
return(false);
}
internal RaftNode GetNodeByEntityName(string entityName)
{
RaftNode rn = null;
raftNodes.TryGetValue(entityName, out rn);
return(rn);
}
// Creates and returns a configured array of raftnodes using the test cluster
static internal RaftNode[] ConfigureRaftCluster(int numberOfNodes, SM sm)
{
RaftNode[] nodes = new RaftNode[numberOfNodes];
// Create nodes
for (uint i = 0; i < numberOfNodes; i++)
{
if (sm == SM.Numeral)
{
nodes[i] = new RaftNode(i, new NumeralStateMachine());
}
else
{
nodes[i] = new RaftNode(i, new DictionaryStateMachine());
}
}
// Adding them to a cluster and configuring them
foreach (RaftNode node in nodes)
{
var c = new RaftCluster();
Array.ForEach(nodes, x => c.AddNode(new ObjectRaftConnector(x.NodeId, x)));
node.Configure(c);
}
return(nodes);
}
public ClusterService()
{
var numberOfNodes = 5;
var _nodes = new List <RaftNode>();
// Create nodes
for (uint i = 0; i < numberOfNodes; i++)
{
var node = new RaftNode(i, new NumeralStateMachine());
_nodes.Add(node);
}
// Adding them to a cluster and configuring them
foreach (RaftNode node in _nodes)
{
var c = new RaftCluster();
_nodes.ForEach(x => c.AddNode(new ObjectRaftConnector(x.NodeId, x)));
node.Configure(c);
}
this.cluster = _nodes[0].Cluster;
_nodes.ForEach(node => node.Run());
this.nodes = _nodes;
}
public void StartEmulateNodes(int nodesQuantity)
{
RaftNode rn = null;
RaftEntitySettings re_settings = new RaftEntitySettings()
{
};
for (int i = 0; i < nodesQuantity; i++)
{
rn = new RaftNode(re_settings, new DBreeze.DBreezeEngine(@"D:\Temp\RaftDBreeze\node" + (4250 + i)),
this,
this,
new DefaultHandler()
);
//rn.Verbose = true;
rn.SetNodesQuantityInTheCluster((uint)nodesQuantity);
rn.NodeAddress.NodeAddressId = i + 1;
lock (sync_nodes)
{
nodes.Add(rn.NodeAddress.NodeAddressId, rn);
}
System.Threading.Thread.Sleep((new Random()).Next(30, 150));
// System.Threading.Thread.Sleep(500);
rn.NodeStart();
}
}
static void Main(string[] args)
{
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Debug()
.WriteTo.ColoredConsole()
.CreateLogger();
var nodes = new[]
{
"localhost:13000",
"localhost:13001",
"localhost:13002"
};
var index = int.Parse(args[0]);
Console.WriteLine($"Listening on {nodes[index]}");
using (var communication = new TcpRaftCommunication(nodes[index]))
{
communication.Start();
var node = new RaftNode(communication, nodes, nodes[index])
{
MinEllectionTimeoutMs = 150,
MaxEllectionTimeoutMs = 300
};
node.Start();
Console.ReadLine();
}
}
public void ShouldBecomeFollowerIfAppendEntriesReceivedFromNewLeader()
{
var electionTimeoutInMilliseconds = 1000;
var initialTerm = 0;
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
var nodeId = Guid.NewGuid();
var numberOfOtherActors = 2;
IEnumerable <Guid> getOtherActors()
{
return(Enumerable.Range(0, numberOfOtherActors).Select(_ => Guid.NewGuid()));
}
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, getOtherActors, initialTerm,
electionTimeoutInMilliseconds);
void BlockUntilTrue(Func <bool> condition)
{
while (!condition())
{
}
}
// Start the node and let it timeout to trigger an election
_raftNode.Tell(new Initialize());
eventLog.BlockUntilAny(msg => msg is RoleStateChanged rsc && rsc.Term == initialTerm);
// Look for the change in state to a candidate node
Assert.True(eventLog.Count(msg => msg is RoleStateChanged rsc &&
rsc.NewState == RoleState.Candidate &&
rsc.Term == initialTerm) == 1);
// Send a single heartbeat with a higher term so that the node reverts to
// a follower state
var leaderId = Guid.NewGuid();
var requesterId = Guid.NewGuid();
_raftNode.Tell(new Request <AppendEntries>(requesterId,
new AppendEntries(42, leaderId, 0, 0, new object[0], 0)));
// Wait for a response
outbox.BlockUntilAny(msg =>
msg is Response <AppendEntries> response && response.ResponseMessage is AppendEntriesResult ae &&
ae.Term == 42);
var result = outbox.CastAs <Response <AppendEntries> >().First();
Assert.IsType <AppendEntriesResult>(result.ResponseMessage);
var responseMessage = (AppendEntriesResult)result.ResponseMessage;
Assert.True(responseMessage.Success);
// There should also be a state change that shows
// that the candidate node reverted back to a follower state
eventLog.ShouldHaveAtLeastOne(msg => msg is RoleStateChanged rsc && rsc.NewState == RoleState.Follower && rsc.Term > 42);
}
public void ShouldAcceptAppendEntriesOnFirstRequest()
{
var numberOfOtherActors = 3;
var otherActors = Enumerable.Range(0, numberOfOtherActors).Select(_ => Guid.NewGuid());
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
var startingTerm = 0;
var leaderId = Guid.NewGuid();
var nodeId = Guid.NewGuid();
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => otherActors, startingTerm);
_raftNode.Tell(new Initialize());
var term = 42;
var response = _raftNode.Request(nodeId,
() => new AppendEntries(term, leaderId, 0, 0, new object[] { "Hello, World" }, 1));
// The response must be valid
Assert.NotEqual(Response <AppendEntries> .Empty, response);
Assert.IsType <AppendEntriesResult>(response.ResponseMessage);
// The first entry must be successful since there are no prior entries
var firstResult = (AppendEntriesResult)response.ResponseMessage;
Assert.True(firstResult.Success);
Assert.Equal(term, firstResult.Term);
}
public void EmulationSetValue(byte[] data, string entityName = "default")
{
RaftNode rn = null;
if (this.raftNodes.TryGetValue(entityName, out rn))
{
rn.AddLogEntry(data);
}
}
/// <summary>
///
/// </summary>
/// <param name="entityName"></param>
public void Debug_PrintOutInMemory(string entityName = "default")
{
RaftNode rn = null;
if (this.raftNodes.TryGetValue(entityName, out rn))
{
rn.Debug_PrintOutInMemory();
}
}
public void Stop(int nodeId)
{
RaftNode node = null;
if (nodes.TryGetValue(nodeId, out node))
{
node.NodeStop();
}
}
// Creates a single-node cluster with a numeral state machine and returns the node
static internal RaftNode CreateNode()
{
RaftNode node = new RaftNode(1, new NumeralStateMachine());
var c = new RaftCluster();
c.AddNode(new ObjectRaftConnector(node.NodeId, node));
node.Configure(c);
return(node);
}
public bool NodeIsInLatestState(string entityName = "default")
{
RaftNode rn = null;
if (this.raftNodes.TryGetValue(entityName, out rn))
{
return(rn.NodeIsInLatestState);
}
return(false);
}
/// <summary>
/// Test method
/// </summary>
/// <param name="nodeId"></param>
/// <param name="stateLogId"></param>
/// <returns></returns>
public bool ContainsStateLogIdData(int nodeId, ulong stateLogId)
{
RaftNode node = null;
if (!nodes.TryGetValue(nodeId, out node))
{
return(false);
}
return(node.ContainsStateLogEntryId(stateLogId));
}
/// <summary>
/// Test method
/// </summary>
/// <param name="nodeId"></param>
/// <param name="data"></param>
public void SendData(int nodeId, string data)
{
RaftNode node = null;
if (!nodes.TryGetValue(nodeId, out node))
{
return;
}
node.AddLogEntry(System.Text.Encoding.UTF8.GetBytes(data), 0);
}
public RaftHost(INetworkModel networkModel, IRaftEventListener raftEventListener, IRaftNodeSettings raftSettings, string Id) :
base(networkModel)
{
this.Id = Id;
if (Id == null)
{
throw new ArgumentException("Id");
}
Raft = new RaftNode <string>(raftEventListener, raftSettings, Id);
Server = networkModel.CreateServer(Id, startListening: false);
}
internal List <TcpClusterEndPoint> clusterEndPoints = new List <TcpClusterEndPoint>(); //init clusterEndPoints creating 1-N connection
public TcpRaftNode(List <TcpClusterEndPoint> clusterEndPoints, List <RaftNodeSettings> raftNodes, string dbreezePath, Func <string, ulong, byte[], bool> OnCommit, int port = 4250, IWarningLog log = null)
{
//this.rn_settings = rn_settings ?? new RaftNodeSettings();
this.log = log;
this.port = port;
if (clusterEndPoints != null)
{
var bt = clusterEndPoints.SerializeBiser();
var decoder = new Biser.Decoder(bt);
this.clusterEndPoints = new List <TcpClusterEndPoint>();
decoder.GetCollection(() => { return(TcpClusterEndPoint.BiserDecode(extDecoder: decoder)); }, this.clusterEndPoints, false);
//this.clusterEndPoints.AddRange(clusterEndPoints.SerializeProtobuf().DeserializeProtobuf<List<TcpClusterEndPoint>>());
}
spider = new TcpSpider(this);
bool firstNode = true;
foreach (var rn_settings in raftNodes)
{
if (firstNode)
{
rn_settings.EntityName = "default";
firstNode = false;
}
if (String.IsNullOrEmpty(rn_settings.EntityName))
{
throw new Exception("Raft.Net: entities must have unique names. Change RaftNodeSettings.EntityName.");
}
if (this.raftNodes.ContainsKey(rn_settings.EntityName))
{
throw new Exception("Raft.Net: entities must have unique names. Change RaftNodeSettings.EntityName.");
}
var rn = new RaftNode(rn_settings ?? new RaftNodeSettings(), dbreezePath, this.spider, this.log, OnCommit);
#if DEBUG
rn.Verbose = rn_settings.VerboseRaft; //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DEBUG PURPOSES
#endif
rn.SetNodesQuantityInTheCluster((uint)this.clusterEndPoints.Count); //!!!!!!!!!!!! ENABLE 1 for debug, make it dynamic (but not less then 3 if not DEBUG)
rn.NodeAddress.NodeAddressId = port; //for debug/emulation purposes
rn.NodeAddress.NodeUId = Guid.NewGuid().ToByteArray().Substring(8, 8).To_Int64_BigEndian();
this.raftNodes[rn_settings.EntityName] = rn;
rn.NodeStart();
}
}
public AddLogEntryResult AddLogEntry(byte[] data, string entityName = "default")
{
RaftNode rn = null;
if (this.raftNodes.TryGetValue(entityName, out rn))
{
return(rn.AddLogEntry(data));
}
return(new AddLogEntryResult {
AddResult = AddLogEntryResult.eAddLogEntryResult.NODE_NOT_FOUND_BY_NAME
});
}
private static RaftNode StartNode(Configuration conf)
{
var id = conf.CurrentNode.Id;
var node = new RaftNode(id, new RaftCore.StateMachine.Implementations.NumeralStateMachine());
var clusterConfiguration = new RaftCluster();
foreach (var anotherNode in conf.Nodes)
{
clusterConfiguration.AddNode(new APIRaftConnector(anotherNode.Id, anotherNode.BaseUrl + "/RaftApi"));
}
node.Configure(clusterConfiguration);
node.Run();
return(node);
}
public void MustVoteForCandidateIfCandidateTermIsHigherThanCurrentTerm()
{
var currentTerm = 42;
_raftNode = new RaftNode(Guid.NewGuid(), delegate { }, delegate { }, () => new Guid[0], currentTerm);
var response = _raftNode.Request(Guid.NewGuid(), () => new RequestVote(43, Guid.NewGuid(), 0, 0));
Assert.NotNull(response);
Assert.IsType <RequestVoteResult>(response.ResponseMessage);
var result = (RequestVoteResult)response.ResponseMessage;
Assert.Equal(currentTerm, result.Term);
Assert.True(result.VoteGranted);
}
public void MustRejectVoteIfVoteRequestIsInvalid()
{
// Reply false if term < currentTerm (§5.1)
var currentTerm = 42;
_raftNode = new RaftNode(Guid.NewGuid(), delegate { }, delegate { }, () => new Guid[0], currentTerm);
var response = _raftNode.Request(Guid.NewGuid(), () => new RequestVote(0, Guid.NewGuid(), 0, 0));
Assert.NotNull(response);
Assert.IsType <RequestVoteResult>(response.ResponseMessage);
var result = (RequestVoteResult)response.ResponseMessage;
Assert.Equal(currentTerm, result.Term);
Assert.False(result.VoteGranted);
}
private static void StartKestrel(Configuration conf, RaftNode node)
{
var host = new WebHostBuilder()
.UseKestrel()
.UseContentRoot(Directory.GetCurrentDirectory())
.UseStartup <Startup>()
.ConfigureServices(services =>
{
services.Add(ServiceDescriptor.Singleton(typeof(RaftNode), node));
}
)
.UseUrls(conf.CurrentNode.BaseUrl)
.Build();
host.Run();
}
public void ShouldBeAbleToGetFollowerIdWheneverIdIsRequested()
{
var nodeId = Guid.NewGuid();
_raftNode = new RaftNode(nodeId, delegate { }, delegate { }, () => new Guid[0]);
var requesterId = Guid.NewGuid();
Func <object> createMessageToSend = () => new Request <GetId>(requesterId, new GetId());
Action <IEnumerable <object> > checkResults = outbox =>
{
var response = outbox.Cast <Response <GetId> >().First();
Assert.Equal(requesterId, response.RequesterId);
Assert.Equal(nodeId, response.ResponderId);
Assert.Equal(nodeId, response.ResponseMessage);
};
RunTest(createMessageToSend, checkResults);
}
public void ShouldSendRequestVoteToOtherActorsInTheClusterIfElectionTimerExpires()
{
var minMilliseconds = 150;
var maxMilliseconds = 300;
var nodeId = Guid.NewGuid();
var term = 42;
var numberOfActorsInCluster = 5;
var actorIds = Enumerable.Range(0, numberOfActorsInCluster)
.Select(_ => Guid.NewGuid()).ToArray();
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => actorIds, term);
// Set the request timeout to be from 150-300ms
var requesterId = Guid.NewGuid();
_raftNode.Request(requesterId, () => new SetElectionTimeoutRange(minMilliseconds, maxMilliseconds));
// Start the node
_raftNode.Tell(new Initialize());
// Let the timer expire
Thread.Sleep(1000);
var voteRequests = outbox.Where(msg => msg is Request <RequestVote> rv && rv.RequestMessage is RequestVote)
.Cast <Request <RequestVote> >().ToArray();
Assert.NotEmpty(voteRequests);
Assert.True(voteRequests.Count() == numberOfActorsInCluster);
for (var i = 0; i < numberOfActorsInCluster; i++)
{
var request = voteRequests[i];
var voteRequest = request.RequestMessage;
Assert.Equal(nodeId, voteRequest.CandidateId);
// Note: The new candidate must increment the current vote by one
Assert.Equal(term + 1, voteRequest.Term);
}
}
public void StartEmulateNodes(int nodesQuantity)
{
RaftNode rn = null;
RaftNodeSettings rn_settings = new RaftNodeSettings()
{
};
for (int i = 0; i < nodesQuantity; i++)
{
rn = new RaftNode(rn_settings, this, this);
rn.Verbose = true;
rn.SetNodesQuantityInTheCluster((uint)nodesQuantity);
rn.NodeAddress.NodeAddressId = i + 1;
nodes.Add(rn.NodeAddress.NodeAddressId, rn);
System.Threading.Thread.Sleep((new Random()).Next(30, 150));
// System.Threading.Thread.Sleep(500);
rn.NodeStart();
}
}
public void ShouldResetElectionTimerWhenVoteIsGrantedToCandidate()
{
// Route all the network output to the collection object
var nodeId = Guid.NewGuid();
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => new Guid[0]);
// The first response should be DateTime.Never
var requesterId = Guid.NewGuid();
var response = _raftNode.Request(requesterId, () => new GetLastUpdatedTimestamp());
Assert.NotNull(response);
Assert.IsType <DateTime>(response.ResponseMessage);
var lastUpdated = (DateTime)response.ResponseMessage;
Assert.True(lastUpdated.Equals(default(DateTime)));
Assert.True(lastUpdated.Equals(DateTime.MinValue));
var currentTime = DateTime.UtcNow;
var candidateId = Guid.NewGuid();
var requestVote = new Request <RequestVote>(requesterId, new RequestVote(42, candidateId, 0, 0));
_raftNode.Tell(requestVote);
var secondResponse = _raftNode.Request(requesterId, () => new GetLastUpdatedTimestamp());
Assert.NotNull(secondResponse);
Assert.IsType <DateTime>(secondResponse.ResponseMessage);
// The last updated timestamp should be relatively similar to the current time
var timestamp = (DateTime)secondResponse.ResponseMessage;
var timeDifference = timestamp - currentTime;
Assert.True(timeDifference.TotalMilliseconds >= 0 && timeDifference <= TimeSpan.FromSeconds(5));
}
public void ShouldReturnCurrentTermWheneverTermIsRequested()
{
var currentTerm = 42;
_raftNode = new RaftNode(Guid.NewGuid(), delegate { }, delegate { }, () => new Guid[0], currentTerm);
// Queue the request
var requesterId = Guid.NewGuid();
Func <object> createMessageToSend = () => new Request <GetCurrentTerm>(requesterId,
new GetCurrentTerm());
// Match the current term
void CheckResults(IEnumerable <object> results)
{
Assert.True(results.Count(msg => msg is Response <GetCurrentTerm>) == 1);
var response = results.Cast <Response <GetCurrentTerm> >().First();
Assert.Equal(currentTerm, response.ResponseMessage);
}
RunTest(createMessageToSend, CheckResults);
}
public void ShouldRejectAppendEntriesIfFollowerCannotFindAMatchForAnEntryInItsOwnLog()
{
/* When sending an AppendEntries RPC,
* the leader includes the term number and index of the entry
* that immediately precedes the new entry.
*
* If the follower cannot find a match for this entry in its own log,
* it rejects the request to append the new entry.
*/
var nodeId = Guid.NewGuid();
var numberOfOtherActors = 3;
var otherActors = Enumerable.Range(0, numberOfOtherActors).Select(_ => Guid.NewGuid());
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
var startingTerm = 0;
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => otherActors, startingTerm);
_raftNode.Tell(new Initialize());
Thread.Sleep(100);
var term = 42;
var leaderId = Guid.NewGuid();
var appendEntries = new AppendEntries(term, leaderId, 1, 41, new object[0], 0);
var requesterId = Guid.NewGuid();
var response = _raftNode.Request(requesterId, () => appendEntries);
Assert.Equal(requesterId, response.RequesterId);
Assert.Equal(nodeId, response.ResponderId);
// Reply false if log doesn’t contain an entry at prevLogIndex whose term matches prevLogTerm (§5.3)
Assert.IsType <AppendEntriesResult>(response.ResponseMessage);
var result = (AppendEntriesResult)response.ResponseMessage;
Assert.False(result.Success);
}
public void ShouldEmitChangeEventsWhenChangingRoles()
{
var nodeId = Guid.NewGuid();
var outbox = new ConcurrentBag <object>();
var eventLog = new ConcurrentBag <object>();
_raftNode = new RaftNode(nodeId, outbox.Add, eventLog.Add, () => new Guid[0]);
// Start the node and let it time out
_raftNode.Tell(new Initialize());
Thread.Sleep(500);
bool ShouldContainChangeEvent(object msg)
{
return(msg is RoleStateChanged rsc &&
rsc.ActorId == nodeId &&
rsc.OldState == RoleState.Follower &&
rsc.NewState == RoleState.Candidate);
}
Assert.NotEmpty(eventLog);
Assert.True(eventLog.Count(ShouldContainChangeEvent) > 0);
}
public void StartEmulateNodes(int nodesQuantity)
{
RaftNode rn = null;
RaftNodeSettings rn_settings = new RaftNodeSettings()
{
};
for (int i = 0; i < nodesQuantity; i++)
{
rn = new RaftNode(rn_settings, @"D:\Temp\RaftDBreeze\node" + (4250 + i), this, this, (entityName, index, data) => { return(true); });
rn.Verbose = true;
rn.SetNodesQuantityInTheCluster((uint)nodesQuantity);
rn.NodeAddress.NodeAddressId = i + 1;
lock (sync_nodes)
{
nodes.Add(rn.NodeAddress.NodeAddressId, rn);
}
System.Threading.Thread.Sleep((new Random()).Next(30, 150));
// System.Threading.Thread.Sleep(500);
rn.NodeStart();
}
}
