public async Task NodeDiscoveryLoop()
{
while (NodeStateService.Role == NodeState.Leader)
{
try
{
var urls = ClusterOptions.GetClusterUrls().Where(cl => !StateMachine.IsNodeContactable(cl));
if (urls.Count() > 0)
{
Logger.LogInformation("Starting discovery...");
var updates = await _discovery.SearchForNodes(urls, TimeSpan.FromMilliseconds(ClusterOptions.LatencyToleranceMs));
if (updates.Count() > 0)
{
Logger.LogInformation("Found a node");
AddNodesToCluster(updates);
var myNodeInfo = updates.Where(u => u.Id == _nodeStorage.Id);
if (myNodeInfo.Count() > 0)
{
NodeStateService.Url = myNodeInfo.First().TransportAddress;
}
}
await Task.Delay(100);
}
else
{
await Task.Delay(1000);
}
}
catch (Exception e)
{
Logger.LogError("Failed to run Cluster Info Timeout Handler with error " + e.StackTrace);
}
}
}
public async void StartElection()
{
try
{
var lastLogTerm = _nodeStorage.GetLastLogTerm();
if (_nodeStorage.CurrentTerm > lastLogTerm + 3)
{
Logger.LogInformation("Detected that the node is too far ahead of its last log (3), restarting election from the term of the last log term " + lastLogTerm);
_nodeStorage.SetCurrentTerm(lastLogTerm + 1);
}
else
{
_nodeStorage.SetCurrentTerm(_nodeStorage.CurrentTerm + 1);
}
//Vote for yourself
_nodeStorage.SetVotedFor(_nodeStorage.Id);
var election = new Election(_loggerFactory.CreateLogger <Election>(), TimeSpan.FromMilliseconds(ClusterOptions.LatencyToleranceMs), ClusterOptions.GetClusterUrls().Where(url => url != NodeStateService.Url));
var collectedNodes = await election.CollectVotes(
_nodeStorage.CurrentTerm,
_nodeStorage.Id,
_nodeStorage.GetLastLogIndex(),
_nodeStorage.GetLastLogTerm());
if (collectedNodes.Count() >= ClusterOptions.MinimumNodes - 1)
{
Logger.LogInformation(NodeStateService.GetNodeLogId() + "Recieved enough votes to be promoted, promoting to leader. Registered nodes: " + (collectedNodes.Count() + 1) + " collection nodes " + ClusterOptions.MinimumNodes);
StopTimer(_electionTimeoutTimer);
SetNodeRole(NodeState.Leader);
AddNodesToCluster(collectedNodes.Select(cn => new NodeInformation()
{
Id = cn.Key,
TransportAddress = cn.Value,
IsContactable = true,
Name = ""
}));
}
else
{
NodeStateService.CurrentLeader = null;
_nodeStorage.SetVotedFor(null);
SetNodeRole(NodeState.Follower);
}
}
catch (Exception e)
{
Logger.LogError("Failed to run election with error " + e.StackTrace);
}
}
public RaftService(
ILoggerFactory logger,
IOptions <ClusterOptions> clusterOptions,
IOptions <NodeOptions> nodeOptions,
IClusterConnectionPool clusterConnectionPool,
INodeStorage <State> nodeStorage,
IStateMachine <State> stateMachine,
NodeStateService nodeStateService,
ClusterClient clusterClient
) : base(logger.CreateLogger <RaftService <State> >(), clusterOptions.Value, nodeOptions.Value, stateMachine, nodeStateService)
{
_nodeStorage = nodeStorage;
_loggerFactory = logger;
//Bootstrap the node
_snapshotService = new Snapshotter <State>(logger.CreateLogger <Snapshotter <State> >(), nodeStorage, stateMachine, nodeStateService);
_bootstrapService = new Bootstrapper <State>(logger.CreateLogger <Bootstrapper <State> >(), clusterOptions.Value, nodeOptions.Value, nodeStorage, StateMachine, NodeStateService);
_commitService = new CommitService <State>(logger.CreateLogger <CommitService <State> >(), clusterOptions.Value, nodeOptions.Value, nodeStorage, StateMachine, NodeStateService);
_discovery = new Discovery(logger.CreateLogger <Discovery>());
_clusterClient = clusterClient;
_clusterConnectionPool = clusterConnectionPool;
NodeStateService.Id = _nodeStorage.Id;
_electionTimeoutTimer = new Timer(ElectionTimeoutEventHandler);
_heartbeatTimer = new Timer(HeartbeatTimeoutEventHandler);
if (!ClusterOptions.TestMode)
{
_bootstrapTask = Task.Run(async() =>
{
//Wait for the rest of the node to bootup
Logger.LogInformation("Starting bootstrap...");
Thread.Sleep(3000);
nodeStateService.Url = await _bootstrapService.GetMyUrl(ClusterOptions.GetClusterUrls(), TimeSpan.FromMilliseconds(ClusterOptions.LatencyToleranceMs));
NodeStateService.IsBootstrapped = true;
SetNodeRole(NodeState.Follower);
});
}
else
{
Logger.LogInformation("Running in test mode...");
SetNodeRole(NodeState.Leader);
NodeStateService.IsBootstrapped = true;
Handle(new ExecuteCommands()
{
Commands = new List <BaseCommand>()
{
{
new UpsertNodeInformation()
{
Id = NodeStateService.Id,
Name = "",
TransportAddress = "https://localhost:5021",
IsContactable = true
}
}
}
}).GetAwaiter().GetResult();
}
}