public void SetCurrentTopology(Topology currentTopology, long index)
{
using (var tx = _env.NewTransaction(TransactionFlags.ReadWrite))
{
SetCurrentTopologyInternal(currentTopology, index, tx);
tx.Commit();
}
}
private static void SetCurrentTopologyInternal(Topology currentTopology, long index, Transaction tx)
{
if (currentTopology.TopologyId == Guid.Empty)
throw new InvalidOperationException("Cannot set topology with an empty TopologyId");
var metadata = tx.ReadTree(MetadataTreeName);
var current = metadata.Read("current-topology");
metadata.Add("previous-topology", current == null ? "{}" : current.Reader.ToStringValue());
metadata.Add("current-topology", JsonConvert.SerializeObject(currentTopology));
metadata.Add("current-topology-index", EndianBitConverter.Little.GetBytes(index));
}
public RaftEngine(RaftEngineOptions raftEngineOptions)
{
//#if DEBUG
// Console.WriteLine("Press any key to continue loading Raft -> opportunity to attach debugger");
// Console.ReadLine();
//#endif
_raftEngineOptions = raftEngineOptions;
Debug.Assert(raftEngineOptions.Stopwatch != null);
_log = LogManager.GetLogger(raftEngineOptions.Name + "." + GetType().FullName);
_eventLoopCancellationTokenSource = new CancellationTokenSource();
Name = raftEngineOptions.Name;
PersistentState = new PersistentState(raftEngineOptions.Name, raftEngineOptions.StorageOptions, _eventLoopCancellationTokenSource.Token)
{
CommandSerializer = new JsonCommandSerializer()
};
_currentTopology = PersistentState.GetCurrentTopology();
//warm up to make sure that the serializer don't take too long and force election timeout
PersistentState.CommandSerializer.Serialize(new NopCommand());
var thereAreOthersInTheCluster = CurrentTopology.QuorumSize > 1;
if (thereAreOthersInTheCluster == false && CurrentTopology.IsVoter(Name))
{
PersistentState.UpdateTermTo(this, PersistentState.CurrentTerm + 1);// restart means new term
SetState(RaftEngineState.Leader);
}
else
{
SetState(RaftEngineState.Follower);
}
_commitIndex = StateMachine.LastAppliedIndex;
_eventLoopTask = Task.Run(() => EventLoop());
}
private void PromoteNodeToVoter(AppendEntriesResponse resp)
{
// if we got a successful append entries response from a promotable node, and it has caught up
// with the committed entries, it means that we can promote it to voting positions, since it
// can now become a leader.
var upgradedNode = Engine.CurrentTopology.GetNodeByName(resp.From);
if (upgradedNode == null)
return;
var requestTopology = new Topology(
Engine.CurrentTopology.TopologyId,
Engine.CurrentTopology.AllVotingNodes.Union(new[] { upgradedNode }),
Engine.CurrentTopology.NonVotingNodes,
Engine.CurrentTopology.PromotableNodes.Where(x => x != upgradedNode)
);
if (Engine.CurrentlyChangingTopology() == false)
{
_log.Info(
"Node {0} is a promotable node, and it has caught up to the current cluster commit index, but we are currently updating the topology, will try again later",
resp.From);
return;
}
_log.Info(
"Node {0} is a promotable node, and it has caught up to the current cluster commit index, promoting to voting member",
resp.From);
Engine.ModifyTopology(requestTopology);
}
public static void SetTopologyExplicitly(RaftEngineOptions options, Topology topology, bool throwIfTopologyExists)
{
using (var ps = new PersistentState("ClusterBootstrap", options.StorageOptions, CancellationToken.None))
{
if (ps.GetCurrentTopology().HasVoters && throwIfTopologyExists)
throw new InvalidOperationException("Cannot set topology on a cluster that already have a topology");
ps.SetCurrentTopology(topology, 0);
}
}
public void AllPeers_and_AllVotingPeers_can_be_persistantly_saved_and_loaded()
{
var cancellationTokenSource = new CancellationTokenSource();
var path = "test" + Guid.NewGuid();
try
{
var expectedAllVotingPeers = new List<string> { "Node123", "Node1", "Node2", "NodeG", "NodeB", "NodeABC" };
using (var options = StorageEnvironmentOptions.ForPath(path))
{
using (var persistentState = new PersistentState("self",options, cancellationTokenSource.Token)
{
CommandSerializer = new JsonCommandSerializer()
})
{
var currentConfiguration = persistentState.GetCurrentTopology();
Assert.Empty(currentConfiguration.AllVotingNodes);
var currentTopology = new Topology(new Guid("355a589b-cadc-463d-a515-5add2ea47205"),
expectedAllVotingPeers.Select(x => new NodeConnectionInfo { Name = x }), Enumerable.Empty<NodeConnectionInfo>(), Enumerable.Empty<NodeConnectionInfo>());
persistentState.SetCurrentTopology(currentTopology, 1);
}
}
using (var options = StorageEnvironmentOptions.ForPath(path))
{
using (var persistentState = new PersistentState("self", options, cancellationTokenSource.Token)
{
CommandSerializer = new JsonCommandSerializer()
})
{
var currentConfiguration = persistentState.GetCurrentTopology();
Assert.Equal(expectedAllVotingPeers.Count, currentConfiguration.AllVotingNodes.Count());
foreach (var nodeConnectionInfo in currentConfiguration.AllVotingNodes)
{
Assert.True(expectedAllVotingPeers.Contains(nodeConnectionInfo.Name));
}
}
}
}
finally
{
new DirectoryInfo(path).Delete(true);
}
}
internal void RevertTopologyTo(Topology previous)
{
_log.Info("Reverting topology because the topology change command was reverted");
Interlocked.Exchange(ref _changingTopology, null);
Interlocked.Exchange(ref _currentTopology, previous);
OnTopologyChanged(new TopologyChangeCommand
{
Requested = previous
});
}
internal Task ModifyTopology(Topology requested)
{
if (State != RaftEngineState.Leader)
throw new InvalidOperationException("Cannot modify topology from a non leader node, current leader is: " +
(CurrentLeader ?? "no leader"));
var tcc = new TopologyChangeCommand
{
Completion = new TaskCompletionSource<object>(),
Requested = requested,
Previous = _currentTopology,
BufferCommand = false,
};
if (Interlocked.CompareExchange(ref _changingTopology, tcc.Completion.Task, null) != null)
throw new InvalidOperationException("Cannot change the cluster topology while another topology change is in progress");
try
{
_log.Debug("Topology change started on leader");
StartTopologyChange(tcc);
AppendCommand(tcc);
return tcc.Completion.Task;
}
catch (Exception)
{
Interlocked.Exchange(ref _changingTopology, null);
throw;
}
}
public Task AddToClusterAsync(NodeConnectionInfo node, bool nonVoting = false)
{
if (_currentTopology.Contains(node.Name))
throw new InvalidOperationException("Node " + node.Name + " is already in the cluster");
var requestedTopology = new Topology(
_currentTopology.TopologyId,
_currentTopology.AllVotingNodes,
nonVoting ? _currentTopology.NonVotingNodes.Union(new[] { node }) : _currentTopology.NonVotingNodes,
nonVoting ? _currentTopology.PromotableNodes : _currentTopology.PromotableNodes.Union(new[] { node })
);
if (_log.IsInfoEnabled)
{
_log.Info("AddToClusterClusterAsync, requestedTopology: {0}", requestedTopology);
}
return ModifyTopology(requestedTopology);
}
public Task RemoveFromClusterAsync(NodeConnectionInfo node)
{
if (_currentTopology.Contains(node.Name) == false)
throw new InvalidOperationException("Node " + node + " was not found in the cluster");
if (string.Equals(node.Name, Name, StringComparison.OrdinalIgnoreCase))
throw new InvalidOperationException("You cannot remove the current node from the cluster, step down this node and then remove it from the new leader");
var requestedTopology = new Topology(
_currentTopology.TopologyId,
_currentTopology.AllVotingNodes.Where(x => string.Equals(x.Name, node.Name, StringComparison.OrdinalIgnoreCase) == false),
_currentTopology.NonVotingNodes.Where(x => string.Equals(x.Name, node.Name, StringComparison.OrdinalIgnoreCase) == false),
_currentTopology.PromotableNodes.Where(x => string.Equals(x.Name, node.Name, StringComparison.OrdinalIgnoreCase) == false)
);
if (_log.IsInfoEnabled)
{
_log.Info("RemoveFromClusterAsync, requestedTopology: {0}", requestedTopology);
}
return ModifyTopology(requestedTopology);
}