public void Follower_as_a_single_node_becomes_leader_automatically()
{
var hub = new InMemoryTransportHub();
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.OwnsPagers = false;
var raftEngineOptions = new RaftEngineOptions(
new NodeConnectionInfo { Name = "node1" },
storageEnvironmentOptions,
hub.CreateTransportFor("node1"),
new DictionaryStateMachine()
)
{
ElectionTimeout = 1000,
HeartbeatTimeout = 1000/6
};
PersistentState.ClusterBootstrap(raftEngineOptions);
storageEnvironmentOptions.OwnsPagers = true;
using (var raftNode = new RaftEngine(raftEngineOptions))
{
Assert.Equal(RaftEngineState.Leader, raftNode.State);
}
}
public override async Task <HttpResponseMessage> ExecuteAsync(HttpControllerContext controllerContext, CancellationToken cancellationToken)
{
RaftEngine = (RaftEngine)controllerContext.Configuration.Properties[typeof(RaftEngine)];
StateMachine = (KeyValueStateMachine)RaftEngine.StateMachine;
try
{
return(await base.ExecuteAsync(controllerContext, cancellationToken));
}
catch (NotLeadingException)
{
var currentLeader = RaftEngine.CurrentLeader;
if (currentLeader == null)
{
return(Request.CreateErrorResponse(HttpStatusCode.PreconditionFailed, "No current leader, try again later"));
}
var leaderNode = RaftEngine.CurrentTopology.GetNodeByName(currentLeader);
if (leaderNode == null)
{
return(Request.CreateErrorResponse(HttpStatusCode.PreconditionFailed, "Current leader " + currentLeader + " is not found in the topology. This should not happen."));
}
return(new HttpResponseMessage(HttpStatusCode.Redirect)
{
Headers =
{
Location = leaderNode.Uri
}
});
}
}
public void Follower_as_a_single_node_becomes_leader_automatically()
{
var hub = new InMemoryTransportHub();
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.OwnsPagers = false;
var raftEngineOptions = new RaftEngineOptions(
new NodeConnectionInfo {
Name = "node1"
},
storageEnvironmentOptions,
hub.CreateTransportFor("node1"),
new DictionaryStateMachine()
)
{
ElectionTimeout = 1000,
HeartbeatTimeout = 1000 / 6
};
PersistentState.ClusterBootstrap(raftEngineOptions);
storageEnvironmentOptions.OwnsPagers = true;
using (var raftNode = new RaftEngine(raftEngineOptions))
{
Assert.Equal(RaftEngineState.Leader, raftNode.State);
}
}
protected RaftEngine NewNodeFor(RaftEngine leader)
{
var raftEngine = new RaftEngine(CreateNodeOptions("node" + _nodes.Count, leader.Options.ElectionTimeout, StorageEnvironmentOptions.CreateMemoryOnly()));
_nodes.Add(raftEngine);
return(raftEngine);
}
protected ManualResetEventSlim WaitForSnapshot(RaftEngine node)
{
var cde = new ManualResetEventSlim();
node.CreatedSnapshot += cde.Set;
return(cde);
}
protected ManualResetEventSlim WaitForSnapshotInstallation(RaftEngine node)
{
var cde = new ManualResetEventSlim();
node.SnapshotInstalled += cde.Set;
return(cde);
}
protected AbstractRaftStateBehavior(RaftEngine engine)
{
Engine = engine;
LastHeartbeatTime = DateTime.UtcNow;
_log = LogManager.GetLogger(engine.Name + "." + GetType().Name);
_actionDispatch = new Dictionary <Type, Action <MessageContext> >
{
{ typeof(RequestVoteRequest), ctx => ctx.Reply(Handle((RequestVoteRequest)ctx.Message)) },
{ typeof(AppendEntriesRequest), ctx => ctx.Reply(Handle((AppendEntriesRequest)ctx.Message)) },
{ typeof(CanInstallSnapshotRequest), ctx => ctx.Reply(Handle((CanInstallSnapshotRequest)ctx.Message)) },
{ typeof(InstallSnapshotRequest), ctx =>
{
var reply = Handle(ctx, (InstallSnapshotRequest)ctx.Message, ctx.Stream);
if (reply != null)
{
ctx.Reply(reply);
}
else
{
ctx.AsyncResponse = true;
}
} },
{ typeof(RequestVoteResponse), ctx => Handle((RequestVoteResponse)ctx.Message) },
{ typeof(AppendEntriesResponse), ctx => Handle((AppendEntriesResponse)ctx.Message) },
{ typeof(CanInstallSnapshotResponse), ctx => Handle((CanInstallSnapshotResponse)ctx.Message) },
{ typeof(InstallSnapshotResponse), ctx => Handle((InstallSnapshotResponse)ctx.Message) },
{ typeof(NothingToDo), ctx => { } },
{ typeof(TimeoutNowRequest), ctx => Handle((TimeoutNowRequest)ctx.Message) },
{ typeof(DisconnectedFromCluster), ctx => Handle((DisconnectedFromCluster)ctx.Message) },
{ typeof(Action), ctx => ((Action)ctx.Message)() },
};
}
public HttpTransportPingTest()
{
_node1Transport = new HttpTransport("node1", _requestsTimeout, CancellationToken.None);
var node1 = new NodeConnectionInfo {
Name = "node1", Uri = new Uri("http://localhost:9079")
};
var engineOptions = new RaftEngineOptions(node1, StorageEnvironmentOptions.CreateMemoryOnly(), _node1Transport, new DictionaryStateMachine())
{
ElectionTimeout = 60 * 1000,
HeartbeatTimeout = 10 * 1000
};
_raftEngine = new RaftEngine(engineOptions);
NonAdminHttp.EnsureCanListenToWhenInNonAdminContext(9079);
_server = WebApp.Start(new StartOptions
{
Urls = { "http://+:9079/" }
}, builder =>
{
var httpConfiguration = new HttpConfiguration();
RaftWebApiConfig.Load();
httpConfiguration.MapHttpAttributeRoutes();
httpConfiguration.Properties[typeof(HttpTransportBus)] = _node1Transport.Bus;
builder.UseWebApi(httpConfiguration);
});
}
public RaftNode(int port)
{
_name = "node-" + port;
_url = string.Format("http://{0}:{1}", Environment.MachineName, port);
var nodeTransport = new HttpTransport(_name);
var node1 = new NodeConnectionInfo {
Name = _name, Uri = new Uri(_url)
};
var engineOptions = new RaftEngineOptions(
node1,
StorageEnvironmentOptions.CreateMemoryOnly(),
nodeTransport,
new DictionaryStateMachine());
engineOptions.ElectionTimeout *= 2;
engineOptions.HeartbeatTimeout *= 2;
_raftEngine = new RaftEngine(engineOptions);
_server = WebApp.Start(new StartOptions
{
Urls = { string.Format("http://+:{0}/", port) }
}, builder =>
{
var httpConfiguration = new HttpConfiguration();
RaftWebApiConfig.Load();
httpConfiguration.MapHttpAttributeRoutes();
httpConfiguration.Properties[typeof(HttpTransportBus)] = nodeTransport.Bus;
builder.UseWebApi(httpConfiguration);
});
}
protected AbstractRaftStateBehavior(RaftEngine engine)
{
Engine = engine;
LastHeartbeatTime = DateTime.UtcNow;
_log = LogManager.GetLogger(engine.Name + "." + GetType().Name);
_actionDispatch = new Dictionary<Type, Action<MessageContext>>
{
{typeof (RequestVoteRequest), ctx => ctx.Reply(Handle((RequestVoteRequest) ctx.Message))},
{typeof (AppendEntriesRequest), ctx => ctx.Reply(Handle((AppendEntriesRequest) ctx.Message))},
{typeof (CanInstallSnapshotRequest), ctx => ctx.Reply(Handle((CanInstallSnapshotRequest) ctx.Message))},
{typeof (InstallSnapshotRequest), ctx =>
{
var reply = Handle(ctx, (InstallSnapshotRequest) ctx.Message, ctx.Stream);
if (reply != null)
ctx.Reply(reply);
else
ctx.AsyncResponse = true;
}},
{typeof(RequestVoteResponse), ctx => Handle((RequestVoteResponse)ctx.Message)},
{typeof(AppendEntriesResponse), ctx => Handle((AppendEntriesResponse)ctx.Message)},
{typeof(CanInstallSnapshotResponse), ctx => Handle((CanInstallSnapshotResponse)ctx.Message)},
{typeof(InstallSnapshotResponse), ctx => Handle((InstallSnapshotResponse)ctx.Message)},
{typeof(NothingToDo), ctx => { }},
{typeof (TimeoutNowRequest), ctx => Handle((TimeoutNowRequest) ctx.Message)},
{typeof (DisconnectedFromCluster), ctx => Handle((DisconnectedFromCluster) ctx.Message)},
{typeof (Action), ctx => ((Action)ctx.Message)()},
};
}
public FollowerStateBehavior(RaftEngine engine, bool avoidLeadership) : base(engine)
{
_avoidLeadership = avoidLeadership;
_currentTermWhenWeBecameFollowers = engine.PersistentState.CurrentTerm + 1;// we are going to have a new term immediately.
var random = new Random(Engine.Name.GetHashCode() ^ (int)DateTime.Now.Ticks);
Timeout = random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout);
}
public FollowerStateBehavior(RaftEngine engine, bool avoidLeadership) : base(engine)
{
_avoidLeadership = avoidLeadership;
_currentTermWhenWeBecameFollowers = engine.PersistentState.CurrentTerm + 1; // we are going to have a new term immediately.
var random = new Random(Engine.Name.GetHashCode() ^ (int)DateTime.Now.Ticks);
Timeout = random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout);
}
public CandidateStateBehavior(RaftEngine engine, bool forcedElection)
: base(engine)
{
_forcedElection = forcedElection;
_wonTrialElection = forcedElection;
_random = new Random((int)(engine.Name.GetHashCode() + DateTime.UtcNow.Ticks));
Timeout = _random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout);
}
public async Task <HttpResponseMessage> Leave([FromUri] string name)
{
await RaftEngine.RemoveFromClusterAsync(new NodeConnectionInfo
{
Name = name
});
return(new HttpResponseMessage(HttpStatusCode.Accepted));
}
public CandidateStateBehavior(RaftEngine engine, bool forcedElection)
: base(engine)
{
_forcedElection = forcedElection;
_wonTrialElection = forcedElection;
_random = new Random((int)(engine.Name.GetHashCode() + DateTime.UtcNow.Ticks));
Timeout = _random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout);
StartElection();
}
protected ManualResetEventSlim WaitForToplogyChange(RaftEngine node)
{
var mre = new ManualResetEventSlim();
node.TopologyChanged += state =>
{
if (node.CurrentTopology.HasVoters)
mre.Set();
};
return mre;
}
protected ManualResetEventSlim WaitForStateChange(RaftEngine node, RaftEngineState requestedState)
{
var mre = new ManualResetEventSlim();
node.StateChanged += state =>
{
if (state == requestedState)
mre.Set();
};
return mre;
}
public ClusterManager(RaftEngine engine, DatabasesLandlord databasesLandlord)
{
Engine = engine;
Client = new ClusterManagementHttpClient(engine);
DatabasesLandlord = databasesLandlord;
engine.StateChanged += OnRaftEngineStateChanged;
engine.ProposingCandidacy += OnProposingCandidacy;
maxReplicationLatency = DatabasesLandlord.SystemDatabase.Configuration.Cluster.MaxReplicationLatency;
clusterManagerStartTime = DateTime.UtcNow;
}
protected void RestartAllNodes()
{
foreach (var raftEngine in _nodes)
{
raftEngine.Options.StorageOptions.OwnsPagers = false;
raftEngine.Dispose();
}
for (int i = 0; i < _nodes.Count; i++)
{
_nodes[i] = new RaftEngine(_nodes[i].Options);
}
}
public async Task <HttpResponseMessage> Read([FromUri] string key, [FromUri] string mode = null)
{
switch (mode)
{
case "quorum":
var taskCompletionSource = new TaskCompletionSource <object>();
try
{
RaftEngine.AppendCommand(new GetCommand
{
Key = key,
Completion = taskCompletionSource
});
}
catch (NotLeadingException e)
{
return(RedirectToLeader(e.CurrentLeader, Request.RequestUri));
}
var consistentRead = await taskCompletionSource.Task;
return(Request.CreateResponse(HttpStatusCode.OK, new
{
RaftEngine.State,
Key = key,
Value = consistentRead,
Missing = consistentRead == null
}));
case "leader":
if (RaftEngine.State != RaftEngineState.Leader)
{
return(RedirectToLeader(RaftEngine.CurrentLeader, Request.RequestUri));
}
goto case null;
case "any":
case null:
var read = StateMachine.Read(key);
return(Request.CreateResponse(HttpStatusCode.OK, new
{
RaftEngine.State,
Key = key,
Value = read,
Missing = read == null
}));
default:
return(Request.CreateResponse(HttpStatusCode.BadRequest, new
{
Error = "Unknown read mode"
}));
}
}
protected ManualResetEventSlim WaitForNodeToBecomeVoter(RaftEngine node)
{
var mre = new ManualResetEventSlim();
node.TopologyChanged += state =>
{
if (node.CurrentTopology.AllVotingNodes.Select(ni => ni.Name).Contains(node.Name))
{
mre.Set();
}
};
return(mre);
}
protected ManualResetEventSlim WaitForStateChange(RaftEngine node, RaftEngineState requestedState)
{
var mre = new ManualResetEventSlim();
node.StateChanged += state =>
{
if (state == requestedState)
{
mre.Set();
}
};
return(mre);
}
protected ManualResetEventSlim WaitForToplogyChange(RaftEngine node)
{
var mre = new ManualResetEventSlim();
node.TopologyChanged += state =>
{
if (node.CurrentTopology.HasVoters)
{
mre.Set();
}
};
return(mre);
}
public static ClusterManager Create(DocumentDatabase systemDatabase, DatabasesLandlord databasesLandlord)
{
if (systemDatabase == null)
{
throw new ArgumentNullException("systemDatabase");
}
if (databasesLandlord == null)
{
throw new ArgumentNullException("databasesLandlord");
}
DatabaseHelper.AssertSystemDatabase(systemDatabase);
var configuration = systemDatabase.Configuration;
var nodeConnectionInfo = CreateSelfConnection(systemDatabase);
StorageEnvironmentOptions options;
if (configuration.Core.RunInMemory == false)
{
var directoryPath = Path.Combine(configuration.Core.DataDirectory ?? AppDomain.CurrentDomain.BaseDirectory, "Raft");
if (Directory.Exists(directoryPath) == false)
{
Directory.CreateDirectory(directoryPath);
}
options = StorageEnvironmentOptions.ForPath(directoryPath);
}
else
{
options = StorageEnvironmentOptions.CreateMemoryOnly(configuration.Storage.TempPath);
}
var transport = new HttpTransport(nodeConnectionInfo.Name, systemDatabase.WorkContext.CancellationToken);
var stateMachine = new ClusterStateMachine(systemDatabase, databasesLandlord);
var raftEngineOptions = new RaftEngineOptions(nodeConnectionInfo, options, transport, stateMachine)
{
ElectionTimeout = (int)configuration.Cluster.ElectionTimeout.AsTimeSpan.TotalMilliseconds,
HeartbeatTimeout = (int)configuration.Cluster.HeartbeatTimeout.AsTimeSpan.TotalMilliseconds,
MaxLogLengthBeforeCompaction = configuration.Cluster.MaxLogLengthBeforeCompaction,
MaxEntriesPerRequest = configuration.Cluster.MaxEntriesPerRequest,
MaxStepDownDrainTime = configuration.Cluster.MaxStepDownDrainTime.AsTimeSpan
};
var raftEngine = new RaftEngine(raftEngineOptions);
stateMachine.RaftEngine = raftEngine;
return(new ClusterManager(raftEngine));
}
public async Task <HttpResponseMessage> Join([FromUri] string url, [FromUri] string name)
{
var uri = new Uri(url);
name = name ?? uri.Host + (uri.IsDefaultPort ? "" : ":" + uri.Port);
await RaftEngine.AddToClusterAsync(new NodeConnectionInfo
{
Name = name,
Uri = uri
});
return(new HttpResponseMessage(HttpStatusCode.Accepted));
}
public SteppingDownStateBehavior(RaftEngine engine)
: base(engine)
{
_stepdownDuration = Stopwatch.StartNew();
// we are sending this to ourselves because we want to make
// sure that we immediately check if we can step down
Engine.Transport.SendToSelf(new AppendEntriesResponse
{
CurrentTerm = Engine.PersistentState.CurrentTerm,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
LeaderId = Engine.Name,
Message = "Forcing step down evaluation",
Success = true
});
}
public SteppingDownStateBehavior(RaftEngine engine)
: base(engine)
{
_stepdownDuration = Stopwatch.StartNew();
// we are sending this to ourselves because we want to make
// sure that we immediately check if we can step down
Engine.Transport.SendToSelf(new AppendEntriesResponse
{
CurrentTerm = Engine.PersistentState.CurrentTerm,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
LeaderId = Engine.Name,
Message = "Forcing step down evaluation",
Success = true
});
}
public LeaderStateBehavior(RaftEngine engine)
: base(engine)
{
Timeout = engine.Options.ElectionTimeout / 2;
engine.TopologyChanged += OnTopologyChanged;
var lastLogEntry = Engine.PersistentState.LastLogEntry();
foreach (var peer in Engine.CurrentTopology.AllNodeNames)
{
_nextIndexes[peer] = lastLogEntry.Index + 1;
_matchIndexes[peer] = 0;
}
AppendCommand(new NopCommand());
_stopHeartbeatCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(Engine.CancellationToken, _disposedCancellationTokenSource.Token);
_heartbeatTask = Task.Run(() => Heartbeat(), _stopHeartbeatCancellationTokenSource.Token);
}
public LeaderStateBehavior(RaftEngine engine)
: base(engine)
{
Timeout = engine.Options.ElectionTimeout / 2;
engine.TopologyChanged += OnTopologyChanged;
var lastLogEntry = Engine.PersistentState.LastLogEntry();
foreach (var peer in Engine.CurrentTopology.AllNodeNames)
{
_nextIndexes[peer] = lastLogEntry.Index + 1;
_matchIndexes[peer] = 0;
}
AppendCommand(new NopCommand());
_stopHeartbeatCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(Engine.CancellationToken, _disposedCancellationTokenSource.Token);
_heartbeatTask = Task.Factory.StartNew(Heartbeat, _stopHeartbeatCancellationTokenSource.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
}
protected ManualResetEventSlim WaitForCommit(RaftEngine node, Func<DictionaryStateMachine, bool> predicate)
{
var cde = new ManualResetEventSlim();
node.CommitApplied += command =>
{
if (predicate((DictionaryStateMachine)node.StateMachine))
cde.Set();
};
node.SnapshotInstalled += () =>
{
var state = (DictionaryStateMachine)node.StateMachine;
if (predicate(state))
{
cde.Set();
}
};
return cde;
}
private async Task <HttpResponseMessage> Batch(KeyValueOperation[] operations)
{
var taskCompletionSource = new TaskCompletionSource <object>();
try
{
RaftEngine.AppendCommand(new OperationBatchCommand
{
Batch = operations,
Completion = taskCompletionSource
});
}
catch (NotLeadingException e)
{
return(RedirectToLeader(e.CurrentLeader, Request.RequestUri));
}
await taskCompletionSource.Task;
return(Request.CreateResponse(HttpStatusCode.Accepted));
}
public void AppendToLog(RaftEngine engine, IEnumerable <LogEntry> entries, long removeAllAfter)
{
using (var tx = _env.NewTransaction(TransactionFlags.ReadWrite))
{
var logs = tx.ReadTree(LogsTreeName);
var terms = tx.ReadTree(EntryTermsTreeName);
var metadata = tx.ReadTree(MetadataTreeName);
var changed = ReadIsTopologyChanged(metadata);
if (changed > removeAllAfter)
{
// need to reset the topology
var current = metadata.Read("current-topology");
var prevTopologyStr = current == null ? "{}" : current.Reader.ToStringValue();
metadata.Add("current-topology", prevTopologyStr);
metadata.Add("current-topology-index", EndianBitConverter.Little.GetBytes(0L));
var prevTopology = JsonConvert.DeserializeObject <Topology>(prevTopologyStr);
engine.RevertTopologyTo(prevTopology);
}
using (var it = logs.Iterate())
{
if (it.Seek(new Slice(EndianBitConverter.Big.GetBytes(removeAllAfter))) &&
it.MoveNext())
{
do
{
terms.Delete(it.CurrentKey);
} while (it.DeleteCurrentAndMoveNext());
}
}
foreach (var entry in entries)
{
var key = new Slice(EndianBitConverter.Big.GetBytes(entry.Index));
logs.Add(key, entry.Data);
terms.Add(key, new Slice(BitConverter.GetBytes(entry.Term)));
}
tx.Commit();
}
}
protected ManualResetEventSlim WaitForCommit(RaftEngine node, Func <DictionaryStateMachine, bool> predicate)
{
var cde = new ManualResetEventSlim();
node.CommitApplied += command =>
{
if (predicate((DictionaryStateMachine)node.StateMachine))
{
cde.Set();
}
};
node.SnapshotInstalled += () =>
{
var state = (DictionaryStateMachine)node.StateMachine;
if (predicate(state))
{
cde.Set();
}
};
return(cde);
}
public static NodeConnectionInfo GetLeaderNode(this RaftEngine engine, int waitTimeoutInSeconds = 0)
{
if (waitTimeoutInSeconds > 0)
{
engine.WaitForLeader(waitTimeoutInSeconds * 1000);
}
var leader = engine.CurrentLeader;
if (leader == null)
{
if (waitTimeoutInSeconds > 0)
{
throw new InvalidOperationException($"No leader. Waited {waitTimeoutInSeconds} seconds.");
}
return(null);
}
return(engine.CurrentTopology.AllNodes.Single(x => x.Name == leader));
}
public void UpdateTermTo(RaftEngine engine, long term)
{
using (var tx = _env.NewTransaction(TransactionFlags.ReadWrite))
{
var metadata = tx.ReadTree(MetadataTreeName);
metadata.Add("current-term", EndianBitConverter.Little.GetBytes(term));
metadata.Add("voted-for", new byte[0]); // clearing who we voted for
metadata.Add("voted-for-term", EndianBitConverter.Little.GetBytes(-1L)); // clearing who we voted for
VotedFor = null;
VotedForTerm = -1;
CurrentTerm = term;
if (engine != null)
{
engine.OnNewTerm(term);
}
tx.Commit();
}
}
public void Follower_on_timeout_should_become_candidate()
{
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.OwnsPagers = false;
var nodeOptions = new RaftEngineOptions(new NodeConnectionInfo {
Name = "real"
}, storageEnvironmentOptions, _inMemoryTransportHub.CreateTransportFor("real"), new DictionaryStateMachine());
PersistentState.SetTopologyExplicitly(nodeOptions,
new Topology(
new Guid("355a589b-cadc-463d-a515-5add2ea47205"),
new[]
{
new NodeConnectionInfo {
Name = "real"
}, new NodeConnectionInfo {
Name = "u2"
}, new NodeConnectionInfo {
Name = "pj"
},
}, new NodeConnectionInfo[0], new NodeConnectionInfo[0]), throwIfTopologyExists: true);
storageEnvironmentOptions.OwnsPagers = true;
using (var node = new RaftEngine(nodeOptions))
{
var timeoutEvent = new ManualResetEventSlim();
node.StateTimeout += timeoutEvent.Set;
ForceTimeout("real");
timeoutEvent.Wait();
Assert.Equal(RaftEngineState.Candidate, node.State);
}
}
public HttpTransportPingTest()
{
_node1Transport = new HttpTransport("node1");
var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") };
var engineOptions = new RaftEngineOptions(node1, StorageEnvironmentOptions.CreateMemoryOnly(), _node1Transport, new DictionaryStateMachine())
{
ElectionTimeout = 60 * 1000,
HeartbeatTimeout = 10 * 1000
};
_raftEngine = new RaftEngine(engineOptions);
_server = WebApp.Start(new StartOptions
{
Urls = { "http://+:9079/" }
}, builder =>
{
var httpConfiguration = new HttpConfiguration();
RaftWebApiConfig.Load();
httpConfiguration.MapHttpAttributeRoutes();
httpConfiguration.Properties[typeof(HttpTransportBus)] = _node1Transport.Bus;
builder.UseWebApi(httpConfiguration);
});
}
public void Follower_on_timeout_should_become_candidate()
{
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.OwnsPagers = false;
var nodeOptions = new RaftEngineOptions(new NodeConnectionInfo { Name = "real" }, storageEnvironmentOptions, _inMemoryTransportHub.CreateTransportFor("real"), new DictionaryStateMachine());
PersistentState.SetTopologyExplicitly(nodeOptions,
new Topology(
new Guid("355a589b-cadc-463d-a515-5add2ea47205"),
new[]
{
new NodeConnectionInfo {Name = "real"}, new NodeConnectionInfo {Name = "u2"}, new NodeConnectionInfo {Name = "pj"},
}, new NodeConnectionInfo[0], new NodeConnectionInfo[0]), throwIfTopologyExists: true);
storageEnvironmentOptions.OwnsPagers = true;
using (var node = new RaftEngine(nodeOptions))
{
var timeoutEvent = new ManualResetEventSlim();
node.StateTimeout += timeoutEvent.Set;
ForceTimeout("real");
timeoutEvent.Wait();
Assert.Equal(RaftEngineState.Candidate, node.State);
}
}
protected ManualResetEventSlim WaitForSnapshot(RaftEngine node)
{
var cde = new ManualResetEventSlim();
node.CreatedSnapshot += cde.Set;
return cde;
}
protected ManualResetEventSlim WaitForSnapshotInstallation(RaftEngine node)
{
var cde = new ManualResetEventSlim();
node.SnapshotInstalled += cde.Set;
return cde;
}
public void UpdateTermTo(RaftEngine engine, long term)
{
using (var tx = _env.NewTransaction(TransactionFlags.ReadWrite))
{
var metadata = tx.ReadTree(MetadataTreeName);
metadata.Add("current-term", EndianBitConverter.Little.GetBytes(term));
metadata.Add("voted-for", new byte[0]); // clearing who we voted for
metadata.Add("voted-for-term", EndianBitConverter.Little.GetBytes(-1L)); // clearing who we voted for
VotedFor = null;
VotedForTerm = -1;
CurrentTerm = term;
if (engine != null)
engine.OnNewTerm(term);
tx.Commit();
}
}
public ClusterManager(RaftEngine engine)
{
Engine = engine;
Client = new ClusterManagementHttpClient(engine);
}
protected RaftEngine CreateNetworkAndGetLeader(int nodeCount, int messageTimeout = -1)
{
var leaderIndex = new Random().Next(0, nodeCount);
if (messageTimeout == -1)
messageTimeout = Debugger.IsAttached ? 3 * 1000 : 500;
var nodeNames = new string[nodeCount];
for (int i = 0; i < nodeCount; i++)
{
nodeNames[i] = "node" + i;
}
WriteLine("{0} selected as seed", nodeNames[leaderIndex]);
var allNodesFinishedJoining = new ManualResetEventSlim();
for (int index = 0; index < nodeNames.Length; index++)
{
var nodeName = nodeNames[index];
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.OwnsPagers = false;
var options = CreateNodeOptions(nodeName, messageTimeout, storageEnvironmentOptions, nodeNames);
if (leaderIndex == index)
{
PersistentState.ClusterBootstrap(options);
}
storageEnvironmentOptions.OwnsPagers = true;
var engine = new RaftEngine(options);
_nodes.Add(engine);
if (leaderIndex == index)
{
engine.TopologyChanged += command =>
{
if (command.Requested.AllNodeNames.All(command.Requested.IsVoter))
{
allNodesFinishedJoining.Set();
}
};
for (int i = 0; i < nodeNames.Length; i++)
{
if (i == leaderIndex)
continue;
Assert.True(engine.AddToClusterAsync(new NodeConnectionInfo { Name = nodeNames[i] }).Wait(3000));
}
}
}
if (nodeCount == 1)
allNodesFinishedJoining.Set();
Assert.True(allNodesFinishedJoining.Wait(5000 * nodeCount));
var raftEngine = _nodes[leaderIndex];
var transport = (InMemoryTransportHub.InMemoryTransport)_inMemoryTransportHub.CreateTransportFor(raftEngine.Name);
transport.ForceTimeout();
Assert.True(_nodes[leaderIndex].WaitForLeader());
var leader = _nodes.FirstOrDefault(x => x.State == RaftEngineState.Leader);
Assert.NotNull(leader);
return _nodes[leaderIndex];
}
protected RaftEngine NewNodeFor(RaftEngine leader)
{
var raftEngine = new RaftEngine(CreateNodeOptions("node" + _nodes.Count, leader.Options.ElectionTimeout, StorageEnvironmentOptions.CreateMemoryOnly()));
_nodes.Add(raftEngine);
return raftEngine;
}
public SnapshotInstallationStateBehavior(RaftEngine engine) : base(engine)
{
_random = new Random((int)(engine.Name.GetHashCode() + DateTime.UtcNow.Ticks));
Timeout = _random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout);
}
public ClusterManagementHttpClient(RaftEngine raftEngine)
{
this.raftEngine = raftEngine;
}
public void AppendToLog(RaftEngine engine, IEnumerable<LogEntry> entries, long removeAllAfter)
{
using (var tx = _env.NewTransaction(TransactionFlags.ReadWrite))
{
var logs = tx.ReadTree(LogsTreeName);
var terms = tx.ReadTree(EntryTermsTreeName);
var metadata = tx.ReadTree(MetadataTreeName);
var changed = ReadIsTopologyChanged(metadata);
if (changed > removeAllAfter)
{
// need to reset the topology
var current = metadata.Read("current-topology");
var prevTopologyStr = current == null ? "{}" : current.Reader.ToStringValue();
metadata.Add("current-topology", prevTopologyStr);
metadata.Add("current-topology-index", EndianBitConverter.Little.GetBytes(0L));
var prevTopology = JsonConvert.DeserializeObject<Topology>(prevTopologyStr);
engine.RevertTopologyTo(prevTopology);
}
using (var it = logs.Iterate())
{
if (it.Seek(new Slice(EndianBitConverter.Big.GetBytes(removeAllAfter))) &&
it.MoveNext())
{
do
{
terms.Delete(it.CurrentKey);
} while (it.DeleteCurrentAndMoveNext());
}
}
foreach (var entry in entries)
{
var key = new Slice(EndianBitConverter.Big.GetBytes(entry.Index));
logs.Add(key, entry.Data);
terms.Add(key, new Slice(BitConverter.GetBytes(entry.Term)));
}
tx.Commit();
}
}
public SnapshotInstallationStateBehavior(RaftEngine engine) : base(engine)
{
_random = new Random((int)(engine.Name.GetHashCode() + DateTime.UtcNow.Ticks));
Timeout = _random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout);
}
protected void RestartAllNodes()
{
foreach (var raftEngine in _nodes)
{
raftEngine.Options.StorageOptions.OwnsPagers = false;
raftEngine.Dispose();
}
for (int i = 0; i < _nodes.Count; i++)
{
_nodes[i] = new RaftEngine(_nodes[i].Options);
}
}