private void WhenTheServerReceivesFailThenASuccessFromARemoteServer(ServerInCluster serverInCluster)
{
var fail = new AppendEntriesResponse(_server.CurrentTerm, false, serverInCluster.Id, _server.Id);
var success = new AppendEntriesResponse(_server.CurrentTerm, true, serverInCluster.Id, _server.Id);
_messageBus.Setup(x => x.Send(It.IsAny <AppendEntries>())).ReturnsInOrder(Task.FromResult(fail), Task.FromResult(success));
}
public ServerContainer(IWebHost webHost, Server server, string serverUrl, IMessageSender messageSender, ServerInCluster serverInCluster, IMessageBus messageBus, IStateMachine stateMachine)
{
StateMachine = stateMachine;
MessageBus = messageBus;
WebHost = webHost;
Server = server;
ServerUrl = serverUrl;
MessageSender = messageSender;
ServerInCluster = serverInCluster;
}
public RequestVoteResponse Receive(RequestVote requestVote)
{
_logger.LogDebug($"Server: {Id} received request vote in term: {CurrentTerm}");
if (!_serversInClusterInCluster.Contains(requestVote.CandidateId))
{
var remoteServer = new ServerInCluster(requestVote.CandidateId);
_serversInClusterInCluster.Add(remoteServer);
}
// If RPC request or response contains term T > currentTerm:
// set currentTerm = T, convert to follower (§5.1)
if (requestVote.Term > CurrentTerm)
{
BecomeFollowerAndMatchTerm(requestVote.Term, requestVote.CandidateId);
_logger.LogDebug($"Server: {Id} received request vote in term: {CurrentTerm}, became follower");
}
/*
* 1.Reply false if term < currentTerm(�5.1)*/
if (requestVote.Term < CurrentTerm)
{
_logger.LogDebug($"Server: {Id} received request vote in term: {CurrentTerm}, voted false because term was less than current term");
return(new RequestVoteResponse(CurrentTerm, false, requestVote.CandidateId, Id));
}
lock (_lock)
{
/*
* 2.If votedFor is null or candidateId, and candidate�s log is at
* least as up - to - date as receiver�s log, grant vote(�5.2, �5.4)*/
if (VotedForIsNullOrAlreadyVotingForCandidate(requestVote))
{
if (CandidatesLogIsAtLeastUpToDateAsServers(requestVote))
{
VotedFor = requestVote.CandidateId;
_logger.LogDebug($"Server: {Id} received request vote in term: {CurrentTerm}, voted true");
return(new RequestVoteResponse(CurrentTerm, true, requestVote.CandidateId, Id));
}
}
}
_logger.LogDebug($"Server: {Id} received request vote in term: {CurrentTerm}, voted false because already voted for or candidates log isnt up to date");
return(new RequestVoteResponse(CurrentTerm, false, requestVote.CandidateId, Id));
}
private async Task GivenAServerIsRunning(string baseUrl)
{
Server server = null;
IMessageSender messageSender = null;
ServerInCluster serverInCluster = null;
IMessageBus messageBus = null;
IStateMachine stateMachine = null;
var webHost = new WebHostBuilder()
.UseUrls(baseUrl)
.UseKestrel()
.UseContentRoot(Directory.GetCurrentDirectory())
.ConfigureServices(s =>
{
s.AddSingleton <IMessageSender, HttpClientMessageSender>();
s.AddSingleton <IMessageBus, InMemoryBus>();
s.AddSingleton <IStateMachine, FakeStateMachine>();
s.AddSingleton <IServersInCluster, InMemoryServersInCluster>();
s.AddSingleton <IServiceRegistry>(_serviceRegistry);
})
.Configure(app =>
{
var loggerFactory = app.ApplicationServices.GetRequiredService <ILoggerFactory>();
messageSender = app.ApplicationServices.GetRequiredService <IMessageSender>();
messageBus = app.ApplicationServices.GetRequiredService <IMessageBus>();
stateMachine = app.ApplicationServices.GetRequiredService <IStateMachine>();
var result = app.UseRaftyForTesting(new Uri(baseUrl), messageSender, messageBus, stateMachine,
_serviceRegistry, loggerFactory, _serversInCluster);
server = result.server;
serverInCluster = result.serverInCluster;
})
.Build();
webHost.Start();
var serverContainer = new ServerContainer(webHost, server, baseUrl, messageSender, serverInCluster, messageBus, stateMachine);
_servers.Add(serverContainer);
}
private async Task GivenAServerIsRunning(string baseUrl)
{
Server server = null;
HttpClientMessageSender messageSender = null;
ServerInCluster serverInCluster = null;
InMemoryBus messageBus = null;
IStateMachine stateMachine = null;
var webHost = new WebHostBuilder()
.UseUrls(baseUrl)
.UseKestrel()
.UseContentRoot(Directory.GetCurrentDirectory())
.ConfigureServices(x =>
{
})
.Configure(app =>
{
var logger = new ConsoleLogger("ConsoleLogger", (x, y) => true, true);
messageSender = new HttpClientMessageSender(_serviceRegistry, logger);
messageBus = new InMemoryBus(messageSender);
stateMachine = new FakeStateMachine();
var result = app.UseRaftyForTesting(new Uri(baseUrl), messageSender, messageBus, stateMachine,
_serviceRegistry, logger, _serversInCluster);
server = result.server;
serverInCluster = result.serverInCluster;
})
.Build();
webHost.Start();
var serverContainer = new ServerContainer(webHost, server, baseUrl, messageSender, serverInCluster, messageBus, stateMachine);
_servers.Add(serverContainer);
}
private void AddNewServerToServersInCluster(AppendEntries appendEntries)
{
var remoteServer = new ServerInCluster(appendEntries.LeaderId);
_serversInClusterInCluster.Add(remoteServer);
}
private void AddNewServerToServersInCluster(RequestVote requestVote)
{
var remoteServer = new ServerInCluster(requestVote.CandidateId);
_serversInClusterInCluster.Add(remoteServer);
}
public static (IApplicationBuilder builder, Server server, ServerInCluster serverInCluster) UseRaftyForTesting(this IApplicationBuilder builder,
Uri baseUri,
IMessageSender messageSender,
IMessageBus messageBus,
IStateMachine stateMachine,
IServiceRegistry serviceRegistry,
ILoggerFactory loggerFactory,
IServersInCluster serversInCluster,
string raftyBasePath = null)
{
var urlConfig = RaftyUrlConfig.Get(raftyBasePath);
var server = new Server(messageBus, serversInCluster, stateMachine, loggerFactory);
var logger = loggerFactory.CreateLogger <IApplicationBuilder>();
serviceRegistry.Register(new RegisterService(RaftyServiceDiscoveryName.Get(), server.Id, baseUri));
messageSender.SetServer(server);
var serverInCluster = new ServerInCluster(server.Id);
serversInCluster.Add(serverInCluster);
builder.Map(urlConfig.appendEntriesUrl, app =>
{
app.Run(async context =>
{
try
{
var reader = new StreamReader(context.Request.Body);
var content = reader.ReadToEnd();
var appendEntries = JsonConvert.DeserializeObject <AppendEntries>(content, new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.All
});
var appendEntriesResponse = await server.Receive(appendEntries);
await context.Response.WriteAsync(JsonConvert.SerializeObject(appendEntriesResponse));
}
catch (Exception exception)
{
logger.LogError(new EventId(1), exception, $"There was an error handling {urlConfig.appendEntriesUrl}");
}
});
});
builder.Map(urlConfig.requestVoteUrl, app =>
{
app.Run(async context =>
{
try
{
var reader = new StreamReader(context.Request.Body);
var content = reader.ReadToEnd();
var requestVote = JsonConvert.DeserializeObject <RequestVote>(content, new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.All
});
var requestVoteResponse = server.Receive(requestVote);
await context.Response.WriteAsync(JsonConvert.SerializeObject(requestVoteResponse));
}
catch (Exception exception)
{
logger.LogError(new EventId(1), exception, $"There was an error handling {urlConfig.requestVoteUrl}");
}
});
});
builder.Map(urlConfig.commandUrl, app =>
{
app.Run(async context =>
{
try
{
var reader = new StreamReader(context.Request.Body);
var content = reader.ReadToEnd();
var command = JsonConvert.DeserializeObject <Command>(content, new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.All
});
var sendCommandToLeaderResponse = await server.Receive(command);
await context.Response.WriteAsync(JsonConvert.SerializeObject(sendCommandToLeaderResponse));
}
catch (Exception exception)
{
logger.LogError(new EventId(1), exception, $"There was an error handling {urlConfig.commandUrl}");
}
});
});
var applicationLifetime = builder.ApplicationServices.GetRequiredService <IApplicationLifetime>();
applicationLifetime.ApplicationStopping.Register(() => OnStopping(builder.ApplicationServices));
applicationLifetime.ApplicationStopped.Register(() => OnStopped(builder.ApplicationServices));
return(builder, server, serverInCluster);
}
public void Remove(ServerInCluster serverInCluster)
{
All.Remove(serverInCluster);
}
public void Add(ServerInCluster serverInCluster)
{
All.Add(serverInCluster);
}
public async Task <AppendEntriesResponse> Receive(AppendEntries appendEntries)
{
if (!_serversInClusterInCluster.Contains(appendEntries.LeaderId))
{
var remoteServer = new ServerInCluster(appendEntries.LeaderId);
_serversInClusterInCluster.Add(remoteServer);
}
if (State is Leader)
{
BecomeFollowerAndMatchTerm(appendEntries.Term, appendEntries.LeaderId);
}
_lastAppendEntriesMessageId = appendEntries.MessageId;
// If RPC request or response contains term T > currentTerm:
// set currentTerm = T, convert to follower (§5.1)
if (appendEntries.Term > CurrentTerm || State is Candidate)
{
BecomeFollowerAndMatchTerm(appendEntries.Term, appendEntries.LeaderId);
}
/*
* 1.Reply false if term < currentTerm(�5.1)
*/
if (appendEntries.Term < CurrentTerm)
{
return(new AppendEntriesResponse(CurrentTerm, false, Id, appendEntries.LeaderId));
}
if (appendEntries.Entry == null)
{
//todo heartbeat reset election timer
return(new AppendEntriesResponse(CurrentTerm, true, Id, appendEntries.LeaderId));
}
/*
* 2.Reply false if log doesn�t contain an entry at prevLogIndex
* whose term matches prevLogTerm(�5.3)*/
if (Log.Count > 0 && Log.Count > appendEntries.PreviousLogIndex && Log[appendEntries.PreviousLogIndex].Term != appendEntries.PreviousLogTerm)
{
return(new AppendEntriesResponse(CurrentTerm, false, Id, appendEntries.LeaderId));
}
/* 3.If an existing entry conflicts with a new one(same index
* but different terms), delete the existing entry and all that
* follow it*/
var newEntry = appendEntries.Entry;
if (Log.Count > 0 && Log.Count > appendEntries.PreviousLogIndex)
{
var existingEntry = Log[appendEntries.PreviousLogIndex];
if (existingEntry.Term != newEntry.Term)
{
Log.Remove(existingEntry);
}
}
else if (Log.Count > 0 && Log.Count > appendEntries.PreviousLogIndex + 1)
{
var existingEntry = Log[appendEntries.PreviousLogIndex + 1];
if (existingEntry.Term != newEntry.Term)
{
Log.Remove(existingEntry);
}
}
/* 4.Append any new entries not already in the log
*/
Log.Add(newEntry);
CommitIndex = CommitIndex + 1;
/*
* 5.If leaderCommit > commitIndex, set commitIndex =
* min(leaderCommit, index of last new entry)
*/
if (appendEntries.LeaderCommit > CommitIndex)
{
CommitIndex = Math.Min(appendEntries.LeaderCommit, Log.Count - 1);
}
/* If commitIndex > lastApplied: increment lastApplied, apply
* log[lastApplied] to state machine(§5.3)*/
if (CommitIndex > LastApplied)
{
if (Log.Count > 1)
{
LastApplied++;
}
await _stateMachine.Apply(Log[LastApplied].Command);
return(new AppendEntriesResponse(CurrentTerm, true, Id, appendEntries.LeaderId));
}
return(new AppendEntriesResponse(CurrentTerm, false, Id, appendEntries.LeaderId));
}
