private void BroadcastVoteRequests()
{
// BUG: duplicate votes from same follower
if (this.PeriodicTimer != null)
{
this.PeriodicTimer.Dispose();
}
this.PeriodicTimer = this.RegisterTimer(RebroadcastVoteRequests, null,
TimeSpan.FromSeconds(5 + this.Random.Next(20)),
TimeSpan.FromSeconds(5 + this.Random.Next(20)));
foreach (var server in this.Servers)
{
if (server.Key == this.ServerId)
{
continue;
}
var lastLogIndex = this.Logs.Count;
var lastLogTerm = this.GetLogTermForIndex(lastLogIndex);
ActorModel.Log($"<RaftLog> Server {this.ServerId} sending vote request " +
$"to server {server.Key}.");
server.Value.VoteRequest(this.CurrentTerm, this.ServerId, lastLogIndex, lastLogTerm);
}
}
private void BroadcastLastClientRequest()
{
ActorModel.Log($"<RaftLog> Leader {this.ServerId} sends append requests | term " +
$"{this.CurrentTerm} | log {this.Logs.Count}.");
var lastLogIndex = this.Logs.Count;
this.VotesReceived = 1;
foreach (var server in this.Servers)
{
if (server.Key == this.ServerId)
{
continue;
}
if (lastLogIndex < this.NextIndex[server.Value])
{
continue;
}
var logs = this.Logs.GetRange(this.NextIndex[server.Value] - 1,
this.Logs.Count - (this.NextIndex[server.Value] - 1));
var prevLogIndex = this.NextIndex[server.Value] - 1;
var prevLogTerm = this.GetLogTermForIndex(prevLogIndex);
server.Value.AppendEntriesRequest(this.CurrentTerm, this.ServerId, prevLogIndex,
prevLogTerm, logs, this.CommitIndex, this.LatestClientId.Value);
}
}
public async Task HandlePersonTimeout(object args)
{
this.UnregisterTimer(this.PersonTimer);
ActorModel.Log("[LOG] People enter the house.");
foreach (var person in this.People)
{
await person.Enter(Location.House);
}
}
public async Task TryToStealMoney()
{
int numOfPeople = await this.StateManager.GetStateAsync <int>("PeopleInside");
bool isSafeOpen = await this.StateManager.GetStateAsync <bool>("IsSafeOpen");
ActorModel.Log("[LOG] Thief is searching for money. Room has {0} people", numOfPeople);
ActorModel.Assert(!isSafeOpen || numOfPeople > 0, "Thief stole the money.");
}
public override Task OnActivateAsync()
{
ActorModel.Log($"<RaftLog> Client is activating.");
if (this.LatestCommand <= 0)
{
this.Random = new Random(DateTime.Now.Millisecond);
this.LatestCommand = -1;
}
return(base.OnActivateAsync());
}
public Task NotifyLeaderUpdate(int leaderId, int term)
{
if (this.LeaderTerm < term)
{
ActorModel.Log($"<RaftLog> ClusterManager found new leader '{leaderId}'");
this.Leader = this.GrainFactory.GetGrain <IServer>(leaderId);
this.LeaderTerm = term;
}
return(TaskDone.Done);
}
public Task ReceiveReminderAsync(string reminderName, byte[] context, TimeSpan dueTime, TimeSpan period)
{
if (reminderName.Equals("HandleMovementTimeout"))
{
var previousLocation = ActorModel.GetResult(this.StateManager.GetStateAsync <Location>("CurrentLocation"));
var location = ActorModel.GetResult(this.House.GotoRoom());
ActorModel.Log("[LOG] Person entered room {0}", location);
ActorModel.Wait(this.StateManager.SetStateAsync("CurrentLocation", location));
if (previousLocation == Location.Garden)
{
ActorModel.Wait(this.Garden.PersonExits());
}
else if (previousLocation == Location.Kitchen)
{
ActorModel.Wait(this.Kitchen.PersonExits());
}
else if (previousLocation == Location.Bedroom)
{
ActorModel.Wait(this.Bedroom.PersonExits());
}
if (location == Location.Garden)
{
ActorModel.Wait(this.Garden.PersonEnters());
}
else if (location == Location.Kitchen)
{
ActorModel.Wait(this.Kitchen.PersonEnters());
}
else if (location == Location.Bedroom)
{
ActorModel.Wait(this.Bedroom.PersonEnters());
}
}
else if (reminderName.Equals("HandleActionTimeout"))
{
var location = ActorModel.GetResult(this.StateManager.GetStateAsync <Location>("CurrentLocation"));
if (location == Location.Garden)
{
}
else if (location == Location.Kitchen)
{
}
else if (location == Location.Bedroom)
{
ActorModel.Wait(this.Bedroom.AccessSafe());
}
}
return(Task.FromResult(true));
}
private void BecomeCandidate()
{
ActorModel.Log($"<RaftLog> Server {this.ServerId} became CANDIDATE.");
this.Role = Role.Candidate;
this.CurrentTerm++;
this.VotedForCandidate = this.GrainFactory.GetGrain <IServer>(this.ServerId);
this.VotesReceived = 1;
ActorModel.Log($"<RaftLog> Candidate {this.ServerId} | term {this.CurrentTerm} " +
$"| election votes {this.VotesReceived} | log {this.Logs.Count}.");
this.BroadcastVoteRequests();
}
public Task Configure(int clusterId)
{
if (this.ClusterManager == null)
{
ActorModel.Log($"<RaftLog> Client is configuring.");
this.ClusterManager = this.GrainFactory.GetGrain <IClusterManager>(clusterId);
this.RequestTimer = this.RegisterTimer(PumpRequest, null,
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5));
}
return(TaskDone.Done);
}
public async Task AccessSafe()
{
bool isSafeOpen = await this.StateManager.GetStateAsync <bool>("IsSafeOpen");
if (isSafeOpen)
{
ActorModel.Log("[LOG] The safe is closed.");
await this.StateManager.SetStateAsync("IsSafeOpen", false);
}
else
{
ActorModel.Log("[LOG] The safe is open.");
await this.StateManager.SetStateAsync("IsSafeOpen", true);
}
}
public Task RelayClientRequest(int clientId, int command)
{
ActorModel.Log($"<RaftLog> ClusterManager is relaying client request " + command + "\n");
if (this.Leader != null)
{
this.Leader.ProcessClientRequest(clientId, command);
}
else
{
this.Cluster.RedirectClientRequest(clientId, command);
}
return(TaskDone.Done);
}
public Task ProcessResponse()
{
ActorModel.Log($"<RaftLog> Client received a response.");
if (this.RequestTimer != null)
{
this.RequestTimer.Dispose();
this.RequestTimer = null;
}
this.RequestTimer = this.RegisterTimer(PumpRequest, null,
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5));
return(TaskDone.Done);
}
private Task RedirectClientRequest(object args)
{
var request = (Tuple <int, int>)args;
if (this.RetryTimer != null)
{
this.RetryTimer.Dispose();
this.RetryTimer = null;
}
ActorModel.Log($"<RaftLog> ClusterManager is redirecting client request " + request.Item2 + "\n");
this.Cluster.RelayClientRequest(request.Item1, request.Item2);
return(TaskDone.Done);
}
private Task PumpRequest(object args)
{
if (this.RequestTimer != null)
{
this.RequestTimer.Dispose();
this.RequestTimer = null;
}
this.LatestCommand = new Random().Next(100);
ActorModel.Log($"<RaftLog> Client is sending new request {this.LatestCommand}.");
this.ClusterManager.RelayClientRequest(6, this.LatestCommand);
return(TaskDone.Done);
}
public Task ReceiveReminderAsync(string reminderName, byte[] context, TimeSpan dueTime, TimeSpan period)
{
ActorModel.Log("[LOG] Thief reminder {0}", reminderName);
if (reminderName.Equals("HandleMovementTimeout"))
{
var previousLocation = ActorModel.GetResult(this.StateManager.GetStateAsync <Location>("CurrentLocation"));
var location = ActorModel.GetResult(this.House.GotoRoom());
ActorModel.Log("[LOG] Thief tries to enter room {0}", location);
bool canEnter = false;
if (location == Location.Garden)
{
canEnter = ActorModel.GetResult(this.Bedroom.TryEnterRoom());
}
else if (location == Location.Kitchen)
{
canEnter = ActorModel.GetResult(this.Bedroom.TryEnterRoom());
}
else if (previousLocation == Location.Kitchen &&
location == Location.Bedroom)
{
canEnter = ActorModel.GetResult(this.Bedroom.TryEnterRoom());
}
if (canEnter)
{
ActorModel.Log("[LOG] Thief entered room {0}", location);
ActorModel.Wait(this.StateManager.SetStateAsync("CurrentLocation", location));
}
}
else if (reminderName.Equals("HandleActionTimeout"))
{
Console.WriteLine("Thief is handling action timeout");
var location = ActorModel.GetResult(this.StateManager.GetStateAsync <Location>("CurrentLocation"));
if (location == Location.Garden)
{
}
else if (location == Location.Kitchen)
{
}
else if (location == Location.Bedroom)
{
ActorModel.Wait(this.Bedroom.TryToStealMoney());
}
}
return(Task.FromResult(true));
}
public Task Configure(int id, List <int> serverIds, int clusterId)
{
if (this.Servers.Count == 0)
{
this.ServerId = id;
ActorModel.Log($"<RaftLog> Server {id} is configuring.");
foreach (var idx in serverIds)
{
ActorModel.Log($"<RaftLog> Server {id} is setting up server {idx}.");
this.Servers.Add(idx, this.GrainFactory.GetGrain <IServer>(idx));
}
this.ClusterManager = this.GrainFactory.GetGrain <IClusterManager>(clusterId);
this.BecomeFollower();
}
return(TaskDone.Done);
}
public override Task OnActivateAsync()
{
if (this.Servers == null)
{
this.NumberOfServers = 5;
this.Leader = null;
this.LeaderTerm = 0;
this.Cluster = this.GrainFactory.GetGrain <IClusterManager>(0);
this.Client = this.GrainFactory.GetGrain <IClient>(1);
this.Servers = new Dictionary <int, IServer>();
for (int idx = 0; idx < this.NumberOfServers; idx++)
{
ActorModel.Log($"<RaftLog> ClusterManager is creating server {idx + 2}.");
this.Servers.Add(idx + 2, this.GrainFactory.GetGrain <IServer>(idx + 2));
}
}
return(base.OnActivateAsync());
}
private void BecomeFollower()
{
ActorModel.Log($"<RaftLog> Server {this.ServerId} became FOLLOWER.");
this.Role = Role.Follower;
this.Leader = null;
this.VotesReceived = 0;
if (this.ElectionTimer != null)
{
this.ElectionTimer.Dispose();
}
if (this.PeriodicTimer != null)
{
this.PeriodicTimer.Dispose();
}
this.ElectionTimer = this.RegisterTimer(StartLeaderElection, null,
TimeSpan.FromSeconds(5 + this.Random.Next(30)),
TimeSpan.FromSeconds(5 + this.Random.Next(30)));
}
public override Task OnActivateAsync()
{
ActorModel.Log($"<RaftLog> Server is activating.");
if (this.CurrentTerm == 0)
{
this.Leader = null;
this.VotedForCandidate = null;
this.Logs = new List <Log>();
this.CommitIndex = 0;
this.LastApplied = 0;
this.Servers = new Dictionary <int, IServer>();
this.NextIndex = new Dictionary <IServer, int>();
this.MatchIndex = new Dictionary <IServer, int>();
this.Random = new Random(DateTime.Now.Millisecond);
}
return(base.OnActivateAsync());
}
public Task Start()
{
if (this.People == null)
{
this.People = new List <IPerson>();
this.Thief = ActorProxy.Create <IThief>(new ActorId(1), "fabric:/FabricSmartHome");
for (int idx = 2; idx < 9; idx++)
{
this.People.Add(ActorProxy.Create <IPerson>(new ActorId(idx), "fabric:/FabricSmartHome"));
}
ActorProxy.Create <IHouse>(new ActorId(100), "fabric:/FabricSmartHome");
ActorModel.Log("[LOG] Environment started.");
this.PersonTimer = this.RegisterTimer(HandlePersonTimeout, null, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(5));
this.ThiefTimer = this.RegisterTimer(HandleThiefTimeout, null, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(5));
}
return(new Task(() => { }));
}
void ProcessVoteRequest(int term, int candidateId, int lastLogIndex, int lastLogTerm)
{
var candidate = this.GrainFactory.GetGrain <IServer>(candidateId);
if (term < this.CurrentTerm ||
(this.VotedForCandidate != null && this.VotedForCandidate != candidate) ||
this.Logs.Count > lastLogIndex ||
this.GetLogTermForIndex(this.Logs.Count) > lastLogTerm)
{
ActorModel.Log($"<RaftLog> Server {this.ServerId} | term {this.CurrentTerm} " +
$"| log {this.Logs.Count} | vote granted {false}.");
candidate.VoteResponse(this.CurrentTerm, false);
}
else
{
this.VotedForCandidate = candidate;
this.Leader = null;
ActorModel.Log($"<RaftLog> Server {this.ServerId} | term {this.CurrentTerm} " +
$"| log {this.Logs.Count} | vote granted {true}.");
candidate.VoteResponse(this.CurrentTerm, true);
}
}
private void BecomeLeader()
{
ActorModel.Log($"<RaftLog> Server {this.ServerId} became LEADER.");
ActorModel.Log($"<RaftLog> Leader {this.ServerId} | term {this.CurrentTerm} " +
$"| election votes {this.VotesReceived} | log {this.Logs.Count}.");
this.Role = Role.Leader;
this.VotesReceived = 0;
ActorModel.Runtime.InvokeMonitor <SafetyMonitor>(new SafetyMonitor.NotifyLeaderElected(this.CurrentTerm));
var logIndex = this.Logs.Count;
var logTerm = this.GetLogTermForIndex(logIndex);
this.NextIndex.Clear();
this.MatchIndex.Clear();
foreach (var server in this.Servers)
{
if (server.Key == this.ServerId)
{
continue;
}
this.NextIndex.Add(server.Value, logIndex + 1);
this.MatchIndex.Add(server.Value, 0);
}
foreach (var server in this.Servers)
{
if (server.Key == this.ServerId)
{
continue;
}
server.Value.AppendEntriesRequest(this.CurrentTerm, this.ServerId,
logIndex, logTerm, new List <Log>(), this.CommitIndex, -1);
}
}
public Task AppendEntriesResponse(int term, bool success, int serverId, int clientId)
{
ActorModel.Log($"<RaftLog> Server {this.ServerId} | term {this.CurrentTerm} " +
$"| append response {success}.");
if (this.Role == Role.Follower)
{
if (term > this.CurrentTerm)
{
this.CurrentTerm = term;
this.VotedForCandidate = null;
}
}
else if (this.Role == Role.Candidate)
{
if (term > this.CurrentTerm)
{
this.CurrentTerm = term;
this.VotedForCandidate = null;
this.BecomeFollower();
}
}
else if (this.Role == Role.Leader)
{
if (term > this.CurrentTerm)
{
this.CurrentTerm = term;
this.VotedForCandidate = null;
if (this.LatestClientId != null)
{
this.ClusterManager.RelayClientRequest(this.LatestClientId.Value,
this.LatestClientCommand.Value);
}
this.BecomeFollower();
}
else if (term != this.CurrentTerm)
{
return(TaskDone.Done);
}
var server = this.GrainFactory.GetGrain <IServer>(serverId);
if (success)
{
this.NextIndex[server] = this.Logs.Count + 1;
this.MatchIndex[server] = this.Logs.Count;
this.VotesReceived++;
if (clientId >= 0 &&
this.VotesReceived >= (this.Servers.Count / 2) + 1)
{
ActorModel.Log($"<RaftLog> Leader {this.ServerId} | term {this.CurrentTerm} " +
$"| append votes {this.VotesReceived} | append success.");
var commitIndex = this.MatchIndex[server];
if (commitIndex > this.CommitIndex &&
this.Logs[commitIndex - 1].Term == this.CurrentTerm)
{
this.CommitIndex = commitIndex;
ActorModel.Log($"<RaftLog> Leader {this.ServerId} | term {this.CurrentTerm} " +
$"| log {this.Logs.Count} | command {this.Logs[commitIndex - 1].Command}.");
}
this.VotesReceived = 0;
this.LatestClientId = null;
this.LatestClientCommand = null;
var client = this.GrainFactory.GetGrain <IClient>(clientId);
client.ProcessResponse();
}
}
else
{
if (this.NextIndex[server] > 1)
{
this.NextIndex[server] = this.NextIndex[server] - 1;
}
var logs = this.Logs.GetRange(this.NextIndex[server] - 1,
this.Logs.Count - (this.NextIndex[server] - 1));
var prevLogIndex = this.NextIndex[server] - 1;
var prevLogTerm = this.GetLogTermForIndex(prevLogIndex);
ActorModel.Log($"<RaftLog> Leader {this.ServerId} | term {this.CurrentTerm} | log " +
$"{this.Logs.Count} | append votes {this.VotesReceived} " +
$"append fail (next idx = {this.NextIndex[server]}).");
server.AppendEntriesRequest(this.CurrentTerm, this.ServerId, prevLogIndex,
prevLogTerm, logs, this.CommitIndex, clientId);
}
}
return(TaskDone.Done);
}
public async Task HandleThiefTimeout(object args)
{
this.UnregisterTimer(this.ThiefTimer);
ActorModel.Log("[LOG] Thief enters the house.");
await this.Thief.Enter(Location.House);
}
private void AppendEntries(int term, int leaderId, int prevLogIndex,
int prevLogTerm, List <Log> entries, int leaderCommit, int clientId)
{
var leader = this.GrainFactory.GetGrain <IServer>(leaderId);
if (term < this.CurrentTerm)
{
ActorModel.Log($"<RaftLog> Server {this.ServerId} | term {this.CurrentTerm} | log " +
$"{this.Logs.Count} | last applied {this.LastApplied} | append false (< term).");
leader.AppendEntriesResponse(this.CurrentTerm, false, this.ServerId, clientId);
}
else
{
if (prevLogIndex > 0 &&
(this.Logs.Count < prevLogIndex ||
this.Logs[prevLogIndex - 1].Term != prevLogTerm))
{
ActorModel.Log($"<RaftLog> Server {this.ServerId} | term {this.CurrentTerm} | log " +
$"{this.Logs.Count} | last applied {this.LastApplied} | append false (not in log).");
leader.AppendEntriesResponse(this.CurrentTerm, false, this.ServerId, clientId);
}
else
{
if (entries.Count > 0)
{
var currentIndex = prevLogIndex + 1;
foreach (var entry in entries)
{
if (this.Logs.Count < currentIndex)
{
this.Logs.Add(entry);
}
else if (this.Logs[currentIndex - 1].Term != entry.Term)
{
this.Logs.RemoveRange(currentIndex - 1, this.Logs.Count - (currentIndex - 1));
this.Logs.Add(entry);
}
currentIndex++;
}
}
if (leaderCommit > this.CommitIndex &&
this.Logs.Count < leaderCommit)
{
this.CommitIndex = this.Logs.Count;
}
else if (leaderCommit > this.CommitIndex)
{
this.CommitIndex = leaderCommit;
}
if (this.CommitIndex > this.LastApplied)
{
this.LastApplied++;
}
ActorModel.Log($"<RaftLog> Server {this.ServerId} | term {this.CurrentTerm} | log " +
$"{this.Logs.Count} | entries received {entries.Count} | last applied " +
$"{this.LastApplied} | append true.");
leader.AppendEntriesResponse(this.CurrentTerm, true, this.ServerId, clientId);
}
}
}
