private void SetupPeerAppendLogJobs(IEnumerable <Peer> peers)
{
foreach (var w in _workers.GetWorkers(Queues.PeerAppendLog))
{
w.Start();
}
_workers.GetWorkers(Queues.ProcessCommandQueue).Single().Start();
foreach (var p in peers)
{
var localP = p;
var q = Queues.PeerAppendLog + localP.Address;
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] setting up peer append log for queue {q}");
var todo = PeerAppendLog(localP);
_workers.Enqueue(q,
new Job(todo,
TheTrace.LogPolicy(_meAsAPeer.ShortName).WaitAndRetryAsync(3, (i) => TimeSpan.FromMilliseconds(i * i * 50)),
TimeSpan.FromMilliseconds(30)));
}
// Applying commands received from the clients
Func <CancellationToken, Task> pcq = ProcessCommandsQueue;
_workers.Enqueue(Queues.ProcessCommandQueue,
new Job(pcq,
TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync(),
_settings.ElectionTimeoutMin.Multiply(0.2)));
}
public void CanAdd2And2EvenIfItFails()
{
var maths = new Worker("maths");
var ran = false;
var timesForExceptions = 4;
var job = new Job <int>(c =>
{
if (timesForExceptions-- > 0)
{
throw new ApplicationException();
}
ran = true;
return(Task.FromResult(2 + 2));
},
TheTrace.LogPolicy("this").RetryForeverAsync()
);
maths.Start();
maths.Enqueue(job);
while (!job.IsFinished)
{
Thread.Sleep(100);
}
Assert.True(ran);
}
private async Task Candidacy(CancellationToken c)
{
while (_role == Role.Candidate)
{
var forMe = 1; // vote for yourself
var againstMe = 0;
var peers = _peerManager.GetPeers().ToArray();
var concensus = (peers.Length / 2) + 1;
var proxies = peers.Select(x => _peerManager.GetProxy(x.Address));
var retry = TheTrace.LogPolicy(_meAsAPeer.ShortName).WaitAndRetryAsync(3, (i) => TimeSpan.FromMilliseconds(i * i * 30));
var policy = Policy.TimeoutAsync(_settings.CandidacyTimeout).WrapAsync(retry);
var request = new RequestVoteRequest()
{
CandidateId = State.Id,
CurrentTerm = State.CurrentTerm,
LastLogIndex = _logPersister.LastIndex,
LastLogTerm = _logPersister.LastEntryTerm
};
var all = await Task.WhenAll(proxies.Select(p => policy.ExecuteAndCaptureAsync(() => p.RequestVoteAsync(request))));
var maxTerm = 0L;
foreach (var r in all)
{
if (r.Outcome == OutcomeType.Successful)
{
if (r.Result.CurrentTerm > maxTerm)
{
maxTerm = r.Result.CurrentTerm;
}
if (r.Result != null && r.Result.VoteGranted)
{
forMe++;
}
else
{
againstMe++;
}
}
}
if (againstMe >= concensus)
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Result of the candidacy for term {State.CurrentTerm}. I got rejected with {againstMe} votes :/");
BecomeFollower(maxTerm);
}
else if (forMe >= concensus)
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Result of the candidacy for term {State.CurrentTerm}. I got elected with {forMe} votes! :)");
BecomeLeader();
}
else
{
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Result of the candidacy for term {State.CurrentTerm}. Non-conclusive with {forMe} for me and {againstMe} against me.");
}
}
}
private async Task HeartBeatSend(CancellationToken c)
{
if (_role != Role.Leader)
{
return;
}
if (_lastHeartbeatSent.Since() < _settings.ElectionTimeoutMin.Multiply(0.2))
{
return;
}
var currentTerm = State.CurrentTerm; // create a var. Could change during the method leading to confusing logs.
var req = new AppendEntriesRequest()
{
CurrentTerm = currentTerm,
Entries = new byte[0][],
LeaderCommitIndex = _volatileState.CommitIndex,
LeaderId = State.Id,
PreviousLogIndex = long.MaxValue,
PreviousLogTerm = long.MaxValue
};
var peers = _peerManager.GetPeers().ToArray();
var proxies = peers.Select(x => _peerManager.GetProxy(x.Address));
var retry = TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync();
var policy = Policy.TimeoutAsync(_settings.ElectionTimeoutMin.Multiply(0.2)).WrapAsync(retry);
var all = await Task.WhenAll(proxies.Select(p => policy.ExecuteAndCaptureAsync(() => p.AppendEntriesAsync(req))));
var maxTerm = currentTerm;
foreach (var r in all)
{
if (r.Outcome == OutcomeType.Successful)
{
if (!r.Result.IsSuccess)
{
TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Got this reason for unsuccessful AppendEntriesAsync from a peer: {r.Result.Reason}");
}
// NOTE: We do NOT change leadership if they send higher term, since they could be candidates whom will not become leaders
// we actually do not need to do anything with the result other than logging it
if (r.Result.CurrentTerm > maxTerm)
{
maxTerm = r.Result.CurrentTerm;
}
}
}
if (maxTerm > State.CurrentTerm)
{
TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Revolution brewing. Terms as high as {maxTerm} (vs my {currentTerm}) were seen.");
}
_lastHeartbeatSent.Set();
}
private void SetupPool()
{
var names = new List <string>();
foreach (var p in _peerManager.GetPeers())
{
names.Add(Queues.PeerAppendLog + p.Address);
}
names.Add(Queues.LogCommit);
names.Add(Queues.HeartBeatReceiveAndCandidacy);
names.Add(Queues.HeartBeatSend);
names.Add(Queues.ProcessCommandQueue);
names.Add(Queues.CreateSnapshot);
_workers = new WorkerPool(names.ToArray());
_workers.Start();
// LogCommit
Func <CancellationToken, Task> logCommit = LogCommit;
_workers.Enqueue(Queues.LogCommit,
new Job(logCommit,
TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync(),
_settings.ElectionTimeoutMin.Multiply(0.2)));
// receiving heartbeat
Func <CancellationToken, Task> hbr = HeartBeatReceive;
_workers.Enqueue(Queues.HeartBeatReceiveAndCandidacy,
new Job(hbr,
TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync(),
_settings.ElectionTimeoutMin.Multiply(0.2)));
// sending heartbeat
Func <CancellationToken, Task> hbs = HeartBeatSend;
_workers.Enqueue(Queues.HeartBeatSend,
new Job(hbs,
TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync(),
_settings.ElectionTimeoutMin.Multiply(0.2)));
// Applying commands received from the clients
Func <CancellationToken, Task> cs = CreateSnapshot;
_workers.Enqueue(Queues.CreateSnapshot,
new Job(cs,
TheTrace.LogPolicy(_meAsAPeer.ShortName).WaitAndRetryAsync(2, (i) => TimeSpan.FromMilliseconds(i * i * 50)),
_settings.ElectionTimeoutMin.Multiply(0.2)));
TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Setup finished.");
}
public void CanAdd2And2()
{
var maths = new Worker("maths");
var ran = false;
var job = new Job <int>(c => Task.FromResult(2 + 2),
TheTrace.LogPolicy("this").RetryForeverAsync(), (n) => { ran = true; });
maths.Start();
maths.Enqueue(job);
Thread.Sleep(100);
Assert.True(ran);
}
private Func <CancellationToken, Task> PeerAppendLog(Peer peer)
{
return((CancellationToken c) =>
{
long nextIndex;
long matchIndex;
var hasMatch = _volatileLeaderState.TryGetMatchIndex(peer.Id, out matchIndex);
var hasNext = _volatileLeaderState.TryGetNextIndex(peer.Id, out nextIndex);
var myLastIndex = _logPersister.LastIndex;
if (!hasMatch)
{
TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Could not find peer with id {peer.Id} and address {peer.Address} in matchIndex dic.");
return Task.CompletedTask;
}
if (!hasNext)
{
TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Could not find peer with id {peer.Id} and address {peer.Address} in nextIndex dic.");
return Task.CompletedTask;
}
if (nextIndex > myLastIndex)
{
TheTrace.TraceVerbose($"[{_meAsAPeer.ShortName}] PeerAppendLog - Nothing to do for peer {peer.ShortName} since myIndex is {myLastIndex} and nextIndex is {nextIndex}.");
return Task.CompletedTask; // nothing to do
}
var count = (int)Math.Min(_settings.MaxNumberLogEntriesToAskToBeAppended, myLastIndex + 1 - nextIndex);
var proxy = _peerManager.GetProxy(peer.Address);
var retry = TheTrace.LogPolicy(_meAsAPeer.ShortName).WaitAndRetryAsync(2, (i) => TimeSpan.FromMilliseconds(20));
var policy = Policy.TimeoutAsync(_settings.CandidacyTimeout).WrapAsync(retry); // TODO: create its own timeout
if (nextIndex >= _logPersister.LogOffset)
{
TheTrace.TraceVerbose($"[{_meAsAPeer.ShortName}] Intending to do SendLog for peer {peer.Address} with nextIndex {nextIndex} and count {count}.");
return SendLogs(proxy, policy, peer, nextIndex, matchIndex, count);
}
else
{
TheTrace.TraceVerbose($"[{_meAsAPeer.ShortName}] Intending to do SendSnapshot for peer {peer.Address}.");
return SendSnapshot(proxy, policy, peer, nextIndex, matchIndex);
}
});
}
public async Task CanAdd2And2ThousandTimes()
{
var maths = new Worker("maths");
var ran = 0;
var job = new Job <int>(c =>
{
ran++;
return(Task.FromResult(2 + 2));
},
TheTrace.LogPolicy("this")
.RetryForeverAsync()
);
maths.Start();
for (int i = 0; i < 1000; i++)
{
maths.Enqueue(job);
}
Thread.Sleep(10000);
Assert.Equal(1000, ran);
}