private static async Task <Result <VotingResult> > VoteAsync(IRaftClusterMember voter, long term, IAuditTrail <IRaftLogEntry> auditTrail, CancellationToken token)
{
var lastIndex = auditTrail.GetLastIndex(false);
var lastTerm = await auditTrail.GetTermAsync(lastIndex, token).ConfigureAwait(false);
VotingResult result;
try
{
var response = await voter.VoteAsync(term, lastIndex, lastTerm, token).ConfigureAwait(false);
term = response.Term;
result = response.Value ? VotingResult.Granted : VotingResult.Rejected;
}
catch (OperationCanceledException)
{
result = VotingResult.Canceled;
}
catch (MemberUnavailableException)
{
result = VotingResult.NotAvailable;
term = -1L;
}
return(new Result <VotingResult>(term, result));
}
//true if at least one entry from current term is stored on this node; otherwise, false
private static async Task <Result <bool> > AppendEntriesAsync(IRaftClusterMember member, long commitIndex,
long term,
IAuditTrail <ILogEntry> transactionLog, ILogger logger, CancellationToken token)
{
var currentIndex = transactionLog.GetLastIndex(false);
logger.ReplicationStarted(member.Endpoint, currentIndex);
var precedingIndex = Math.Max(0, member.NextIndex - 1);
var precedingTerm = (await transactionLog.GetEntryAsync(precedingIndex).ConfigureAwait(false) ??
transactionLog.First).Term;
var entries = currentIndex >= member.NextIndex
? await transactionLog.GetEntriesAsync(member.NextIndex).ConfigureAwait(false)
: Array.Empty <ILogEntry>();
logger.ReplicaSize(member.Endpoint, entries.Count, precedingIndex, precedingTerm);
//trying to replicate
var result = await member
.AppendEntriesAsync(term, entries, precedingIndex, precedingTerm, commitIndex, token)
.ConfigureAwait(false);
if (result.Value)
{
logger.ReplicationSuccessful(member.Endpoint, member.NextIndex);
member.NextIndex.VolatileWrite(currentIndex + 1);
//checks whether the at least one entry from current term is stored on this node
result = result.SetValue(entries.Any(entry => entry.Term == term));
}
else
{
logger.ReplicationFailed(member.Endpoint, member.NextIndex.DecrementAndGet());
}
return(result);
}
internal static async Task WaitForCommitAsync(IAuditTrail log, IAsyncEvent commitEvent, long index, TimeSpan timeout, CancellationToken token)
{
for (var timeoutMeasurement = new Timeout(timeout); log.GetLastIndex(true) < index; await commitEvent.Wait(timeout, token).ConfigureAwait(false))
{
timeoutMeasurement.ThrowIfExpired(out timeout);
}
}
internal static async ValueTask <bool> IsUpToDateAsync(this IAuditTrail <ILogEntry> auditTrail, long index, long term)
{
var localIndex = auditTrail.GetLastIndex(false);
var localTerm = (await auditTrail.GetEntryAsync(localIndex).ConfigureAwait(false) ?? auditTrail.First)
.Term;
return(index >= localIndex && term >= localTerm);
}
/// <summary>
/// Starts cluster synchronization.
/// </summary>
/// <param name="period">Time period of Heartbeats.</param>
/// <param name="transactionLog">Transaction log.</param>
/// <param name="token">The toke that can be used to cancel the operation.</param>
internal LeaderState StartLeading(TimeSpan period, IAuditTrail <IRaftLogEntry> transactionLog, CancellationToken token)
{
foreach (var member in stateMachine.Members)
{
member.NextIndex = transactionLog.GetLastIndex(false) + 1;
}
heartbeatTask = DoHeartbeats(period, transactionLog, token);
return(this);
}
internal ValueTask <Result <bool> > Start(IAuditTrail <IRaftLogEntry> auditTrail)
{
var currentIndex = this.currentIndex = auditTrail.GetLastIndex(false);
logger.ReplicationStarted(member.Endpoint, currentIndex);
return(currentIndex >= member.NextIndex ?
auditTrail.ReadAsync <Replicator, Result <bool> >(this, member.NextIndex, token) :
ReadAsync <IRaftLogEntry, IRaftLogEntry[]>(Array.Empty <IRaftLogEntry>(), null, token));
}
private async Task <bool> DoHeartbeats(IAuditTrail <ILogEntry> transactionLog)
{
ICollection <Task <Result <MemberHealthStatus> > > tasks = new LinkedList <Task <Result <MemberHealthStatus> > >();
//send heartbeat in parallel
var commitIndex = transactionLog.GetLastIndex(true);
foreach (var member in stateMachine.Members)
{
if (member.IsRemote)
{
tasks.Add(AppendEntriesAsync(member, commitIndex, currentTerm, transactionLog, stateMachine.Logger, timerCancellation.Token)
.ContinueWith(HealthStatusContinuation, default, TaskContinuationOptions.ExecuteSynchronously,
private async Task <bool> DoHeartbeats(IAuditTrail <IRaftLogEntry> auditTrail, CancellationToken token)
{
var timeStamp = Timestamp.Current;
var tasks = new LinkedList <ValueTask <Result <bool> > >();
long commitIndex = auditTrail.GetLastIndex(true), currentIndex = auditTrail.GetLastIndex(false);
var term = currentTerm;
//send heartbeat in parallel
foreach (var member in stateMachine.Members)
{
if (member.IsRemote)
{
long precedingIndex = Math.Max(0, member.NextIndex - 1), precedingTerm = await auditTrail.GetTermAsync(precedingIndex, token);
tasks.AddLast(new Replicator(member, commitIndex, currentIndex, term, precedingIndex, precedingTerm, stateMachine.Logger, token).Start(auditTrail));
}
}
var quorum = 1; //because we know that the entry is replicated in this node
var commitQuorum = 1;
for (var task = tasks.First; task != null; task.Value = default, task = task.Next)
{
try
{
var result = await task.Value.ConfigureAwait(false);
term = Math.Max(term, result.Term);
quorum += 1;
commitQuorum += result.Value ? 1 : -1;
}
catch (MemberUnavailableException)
{
quorum -= 1;
commitQuorum -= 1;
}
}
internal static async Task <bool> ContainsAsync(this IAuditTrail <IRaftLogEntry> auditTrail, long index, long term, CancellationToken token) => index <= auditTrail.GetLastIndex(false) && term == await auditTrail.GetTermAsync(index, token).ConfigureAwait(false);
internal static async Task <bool> IsUpToDateAsync(this IAuditTrail <IRaftLogEntry> auditTrail, long index, long term, CancellationToken token)
{
var localIndex = auditTrail.GetLastIndex(false);
return(index >= localIndex && term >= await auditTrail.GetTermAsync(localIndex, token).ConfigureAwait(false));
}
private async Task <bool> DoHeartbeats(IAuditTrail <IRaftLogEntry> auditTrail, CancellationToken token)
{
var timeStamp = Timestamp.Current;
var tasks = new AsyncResultSet();
long commitIndex = auditTrail.GetLastIndex(true),
currentIndex = auditTrail.GetLastIndex(false),
term = currentTerm,
minPrecedingIndex = 0L;
// send heartbeat in parallel
foreach (var member in stateMachine.Members)
{
if (member.IsRemote)
{
long precedingIndex = Math.Max(0, member.NextIndex - 1), precedingTerm;
minPrecedingIndex = Math.Min(minPrecedingIndex, precedingIndex);
// try to get term from the cache to avoid touching audit trail for each member
if (!precedingTermCache.TryGetValue(precedingIndex, out precedingTerm))
{
precedingTermCache.Add(precedingIndex, precedingTerm = await auditTrail.GetTermAsync(precedingIndex, token).ConfigureAwait(false));
}
tasks.AddLast(new Replicator(member, commitIndex, currentIndex, term, precedingIndex, precedingTerm, stateMachine.Logger, token).Start(auditTrail));
}
}
// clear cache
if (precedingTermCache.Count > MaxTermCacheSize)
{
precedingTermCache.Clear();
}
else
{
precedingTermCache.RemoveHead(minPrecedingIndex);
}
int quorum = 1, commitQuorum = 1; // because we know that the entry is replicated in this node
#if NETSTANDARD2_1
for (var task = tasks.First; task is not null; task.Value = default, task = task.Next)
#else
for (var task = tasks.First; task is not null; task.ValueRef = default, task = task.Next)
#endif
{
try
{
#if NETSTANDARD2_1
var result = await task.Value.ConfigureAwait(false);
#else
var result = await task.ValueRef.ConfigureAwait(false);
#endif
term = Math.Max(term, result.Term);
quorum += 1;
commitQuorum += result.Value ? 1 : -1;
}
catch (MemberUnavailableException)
{
quorum -= 1;
commitQuorum -= 1;
}
catch (OperationCanceledException)
{
// leading was canceled
tasks.Clear();
Metrics?.ReportBroadcastTime(timeStamp.Elapsed);
return(false);
}
catch (Exception e)
{
stateMachine.Logger.LogError(e, ExceptionMessages.UnexpectedError);
}
}
tasks.Clear();
Metrics?.ReportBroadcastTime(timeStamp.Elapsed);
// majority of nodes accept entries with a least one entry from the current term
if (commitQuorum > 0)
{
var count = await auditTrail.CommitAsync(currentIndex, token).ConfigureAwait(false); // commit all entries started from first uncommitted index to the end
stateMachine.Logger.CommitSuccessful(commitIndex + 1, count);
goto check_term;
}
stateMachine.Logger.CommitFailed(quorum, commitIndex);
// majority of nodes replicated, continue leading if current term is not changed
if (quorum <= 0 && !allowPartitioning)
{
goto stop_leading;
}
check_term:
if (term <= currentTerm)
{
return(true);
}
// it is partitioned network with absolute majority, not possible to have more than one leader
stop_leading:
stateMachine.MoveToFollowerState(false, term);
return(false);
}
private async Task <bool> DoHeartbeats(IAuditTrail <IRaftLogEntry> auditTrail, CancellationToken token)
{
var timeStamp = Timestamp.Current;
var tasks = new LinkedList <ValueTask <Result <bool> > >();
long commitIndex = auditTrail.GetLastIndex(true), currentIndex = auditTrail.GetLastIndex(false);
var term = currentTerm;
// send heartbeat in parallel
foreach (var member in stateMachine.Members)
{
if (member.IsRemote)
{
long precedingIndex = Math.Max(0, member.NextIndex - 1), precedingTerm = await auditTrail.GetTermAsync(precedingIndex, token).ConfigureAwait(false);
tasks.AddLast(new Replicator(member, commitIndex, currentIndex, term, precedingIndex, precedingTerm, stateMachine.Logger, token).Start(auditTrail));
}
}
var quorum = 1; // because we know that the entry is replicated in this node
var commitQuorum = 1;
for (var task = tasks.First; task != null; task.Value = default, task = task.Next)
{
try
{
var result = await task.Value.ConfigureAwait(false);
term = Math.Max(term, result.Term);
quorum += 1;
commitQuorum += result.Value ? 1 : -1;
}
catch (MemberUnavailableException)
{
quorum -= 1;
commitQuorum -= 1;
}
catch (OperationCanceledException)
{
// leading was canceled
tasks.Clear();
Metrics?.ReportBroadcastTime(timeStamp.Elapsed);
return(false);
}
catch (Exception e)
{
stateMachine.Logger.LogError(e, ExceptionMessages.UnexpectedError);
}
}
tasks.Clear();
Metrics?.ReportBroadcastTime(timeStamp.Elapsed);
// majority of nodes accept entries with a least one entry from current term
if (commitQuorum > 0)
{
var count = await auditTrail.CommitAsync(currentIndex, token).ConfigureAwait(false); // commit all entries started from first uncommitted index to the end
stateMachine.Logger.CommitSuccessful(commitIndex + 1, count);
return(CheckTerm(term));
}
stateMachine.Logger.CommitFailed(quorum, commitIndex);
// majority of nodes replicated, continue leading if current term is not changed
if (quorum > 0 || allowPartitioning)
{
return(CheckTerm(term));
}
// it is partitioned network with absolute majority, not possible to have more than one leader
stateMachine.MoveToFollowerState(false, term);
return(false);
}