Exemplo n.º 1
0
        private async Task HeartBeatSend(CancellationToken c)
        {
            if (_role != Role.Leader)
            {
                return;
            }

            if (_lastHeartbeatSent.Since() < _settings.ElectionTimeoutMin.Multiply(0.2))
            {
                return;
            }

            var currentTerm = State.CurrentTerm; // create a var. Could change during the method leading to confusing logs.

            var req = new AppendEntriesRequest()
            {
                CurrentTerm       = currentTerm,
                Entries           = new byte[0][],
                LeaderCommitIndex = _volatileState.CommitIndex,
                LeaderId          = State.Id,
                PreviousLogIndex  = long.MaxValue,
                PreviousLogTerm   = long.MaxValue
            };

            var peers   = _peerManager.GetPeers().ToArray();
            var proxies = peers.Select(x => _peerManager.GetProxy(x.Address));
            var retry   = TheTrace.LogPolicy(_meAsAPeer.ShortName).RetryForeverAsync();
            var policy  = Policy.TimeoutAsync(_settings.ElectionTimeoutMin.Multiply(0.2)).WrapAsync(retry);
            var all     = await Task.WhenAll(proxies.Select(p => policy.ExecuteAndCaptureAsync(() => p.AppendEntriesAsync(req))));

            var maxTerm = currentTerm;

            foreach (var r in all)
            {
                if (r.Outcome == OutcomeType.Successful)
                {
                    if (!r.Result.IsSuccess)
                    {
                        TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Got this reason for unsuccessful AppendEntriesAsync from a peer: {r.Result.Reason}");
                    }

                    // NOTE: We do NOT change leadership if they send higher term, since they could be candidates whom will not become leaders
                    // we actually do not need to do anything with the result other than logging it
                    if (r.Result.CurrentTerm > maxTerm)
                    {
                        maxTerm = r.Result.CurrentTerm;
                    }
                }
            }

            if (maxTerm > State.CurrentTerm)
            {
                TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Revolution brewing. Terms as high as {maxTerm} (vs my {currentTerm}) were seen.");
            }

            _lastHeartbeatSent.Set();
        }
Exemplo n.º 2
0
        /// <inheritdoc />
        public Task <AppendEntriesResponse> AppendEntriesAsync(AppendEntriesRequest request)
        {
            _lastHeartbeat.Set();

            if (request.Entries != null && request.Entries.Length > 0)
            {
                TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Received the dough {request.Entries.Length} for position after {request.PreviousLogIndex}");
            }


            string message = null;

            lock (State)
            {
                if (request.CurrentTerm > State.CurrentTerm)
                {
                    BecomeFollower(request.CurrentTerm);
                }
            }

            // Reply false if term < currentTerm (§5.1)
            if (request.CurrentTerm < State.CurrentTerm)
            {
                message = $"[{_meAsAPeer.ShortName}] Leader's term is behind ({request.CurrentTerm} vs {State.CurrentTerm}).";
                TheTrace.TraceWarning(message);
                return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, false, ReasonType.TermInconsistency, message)));
            }

            if (request.Entries == null || request.Entries.Length == 0) // it is a heartbeat, set the leader address
            {
                _leaderAddress = _peerManager.GetPeers().Where(x => x.Id == request.LeaderId).FirstOrDefault()?.Address;
                return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, true)));
            }

            // chaos only when has entries
            _chaos.WreakHavoc();

            if (request.PreviousLogIndex > _logPersister.LastIndex)
            {
                message = $"[{_meAsAPeer.ShortName}] Position for last log entry is {_logPersister.LastIndex} but got entries starting at {request.PreviousLogIndex}";
                TheTrace.TraceWarning(message);
                return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, false, ReasonType.LogInconsistency, message)));
            }

            if (request.PreviousLogIndex < _logPersister.LastIndex)
            {
                TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Position for PreviousLogIndex {request.PreviousLogIndex} but my LastIndex {_logPersister.LastIndex}");

                // Reply false if log doesn’t contain an entry at prevLogIndex whose term matches prevLogTerm(§5.3)
                var entry = _logPersister.GetEntries(request.PreviousLogIndex, 1).First();
                if (entry.Term != request.CurrentTerm)
                {
                    message = $"[{_meAsAPeer.ShortName}] Position at {request.PreviousLogIndex} has term {entry.Term} but according to leader {request.LeaderId} it must be {request.PreviousLogTerm}";
                    TheTrace.TraceWarning(message);
                    return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, false, ReasonType.LogInconsistency, message)));
                }

                // If an existing entry conflicts with a new one(same index but different terms), delete the existing entry and all that follow it(§5.3)
                _logPersister.DeleteEntries(request.PreviousLogIndex + 1);
                TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Stripping the log from index {request.PreviousLogIndex + 1}. Last index was {_logPersister.LastIndex}");
            }

            var entries = request.Entries.Select(x => new LogEntry()
            {
                Body = x,
                Term = request.CurrentTerm
            }).ToArray();

            // Append any new entries not already in the log
            TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Current last index is {_logPersister.LastIndex}. About to append {entries.Length} entries at {request.PreviousLogIndex + 1}");
            _logPersister.Append(entries, request.PreviousLogIndex + 1);

            //If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry)
            if (request.LeaderCommitIndex > _volatileState.CommitIndex)
            {
                _volatileState.CommitIndex = Math.Min(request.LeaderCommitIndex, _logPersister.LastIndex);
            }

            message = $"[{_meAsAPeer.ShortName}] Appended {request.Entries.Length} entries at position {request.PreviousLogIndex + 1}";
            TheTrace.TraceInformation(message);
            return(Task.FromResult(new AppendEntriesResponse(State.CurrentTerm, true, ReasonType.None, message)));
        }
Exemplo n.º 3
0
        private async Task SendLogs(
            IRaftServer proxy,
            AsyncPolicy policy,
            Peer peer,
            long nextIndex,
            long matchIndex,
            int count)
        {
            var previousIndexTerm = -1L;

            if (nextIndex > 0)
            {
                if (nextIndex > _logPersister.LogOffset)
                {
                    previousIndexTerm = _logPersister.GetEntries(nextIndex - 1, 1).First().Term;
                }
                else
                {
                    Snapshot ss;
                    if (_snapshotOperator.TryGetLastSnapshot(out ss))
                    {
                        previousIndexTerm = ss.LastIncludedTerm;
                    }
                }
            }

            var request = new AppendEntriesRequest()
            {
                CurrentTerm       = State.CurrentTerm,
                Entries           = _logPersister.GetEntries(nextIndex, count).Select(x => x.Body).ToArray(),
                LeaderCommitIndex = _volatileState.CommitIndex,
                LeaderId          = State.Id,
                PreviousLogIndex  = nextIndex - 1,
                PreviousLogTerm   = previousIndexTerm
            };

            var result = await policy.ExecuteAndCaptureAsync(() => proxy.AppendEntriesAsync(request));

            if (result.Outcome == OutcomeType.Successful)
            {
                if (result.Result.IsSuccess)
                {
                    // If successful: update nextIndex and matchIndex for follower(§5.3)"
                    _volatileLeaderState.SetMatchIndex(peer.Id, nextIndex + count - 1);
                    _volatileLeaderState.SetNextIndex(peer.Id, nextIndex + count);
                    TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Successfully transferred {count} entries from index {nextIndex} to peer {peer.Address} - Next Index is {_volatileLeaderState.NextIndices[peer.Id]}");
                    UpdateCommitIndex();
                }
                else
                {
                    // log reason only
                    TheTrace.TraceWarning($"AppendEntries for start index {nextIndex} and count {count} for peer {peer.Address} with address {peer.Address} in term {State.CurrentTerm} failed with reason type {result.Result.ReasonType} and this reason: {result.Result.Reason}");

                    if (result.Result.ReasonType == ReasonType.LogInconsistency)
                    {
                        var diff = nextIndex - (matchIndex + 1);
                        nextIndex = diff > _settings.MaxNumberOfDecrementForLogsThatAreBehind ?
                                    nextIndex - _settings.MaxNumberOfDecrementForLogsThatAreBehind :
                                    nextIndex - diff;

                        _volatileLeaderState.SetNextIndex(peer.Id, nextIndex);
                        TheTrace.TraceInformation($"[{_meAsAPeer.ShortName}] Updated (decremented) next index for peer {peer.Address} to {nextIndex}");
                    }
                }
            }
            else
            {
                // NUNCA!!
                // not interested in network, etc errors, they get logged in the policy
            }
        }