private Func <CancellationToken, Task> PeerAppendLog(Peer peer) { return((CancellationToken c) => { long nextIndex; long matchIndex; var hasMatch = _volatileLeaderState.TryGetMatchIndex(peer.Id, out matchIndex); var hasNext = _volatileLeaderState.TryGetNextIndex(peer.Id, out nextIndex); var myLastIndex = _logPersister.LastIndex; if (!hasMatch) { TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Could not find peer with id {peer.Id} and address {peer.Address} in matchIndex dic."); return Task.CompletedTask; } if (!hasNext) { TheTrace.TraceWarning($"[{_meAsAPeer.ShortName}] Could not find peer with id {peer.Id} and address {peer.Address} in nextIndex dic."); return Task.CompletedTask; } if (nextIndex > myLastIndex) { TheTrace.TraceVerbose($"[{_meAsAPeer.ShortName}] PeerAppendLog - Nothing to do for peer {peer.ShortName} since myIndex is {myLastIndex} and nextIndex is {nextIndex}."); return Task.CompletedTask; // nothing to do } var count = (int)Math.Min(_settings.MaxNumberLogEntriesToAskToBeAppended, myLastIndex + 1 - nextIndex); var proxy = _peerManager.GetProxy(peer.Address); var retry = TheTrace.LogPolicy(_meAsAPeer.ShortName).WaitAndRetryAsync(2, (i) => TimeSpan.FromMilliseconds(20)); var policy = Policy.TimeoutAsync(_settings.CandidacyTimeout).WrapAsync(retry); // TODO: create its own timeout if (nextIndex >= _logPersister.LogOffset) { TheTrace.TraceVerbose($"[{_meAsAPeer.ShortName}] Intending to do SendLog for peer {peer.Address} with nextIndex {nextIndex} and count {count}."); return SendLogs(proxy, policy, peer, nextIndex, matchIndex, count); } else { TheTrace.TraceVerbose($"[{_meAsAPeer.ShortName}] Intending to do SendSnapshot for peer {peer.Address}."); return SendSnapshot(proxy, policy, peer, nextIndex, matchIndex); } }); }
/// <inheritdocs/> public void Append(LogEntry[] entries, long?startingOffset = null) { if (startingOffset.HasValue && startingOffset.Value != LastIndex + 1) { throw new InvalidOperationException($"Starting index is {startingOffset} but LastIndex is {LastIndex}"); } if (entries.Length == 0) { throw new InvalidOperationException("Entries is empty."); } TheTrace.TraceVerbose($"[{Name}] LmdbPersister.Append - Before Lock"); lock (_lock) { startingOffset = LastIndex + 1; TheTrace.TraceVerbose($"[{Name}] LmdbPersister.Append - Inside Lock. Entries: {entries.Length}. startingOffset: {startingOffset}"); var indices = Enumerable.Range(0, entries.Length).Select(x => x + startingOffset); TheTrace.TraceVerbose($"[{Name}] LmdbPersister.Append - Before tx"); using (var tx = _env.BeginTransaction()) { TheTrace.TraceVerbose($"[{Name}] LmdbPersister.Append - After tx"); foreach (var e in entries.Zip(indices, (l, i) => (i, l)).Select(x => new Bufferable(x.l.Body).PrefixWithIndexAndTerm(x.i.Value, x.l.Term))) { TheTrace.TraceVerbose($"[{Name}] LmdbPersister.Append - Before put"); tx.Put(_logDb, LogKey, e, TransactionPutOptions.AppendDuplicateData); TheTrace.TraceVerbose($"[{Name}] LmdbPersister.Append - After Put"); } tx.Commit(); } TheTrace.TraceVerbose($"[{Name}] LmdbPersister.Append - Before LoadLastTermAndIndex"); LoadLastTermAndIndex(); TheTrace.TraceVerbose($"[{Name}] LmdbPersister.Append - After LoadLastTermAndIndex"); if (LastIndex != startingOffset + entries.Length - 1) { throw new InvalidOperationException($"THIS IS BAD!! expected {startingOffset} + {entries.Length} but found {LastIndex}."); } if (LastEntryTerm != entries.Last().Term) { throw new InvalidOperationException($"THIS IS BAD!! expected last term {entries.Last().Term} but found {LastEntryTerm}."); } } }
private Task ProcessCommandsQueue(CancellationToken c) { if (_isSnapshotting) { return(Task.CompletedTask); } if (Role == Role.Leader) { TheTrace.TraceVerbose($"[{_meAsAPeer.ShortName}] ProcessCommandsQueue with {_commands.Count} items."); } var entries = new List <LogEntry>(); StateMachineCommandRequest command; TheTrace.TraceVerbose($"[{_meAsAPeer.ShortName}] ProcessCommandsQueue - Before with trying to take."); while (_commands.TryTake(out command, 10)) // we can set up a maximum but really we should accept all { TheTrace.TraceVerbose($"[{_meAsAPeer.ShortName}] ProcessCommandsQueue - Got an item."); entries.Add(new LogEntry() { Body = command.Command, Term = State.CurrentTerm }); } if (entries.Any()) { TheTrace.TraceVerbose($"[{_meAsAPeer.ShortName}] ProcessCommandsQueue - Before applying {entries.Count} entries."); _logPersister.Append(entries.ToArray()); TheTrace.TraceVerbose($"[{_meAsAPeer.ShortName}] ProcessCommandsQueue - His Majesty appended {entries.Count} entries."); } return(Task.CompletedTask); }