private void DropUselessPeers()
{
if (DateTime.UtcNow - _lastUselessDrop < TimeSpan.FromSeconds(30))
{
// give some time to monitoring nodes
return;
}
_lastUselessDrop = DateTime.UtcNow;
if ((decimal)PeerCount / PeerMaxCount > 0.5m)
{
long ourNumber = _blockTree.BestSuggested?.Number ?? 0L;
UInt256 ourDifficulty = _blockTree.BestSuggested?.TotalDifficulty ?? UInt256.Zero;
foreach (PeerInfo peerInfo in AllPeers)
{
if (peerInfo.HeadNumber > ourNumber)
{
// as long as we are behind we can use the stuck peers
continue;
}
if (peerInfo.HeadNumber == 0)
{
peerInfo.SyncPeer.Disconnect(DisconnectReason.UselessPeer, "PEER REVIEW / HEAD 0");
}
else if (peerInfo.HeadNumber == 1920000) // mainnet, stuck Geth nodes
{
peerInfo.SyncPeer.Disconnect(DisconnectReason.UselessPeer, "PEER REVIEW / 1920000");
}
else if (peerInfo.HeadNumber == 7280022) // mainnet, stuck Geth nodes
{
peerInfo.SyncPeer.Disconnect(DisconnectReason.UselessPeer, "PEER REVIEW / 7280022");
}
else if (peerInfo.HeadNumber > ourNumber + 1024L && peerInfo.TotalDifficulty < ourDifficulty)
{
// probably classic nodes tht remain connected after we went pass the DAO
// worth to find a better way to discard them at the right time
peerInfo.SyncPeer.Disconnect(DisconnectReason.UselessPeer, "STRAY PEER");
}
}
}
if (PeerCount == PeerMaxCount)
{
int worstLatency = 0;
PeerInfo worstPeer = null;
foreach (PeerInfo peerInfo in AllPeers)
{
long latency = _stats.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageLatency(NodeLatencyStatType.BlockHeaders) ?? 100000;
if (latency > worstLatency)
{
worstPeer = peerInfo;
}
}
worstPeer?.SyncPeer.Disconnect(DisconnectReason.TooManyPeers, "PEER REVIEW / LATENCY");
}
}
public void Write()
{
TimeSpan timeSinceLastEntry;
bool initializedCountChanged;
lock (_writeLock)
{
timeSinceLastEntry = DateTime.UtcNow - _timeOfTheLastFullPeerListLogEntry;
int initializedPeerCount = _peerPool.AllPeers.Count(p => p.IsInitialized);
initializedCountChanged = initializedPeerCount != _currentInitializedPeerCount;
_currentInitializedPeerCount = initializedPeerCount;
}
if (timeSinceLastEntry > _fullPeerListInterval)
{
_timeOfTheLastFullPeerListLogEntry = DateTime.UtcNow;
if (_logger.IsInfo)
{
_logger.Info($"Sync peers {_currentInitializedPeerCount}({_peerPool.PeerCount})/{_peerPool.PeerMaxCount}");
}
foreach (PeerInfo peerInfo in _peerPool.AllPeers)
{
string prefix = peerInfo.IsAllocated ? " * " : " ";
if (_logger.IsInfo)
{
_logger.Info($"{prefix}{peerInfo}[{_stats.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageLatency(NodeLatencyStatType.BlockHeaders) ?? 100000}]");
}
}
}
else if (initializedCountChanged)
{
if (_logger.IsInfo)
{
_logger.Info($"Sync peers {_currentInitializedPeerCount}({_peerPool.PeerCount})/{_peerPool.PeerMaxCount}");
}
foreach (PeerInfo peerInfo in _peerPool.AllPeers)
{
if (peerInfo.IsAllocated)
{
if (_logger.IsInfo)
{
_logger.Info($" * {peerInfo}[{_stats.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageLatency(NodeLatencyStatType.BlockHeaders) ?? 100000}]");
}
}
}
}
else if (_currentInitializedPeerCount == 0)
{
if (_logger.IsInfo)
{
_logger.Info($"Sync peers 0({_peerPool.PeerCount})/{_peerPool.PeerMaxCount}, searching for peers to sync with...");
}
}
}
private long?GetSpeed(INodeStatsManager nodeStatsManager, PeerInfo peerInfo)
{
long?headersSpeed = nodeStatsManager.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageTransferSpeed(TransferSpeedType.Headers);
long?bodiesSpeed = nodeStatsManager.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageTransferSpeed(TransferSpeedType.Bodies);
if (headersSpeed == null && bodiesSpeed == null)
{
return(null);
}
return((headersSpeed ?? 0) + (bodiesSpeed ?? 0));
}
public void Should_allocate_by_speed_post_merge()
{
ulong[] totalDifficulties = { 1, 3, 2 };
int[] averageSpeed = { 5, 8, 10 };
PublicKey[] publicKeys = { TestItem.PublicKeyA, TestItem.PublicKeyB, TestItem.PublicKeyC };
PeerInfo[] peers = new PeerInfo[3];
INodeStatsManager _nodeStatsManager = Substitute.For <INodeStatsManager>();
for (int i = 0; i < 3; i++)
{
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.IsInitialized.Returns(true);
Node node = new Node(publicKeys[i], "192.168.1.18", i);
syncPeer.Node.Returns(node);
syncPeer.TotalDifficulty.Returns(new UInt256(totalDifficulties[i]));
peers[i] = new PeerInfo(syncPeer);
peers[i].HeadNumber.Returns(1);
INodeStats nodeStats = Substitute.For <INodeStats>();
nodeStats.GetAverageTransferSpeed(Arg.Any <TransferSpeedType>()).Returns(averageSpeed[i]);
_nodeStatsManager.GetOrAdd(peers[i].SyncPeer.Node).Returns(nodeStats);
}
IPoSSwitcher poSSwitcher = Substitute.For <IPoSSwitcher>();
poSSwitcher.TerminalTotalDifficulty.Returns(new UInt256(1));
poSSwitcher.HasEverReachedTerminalBlock().Returns(true);
IBeaconPivot beaconPivot = Substitute.For <IBeaconPivot>();
IPeerAllocationStrategy mergePeerAllocationStrategy =
(new MergeBlocksSyncPeerAllocationStrategyFactory(poSSwitcher, beaconPivot, Substitute.For <ILogManager>())).Create(new BlocksRequest());
IBlockTree _blockTree = Substitute.For <IBlockTree>();
PeerInfo? info = mergePeerAllocationStrategy.Allocate(null, peers, _nodeStatsManager, _blockTree);
Assert.AreEqual(info, peers[2]); // peer with highest highest speed
}
private void LoadPeersFromDb(List <Peer> peers)
{
if (!_networkConfig.IsPeersPersistenceOn)
{
return;
}
var networkNodes = _peerStorage.GetPersistedNodes();
if (_logger.IsInfo)
{
_logger.Info($"Initializing persisted peers: {networkNodes.Length}.");
}
foreach (var persistedPeer in networkNodes)
{
var node = new Node(persistedPeer.NodeId, persistedPeer.Host, persistedPeer.Port);
var nodeStats = _stats.GetOrAdd(node);
nodeStats.CurrentPersistedNodeReputation = persistedPeer.Reputation;
peers.Add(new Peer(node));
if (_logger.IsTrace)
{
_logger.Trace($"Adding a new peer candidate {node}");
}
}
}
private void SetupSpeedStats(PublicKey publicKey, int transferSpeed)
{
Node node = new Node(publicKey, "127.0.0.1", 30303);
NodeStatsLight stats = new NodeStatsLight(node, new StatsConfig());
stats.AddTransferSpeedCaptureEvent(TransferSpeedType.Headers, transferSpeed);
_stats.GetOrAdd(Arg.Is <Node>(n => n.Id == publicKey)).Returns(stats);
}
public Context VerifyEthInitialized()
{
INodeStats stats = _nodeStatsManager.GetOrAdd(_currentSession.Node);
Assert.AreEqual(1, stats.EthNodeDetails.ChainId);
Assert.AreEqual(_blockTree.Genesis.Hash, stats.EthNodeDetails.GenesisHash);
Assert.AreEqual(63, stats.EthNodeDetails.ProtocolVersion);
Assert.AreEqual(BigInteger.One, stats.EthNodeDetails.TotalDifficulty);
return this;
}
public PeerInfo Allocate(PeerInfo currentPeer, IEnumerable <PeerInfo> peers, INodeStatsManager nodeStatsManager, IBlockTree blockTree)
{
long nullSpeed = _priority ? -1 : long.MaxValue;
long currentSpeed = currentPeer == null ? nullSpeed : nodeStatsManager.GetOrAdd(currentPeer.SyncPeer.Node).GetAverageTransferSpeed(_speedType) ?? nullSpeed;
(PeerInfo Info, long TransferSpeed)bestPeer = (currentPeer, currentSpeed);
foreach (PeerInfo info in peers)
{
(this as IPeerAllocationStrategy).CheckAsyncState(info);
long averageTransferSpeed = nodeStatsManager.GetOrAdd(info.SyncPeer.Node).GetAverageTransferSpeed(_speedType) ?? 0;
if (_priority ? averageTransferSpeed > bestPeer.TransferSpeed : averageTransferSpeed < bestPeer.TransferSpeed)
{
bestPeer = (info, averageTransferSpeed);
}
}
return(bestPeer.Info);
}
private void SetupLatencyStats(PublicKey publicKey, int milliseconds)
{
Node node = new Node(publicKey, "127.0.0.1", 30303);
NodeStatsLight stats = new NodeStatsLight(node, new StatsConfig());
stats.AddLatencyCaptureEvent(NodeLatencyStatType.BlockHeaders, milliseconds);
stats.AddLatencyCaptureEvent(NodeLatencyStatType.BlockBodies, milliseconds);
stats.AddLatencyCaptureEvent(NodeLatencyStatType.P2PPingPong, milliseconds);
_stats.GetOrAdd(Arg.Is <Node>(n => n.Id == publicKey)).Returns(stats);
}
public INodeLifecycleManager CreateNodeLifecycleManager(Node node)
{
if (DiscoveryManager == null)
{
throw new Exception($"{nameof(DiscoveryManager)} has to be set");
}
return(new NodeLifecycleManager(
node,
DiscoveryManager,
_nodeTable,
_evictionManager,
_nodeStatsManager.GetOrAdd(node),
_selfNodeRecord,
_discoveryConfig,
_timestamper,
_logger));
}
private void Display(List <PeerInfo> peers)
{
if (_currentInitializedPeerCount == 0)
{
if (_logger.IsInfo)
{
_logger.Info($"Sync peers 0({_peerPool.PeerCount})/{_peerPool.PeerMaxCount}, searching for peers to sync with...");
}
return;
}
foreach (PeerInfo peerInfo in peers.Where(pi => pi != null).OrderBy(p => p.SyncPeer?.Node?.Host))
{
string prefix = peerInfo.IsAllocated ? " * " : " ";
if (_logger.IsInfo)
{
_logger.Info($"{prefix}{peerInfo}[{_stats.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageLatency(NodeLatencyStatType.BlockHeaders) ?? 100000}]");
}
}
}
private void LoadPeersFromDb(List <Peer> peers)
{
if (!_networkConfig.IsPeersPersistenceOn)
{
return;
}
var networkNodes = _peerStorage.GetPersistedNodes();
if (_logger.IsInfo)
{
_logger.Info($"Initializing persisted peers: {networkNodes.Length}.");
}
foreach (var persistedPeer in networkNodes)
{
Node node;
try
{
node = new Node(persistedPeer.NodeId, persistedPeer.Host, persistedPeer.Port);
}
catch (Exception)
{
if (_logger.IsDebug)
{
_logger.Error($"ERROR/DEBUG peer could not be loaded for {persistedPeer.NodeId}@{persistedPeer.Host}:{persistedPeer.Port}");
}
continue;
}
var nodeStats = _stats.GetOrAdd(node);
nodeStats.CurrentPersistedNodeReputation = persistedPeer.Reputation;
peers.Add(new Peer(node));
if (_logger.IsTrace)
{
_logger.Trace($"Adding a new peer candidate {node}");
}
}
}
private async Task RunPeerUpdateLoop()
{
int loopCount = 0;
while (true)
{
try
{
if (loopCount++ % 100 == 0)
{
if (_logger.IsTrace)
{
_logger.Trace($"Running peer update loop {loopCount - 1} - active: {_activePeers.Count} | candidates : {_candidatePeers.Count}");
}
}
try
{
CleanupCandidatePeers();
}
catch (Exception e)
{
if (_logger.IsDebug)
{
_logger.Error("Candidate peers clanup failed", e);
}
}
_peerUpdateRequested.Wait(_cancellationTokenSource.Token);
_peerUpdateRequested.Reset();
if (!_isStarted)
{
continue;
}
int availableActiveCount = _networkConfig.ActivePeersMaxCount - _activePeers.Count;
if (availableActiveCount == 0)
{
continue;
}
Interlocked.Exchange(ref tryCount, 0);
Interlocked.Exchange(ref newActiveNodes, 0);
Interlocked.Exchange(ref failedInitialConnect, 0);
Interlocked.Exchange(ref connectionRounds, 0);
SelectAndRankCandidates();
IReadOnlyCollection <Peer> remainingCandidates = _currentSelection.Candidates;
if (!remainingCandidates.Any())
{
continue;
}
if (_cancellationTokenSource.IsCancellationRequested)
{
break;
}
while (true)
{
if (_cancellationTokenSource.IsCancellationRequested)
{
break;
}
availableActiveCount = _networkConfig.ActivePeersMaxCount - _activePeers.Count;
int nodesToTry = Math.Min(remainingCandidates.Count, availableActiveCount);
if (nodesToTry == 0)
{
break;
}
IEnumerable <Peer> candidatesToTry = remainingCandidates.Take(nodesToTry);
remainingCandidates = remainingCandidates.Skip(nodesToTry).ToList();
var workerBlock = new ActionBlock <Peer>(
SetupPeerConnection,
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
CancellationToken = _cancellationTokenSource.Token
});
foreach (var candidateToTry in candidatesToTry)
{
await workerBlock.SendAsync(candidateToTry);
}
workerBlock.Complete();
// Wait for all messages to propagate through the network.
workerBlock.Completion.Wait();
Interlocked.Increment(ref connectionRounds);
}
if (_logger.IsDebug)
{
int activePeersCount = _activePeers.Count;
if (activePeersCount != _prevActivePeersCount)
{
string countersLog = string.Join(", ", _currentSelection.Counters.Select(x => $"{x.Key.ToString()}: {x.Value}"));
_logger.Debug($"RunPeerUpdate | {countersLog}, Incompatible: {GetIncompatibleDesc(_currentSelection.Incompatible)}, EligibleCandidates: {_currentSelection.Candidates.Count()}, " +
$"Tried: {tryCount}, Rounds: {connectionRounds}, Failed initial connect: {failedInitialConnect}, Established initial connect: {newActiveNodes}, " +
$"Current candidate peers: {_candidatePeers.Count}, Current active peers: {_activePeers.Count} " +
$"[InOut: {_activePeers.Count(x => x.Value.OutSession != null && x.Value.InSession != null)} | " +
$"[Out: {_activePeers.Count(x => x.Value.OutSession != null)} | " +
$"In: {_activePeers.Count(x => x.Value.InSession != null)}]");
}
_prevActivePeersCount = activePeersCount;
}
if (_logger.IsTrace)
{
if (_logCounter % 5 == 0)
{
string nl = Environment.NewLine;
_logger.Trace($"{nl}{nl}All active peers: {nl} {string.Join(nl, _activePeers.Values.Select(x => $"{x.Node:s} | P2P: {_stats.GetOrAdd(x.Node).DidEventHappen(NodeStatsEventType.P2PInitialized)} | Eth62: {_stats.GetOrAdd(x.Node).DidEventHappen(NodeStatsEventType.Eth62Initialized)} | {_stats.GetOrAdd(x.Node).P2PNodeDetails?.ClientId} | {_stats.GetOrAdd(x.Node).ToString()}"))} {nl}{nl}");
}
_logCounter++;
}
if (_activePeers.Count != _networkConfig.ActivePeersMaxCount)
{
_peerUpdateRequested.Set();
}
}
catch (Exception e) when(!(e is OperationCanceledException))
{
if (_logger.IsError)
{
_logger.Error($"Peer update loop failure {e}");
}
break;
}
}
if (_logger.IsWarn)
{
_logger.Warn("Exiting peer update loop");
}
await Task.CompletedTask;
}
public INodeLifecycleManager CreateNodeLifecycleManager(Node node)
{
if (DiscoveryManager == null)
{
throw new Exception($"{nameof(DiscoveryManager)} has to be set");
}
return(new NodeLifecycleManager(node, DiscoveryManager, _nodeTable, _logger, _networkConfig, _discoveryMessageFactory, _evictionManager, _nodeStatsManager.GetOrAdd(node)));
}
private void AddPeerInfo(PeerInfo peerInfo)
{
_stringBuilder.Append($" {peerInfo}[{_stats.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageTransferSpeed(TransferSpeedType.Latency) ?? 0}|{_stats.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageTransferSpeed(TransferSpeedType.Headers) ?? 0}|{_stats.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageTransferSpeed(TransferSpeedType.Bodies) ?? 0}|{_stats.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageTransferSpeed(TransferSpeedType.Receipts) ?? 0}|{_stats.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageTransferSpeed(TransferSpeedType.NodeData) ?? 0}]");
}
private void DropUselessPeers()
{
if (DateTime.UtcNow - _lastUselessDrop < TimeSpan.FromSeconds(30))
{
// give some time to monitoring nodes
return;
}
if (_logger.IsTrace)
{
_logger.Trace($"Reviewing {PeerCount} peer usefulness");
}
int peersDropped = 0;
_lastUselessDrop = DateTime.UtcNow;
long ourNumber = _blockTree.BestSuggestedHeader?.Number ?? 0L;
UInt256 ourDifficulty = _blockTree.BestSuggestedHeader?.TotalDifficulty ?? UInt256.Zero;
foreach (PeerInfo peerInfo in AllPeers)
{
if (peerInfo.HeadNumber > ourNumber)
{
// as long as we are behind we can use the stuck peers
continue;
}
if (peerInfo.HeadNumber == 0 &&
ourNumber != 0 &&
!peerInfo.SyncPeer.ClientId.Contains("Nethermind"))
{
peersDropped++;
peerInfo.SyncPeer.Disconnect(DisconnectReason.UselessPeer, "PEER REVIEW / HEAD 0");
}
else if (peerInfo.HeadNumber == 1920000) // mainnet, stuck Geth nodes
{
peersDropped++;
peerInfo.SyncPeer.Disconnect(DisconnectReason.UselessPeer, "PEER REVIEW / 1920000");
}
else if (peerInfo.HeadNumber == 7280022) // mainnet, stuck Geth nodes
{
peersDropped++;
peerInfo.SyncPeer.Disconnect(DisconnectReason.UselessPeer, "PEER REVIEW / 7280022");
}
else if (peerInfo.HeadNumber > ourNumber + 1024L && peerInfo.TotalDifficulty < ourDifficulty)
{
// probably classic nodes tht remain connected after we went pass the DAO
// worth to find a better way to discard them at the right time
peersDropped++;
peerInfo.SyncPeer.Disconnect(DisconnectReason.UselessPeer, "STRAY PEER");
}
}
if (PeerCount == PeerMaxCount)
{
long worstSpeed = long.MaxValue;
PeerInfo worstPeer = null;
foreach (PeerInfo peerInfo in AllPeers)
{
long transferSpeed = _stats.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageTransferSpeed() ?? 0;
if (transferSpeed < worstSpeed)
{
worstPeer = peerInfo;
}
}
peersDropped++;
worstPeer?.SyncPeer.Disconnect(DisconnectReason.TooManyPeers, "PEER REVIEW / LATENCY");
}
if (_logger.IsDebug)
{
_logger.Debug($"Dropped {peersDropped} useless peers");
}
}
public PeerInfo Select(PeerInfo currentPeer, IEnumerable <PeerInfo> peers, INodeStatsManager nodeStatsManager, IBlockTree blockTree)
{
int nullSpeed = -1;
decimal averageSpeed = 0M;
int peersCount = 0;
bool wasNull = currentPeer == null;
long currentSpeed = wasNull ? nullSpeed : nodeStatsManager.GetOrAdd(currentPeer.SyncPeer.Node).GetAverageTransferSpeed() ?? nullSpeed;
(PeerInfo Info, long TransferSpeed)fastestPeer = (currentPeer, currentSpeed);
(PeerInfo Info, long TransferSpeed)bestDiffPeer = (currentPeer, currentSpeed);
UInt256 localTotalDiff = blockTree.BestSuggestedHeader?.TotalDifficulty ?? UInt256.Zero;
foreach (PeerInfo info in peers)
{
(this as IPeerSelectionStrategy).CheckAsyncState(info);
peersCount++;
if (info.HeadNumber < (blockTree.BestSuggestedHeader?.Number ?? 0) + _minBlocksAhead)
{
// we need to be able to download some blocks ahead
continue;
}
if (info.TotalDifficulty <= localTotalDiff)
{
// if we require higher difficulty then we need to discard peers with same diff as ours
continue;
}
if (info.TotalDifficulty - localTotalDiff <= 2 && info.PeerClientType == PeerClientType.Parity)
{
// Parity advertises a better block but never sends it back and then it disconnects after a few conversations like this
// Geth responds all fine here
// note this is only 2 difficulty difference which means that is just for the POA / Clique chains
continue;
}
long averageTransferSpeed = nodeStatsManager.GetOrAdd(info.SyncPeer.Node).GetAverageTransferSpeed() ?? 0;
averageSpeed += averageTransferSpeed;
if (averageTransferSpeed > fastestPeer.TransferSpeed)
{
fastestPeer = (info, averageTransferSpeed);
}
if (info.TotalDifficulty >= (bestDiffPeer.Info?.TotalDifficulty ?? UInt256.Zero))
{
bestDiffPeer = (info, averageTransferSpeed);
}
}
if (peersCount == 0)
{
return(currentPeer);
}
if (bestDiffPeer.Info == null)
{
return(fastestPeer.Info);
}
averageSpeed /= peersCount;
UInt256 difficultyDifference = bestDiffPeer.Info.TotalDifficulty - localTotalDiff;
// at least 1 diff times 16 blocks of diff
if (difficultyDifference > 0 &&
difficultyDifference < ((blockTree.Head?.Difficulty ?? 0) + 1) * 16 &&
bestDiffPeer.TransferSpeed > averageSpeed)
{
return(bestDiffPeer.Info);
}
decimal speedRatio = fastestPeer.TransferSpeed / (decimal)Math.Max(1L, currentSpeed);
if (speedRatio > 1m + MinDiffPercentageForSpeedSwitch &&
fastestPeer.TransferSpeed - currentSpeed > MinDiffForSpeedSwitch)
{
return(fastestPeer.Info);
}
return(currentPeer ?? fastestPeer.Info);
}
private async Task RunPeerUpdateLoop()
{
while (true)
{
try
{
CleanupCandidatePeers();
}
catch (Exception e)
{
if (_logger.IsDebug)
{
_logger.Error("Candidate peers clanup failed", e);
}
}
_peerUpdateRequested.Wait(_cancellationTokenSource.Token);
_peerUpdateRequested.Reset();
if (!_isStarted)
{
continue;
}
int availableActiveCount = _networkConfig.ActivePeersMaxCount - _activePeers.Count;
if (availableActiveCount == 0)
{
continue;
}
if (_cancellationTokenSource.IsCancellationRequested)
{
break;
}
int tryCount = 0;
int newActiveNodes = 0;
int failedInitialConnect = 0;
int connectionRounds = 0;
var candidateSelection = SelectAndRankCandidates();
IReadOnlyCollection <Peer> remainingCandidates = candidateSelection.Candidates;
if (!remainingCandidates.Any())
{
continue;
}
while (true)
{
if (_cancellationTokenSource.IsCancellationRequested)
{
break;
}
availableActiveCount = _networkConfig.ActivePeersMaxCount - _activePeers.Count;
int nodesToTry = Math.Min(remainingCandidates.Count, availableActiveCount);
if (nodesToTry == 0)
{
break;
}
IEnumerable <Peer> candidatesToTry = remainingCandidates.Take(nodesToTry);
remainingCandidates = remainingCandidates.Skip(nodesToTry).ToList();
Parallel.ForEach(candidatesToTry, async(peer, loopState) =>
{
if (loopState.ShouldExitCurrentIteration || _cancellationTokenSource.IsCancellationRequested)
{
return;
}
Interlocked.Increment(ref tryCount);
// Can happen when In connection is received from the same peer and is initialized before we get here
// In this case we do not initialize OUT connection
if (!AddActivePeer(peer.Node.Id, peer, "upgrading candidate"))
{
if (_logger.IsTrace)
{
_logger.Trace($"Active peer was already added to collection: {peer.Node.Id}");
}
return;
}
bool result = await InitializePeerConnection(peer);
if (!result)
{
_stats.ReportEvent(peer.Node, NodeStatsEventType.ConnectionFailed);
Interlocked.Increment(ref failedInitialConnect);
if (peer.OutSession != null)
{
if (_logger.IsTrace)
{
_logger.Trace($"Timeout, doing additional disconnect: {peer.Node.Id}");
}
peer.OutSession?.Disconnect(DisconnectReason.ReceiveMessageTimeout, DisconnectType.Local);
}
DeactivatePeerIfDisconnected(peer, "Failed to initialize connections");
return;
}
Interlocked.Increment(ref newActiveNodes);
});
connectionRounds++;
}
if (_logger.IsDebug)
{
int activePeersCount = _activePeers.Count;
if (activePeersCount != _prevActivePeersCount)
{
string countersLog = string.Join(", ", candidateSelection.Counters.Select(x => $"{x.Key.ToString()}: {x.Value}"));
_logger.Debug($"RunPeerUpdate | {countersLog}, Incompatible: {GetIncompatibleDesc(candidateSelection.IncompatiblePeers)}, EligibleCandidates: {candidateSelection.Candidates.Count()}, " +
$"Tried: {tryCount}, Rounds: {connectionRounds}, Failed initial connect: {failedInitialConnect}, Established initial connect: {newActiveNodes}, " +
$"Current candidate peers: {_candidatePeers.Count}, Current active peers: {_activePeers.Count} " +
$"[InOut: {_activePeers.Count(x => x.Value.OutSession != null && x.Value.InSession != null)} | " +
$"[Out: {_activePeers.Count(x => x.Value.OutSession != null)} | " +
$"In: {_activePeers.Count(x => x.Value.InSession != null)}]");
}
_prevActivePeersCount = activePeersCount;
}
if (_logger.IsTrace)
{
if (_logCounter % 5 == 0)
{
string nl = Environment.NewLine;
_logger.Trace($"{nl}{nl}All active peers: {nl}{string.Join(nl, _activePeers.Values.Select(x => $"{_stats.GetOrAdd(x.Node).ToString()} | P2PInitialized: {_stats.GetOrAdd(x.Node).DidEventHappen(NodeStatsEventType.P2PInitialized)} | Eth62Initialized: {_stats.GetOrAdd(x.Node).DidEventHappen(NodeStatsEventType.Eth62Initialized)} | ClientId: {_stats.GetOrAdd(x.Node).P2PNodeDetails?.ClientId}"))} {nl}{nl}");
}
_logCounter++;
}
if (_activePeers.Count != _networkConfig.ActivePeersMaxCount)
{
_peerUpdateRequested.Set();
}
}
if (_logger.IsWarn)
{
_logger.Warn("Exiting peer update loop");
}
await Task.CompletedTask;
}
private async Task RunPeerUpdateLoop()
{
int loopCount = 0;
long previousActivePeersCount = 0;
while (true)
{
try
{
if (loopCount++ % 100 == 0)
{
if (_logger.IsTrace)
{
_logger.Trace($"Running peer update loop {loopCount - 1} - active: {_activePeers.Count} | candidates : {_peerPool.CandidatePeerCount}");
}
}
try
{
CleanupCandidatePeers();
}
catch (Exception e)
{
if (_logger.IsDebug)
{
_logger.Error("Candidate peers cleanup failed", e);
}
}
_peerUpdateRequested.Wait(_cancellationTokenSource.Token);
_peerUpdateRequested.Reset();
if (!_isStarted)
{
continue;
}
if (AvailableActivePeersCount == 0)
{
continue;
}
Interlocked.Exchange(ref _tryCount, 0);
Interlocked.Exchange(ref _newActiveNodes, 0);
Interlocked.Exchange(ref _failedInitialConnect, 0);
Interlocked.Exchange(ref _connectionRounds, 0);
SelectAndRankCandidates();
List <Peer> remainingCandidates = _currentSelection.Candidates;
if (!remainingCandidates.Any())
{
continue;
}
if (_cancellationTokenSource.IsCancellationRequested)
{
break;
}
int currentPosition = 0;
while (true)
{
if (_cancellationTokenSource.IsCancellationRequested)
{
break;
}
int nodesToTry = Math.Min(remainingCandidates.Count - currentPosition, AvailableActivePeersCount);
if (nodesToTry <= 0)
{
break;
}
ActionBlock <Peer> workerBlock = new ActionBlock <Peer>(
SetupPeerConnection,
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = _parallelism,
CancellationToken = _cancellationTokenSource.Token
});
for (int i = 0; i < nodesToTry; i++)
{
await workerBlock.SendAsync(remainingCandidates[currentPosition + i]);
}
currentPosition += nodesToTry;
workerBlock.Complete();
// Wait for all messages to propagate through the network.
workerBlock.Completion.Wait();
Interlocked.Increment(ref _connectionRounds);
}
if (_logger.IsTrace)
{
int activePeersCount = _activePeers.Count;
if (activePeersCount != previousActivePeersCount)
{
string countersLog = string.Join(", ", _currentSelection.Counters.Select(x => $"{x.Key.ToString()}: {x.Value}"));
_logger.Trace($"RunPeerUpdate | {countersLog}, Incompatible: {GetIncompatibleDesc(_currentSelection.Incompatible)}, EligibleCandidates: {_currentSelection.Candidates.Count()}, " +
$"Tried: {_tryCount}, Rounds: {_connectionRounds}, Failed initial connect: {_failedInitialConnect}, Established initial connect: {_newActiveNodes}, " +
$"Current candidate peers: {_peerPool.CandidatePeerCount}, Current active peers: {_activePeers.Count} " +
$"[InOut: {_activePeers.Count(x => x.Value.OutSession != null && x.Value.InSession != null)} | " +
$"[Out: {_activePeers.Count(x => x.Value.OutSession != null)} | " +
$"In: {_activePeers.Count(x => x.Value.InSession != null)}]");
}
previousActivePeersCount = activePeersCount;
}
if (_logger.IsTrace)
{
if (_logCounter % 5 == 0)
{
string nl = Environment.NewLine;
_logger.Trace($"{nl}{nl}All active peers: {nl} {string.Join(nl, _activePeers.Values.Select(x => $"{x.Node:s} | P2P: {_stats.GetOrAdd(x.Node).DidEventHappen(NodeStatsEventType.P2PInitialized)} | Eth62: {_stats.GetOrAdd(x.Node).DidEventHappen(NodeStatsEventType.Eth62Initialized)} | {_stats.GetOrAdd(x.Node).P2PNodeDetails?.ClientId} | {_stats.GetOrAdd(x.Node).ToString()}"))} {nl}{nl}");
}
_logCounter++;
}
if (_activePeers.Count < MaxActivePeers)
{
_peerUpdateRequested.Set();
}
}
catch (AggregateException e) when(e.InnerExceptions.Any(inner => inner is OperationCanceledException))
{
if (_logger.IsInfo)
{
_logger.Info("Peer update loop canceled.");
}
break;
}
catch (OperationCanceledException)
{
if (_logger.IsInfo)
{
_logger.Info("Peer update loop canceled");
}
break;
}
catch (Exception e)
{
if (_logger.IsError)
{
_logger.Error("Peer update loop failure", e);
}
break;
}
}
}
internal void DropUselessPeers(bool force = false)
{
if (!force && DateTime.UtcNow - _lastUselessPeersDropTime < TimeSpan.FromSeconds(30))
{
// give some time to monitoring nodes
// (monitoring nodes are nodes that are investigating the network but are not synced themselves)
return;
}
if (_logger.IsTrace)
{
_logger.Trace($"Reviewing {PeerCount} peer usefulness");
}
int peersDropped = 0;
_lastUselessPeersDropTime = DateTime.UtcNow;
long ourNumber = _blockTree.BestSuggestedHeader?.Number ?? 0L;
UInt256 ourDifficulty = _blockTree.BestSuggestedHeader?.TotalDifficulty ?? UInt256.Zero;
foreach (PeerInfo peerInfo in AllPeers)
{
if (peerInfo.HeadNumber > ourNumber)
{
// as long as we are behind we can use the stuck peers
continue;
}
if (peerInfo.HeadNumber == 0 &&
peerInfo.IsInitialized &&
ourNumber != 0 &&
peerInfo.PeerClientType != PeerClientType.Nethermind)
// we know that Nethermind reports 0 HeadNumber when it is in sync (and it can still serve a lot of data to other nodes)
{
peersDropped++;
peerInfo.SyncPeer.Disconnect(DisconnectReason.UselessPeer, "PEER REVIEW / HEAD 0");
}
else if (peerInfo.HeadNumber == 1920000) // mainnet, stuck Geth nodes
{
peersDropped++;
peerInfo.SyncPeer.Disconnect(DisconnectReason.UselessPeer, "PEER REVIEW / 1920000");
}
else if (peerInfo.HeadNumber == 7280022) // mainnet, stuck Geth nodes
{
peersDropped++;
peerInfo.SyncPeer.Disconnect(DisconnectReason.UselessPeer, "PEER REVIEW / 7280022");
}
else if (peerInfo.HeadNumber > ourNumber + 1024L && peerInfo.TotalDifficulty < ourDifficulty)
{
// probably Ethereum Classic nodes tht remain connected after we went pass the DAO
// worth to find a better way to discard them at the right time
peersDropped++;
peerInfo.SyncPeer.Disconnect(DisconnectReason.UselessPeer, "STRAY PEER");
}
}
if (PeerCount == PeerMaxCount)
{
long worstSpeed = long.MaxValue;
PeerInfo worstPeer = null;
foreach (PeerInfo peerInfo in AllPeers)
{
long transferSpeed = _stats.GetOrAdd(peerInfo.SyncPeer.Node).GetAverageTransferSpeed() ?? 0;
if (transferSpeed < worstSpeed)
{
worstPeer = peerInfo;
}
}
peersDropped++;
worstPeer?.SyncPeer.Disconnect(DisconnectReason.TooManyPeers, "PEER REVIEW / LATENCY");
}
if (_logger.IsDebug)
{
_logger.Debug($"Dropped {peersDropped} useless peers");
}
}
private void InitEthProtocol(ISession session, Eth62ProtocolHandler handler)
{
handler.ProtocolInitialized += (sender, args) =>
{
if (!RunBasicChecks(session, handler.ProtocolCode, handler.ProtocolVersion))
{
return;
}
var typedArgs = (EthProtocolInitializedEventArgs)args;
_stats.ReportEthInitializeEvent(session.Node, new EthNodeDetails
{
ChainId = typedArgs.ChainId,
BestHash = typedArgs.BestHash,
GenesisHash = typedArgs.GenesisHash,
ProtocolVersion = typedArgs.ProtocolVersion,
TotalDifficulty = typedArgs.TotalDifficulty
});
bool isValid = _protocolValidator.DisconnectOnInvalid(Protocol.Eth, session, args);
if (isValid)
{
handler.ClientId = session.Node.ClientId;
if (_syncPeers.TryAdd(session.SessionId, handler))
{
_syncManager.AddPeer(handler);
_transactionPool.AddPeer(handler);
if (_logger.IsDebug)
{
_logger.Debug($"{handler.ClientId} sync peer {session} created.");
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Not able to add a sync peer on {session} for {session.Node:s}");
}
session.InitiateDisconnect(DisconnectReason.AlreadyConnected, "sync peer");
}
if (_logger.IsTrace)
{
_logger.Trace($"Finalized ETH protocol initialization on {session} - adding sync peer {session.Node:s}");
}
//Add/Update peer to the storage and to sync manager
_peerStorage.UpdateNodes(new[] { new NetworkNode(session.Node.Id, session.Node.Host, session.Node.Port, _stats.GetOrAdd(session.Node).NewPersistedNodeReputation) });
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"|NetworkTrace| {handler.ProtocolCode}{handler.ProtocolVersion} is invalid on {session}");
}
}
};
}
private void InitEthProtocol(ISession session, Eth62ProtocolHandler handler)
{
handler.ProtocolInitialized += (sender, args) =>
{
if (!RunBasicChecks(session, handler.ProtocolCode, handler.ProtocolVersion))
{
return;
}
var typedArgs = (EthProtocolInitializedEventArgs)args;
_stats.ReportEthInitializeEvent(session.Node, new EthNodeDetails
{
ChainId = typedArgs.ChainId,
BestHash = typedArgs.BestHash,
GenesisHash = typedArgs.GenesisHash,
ProtocolVersion = typedArgs.ProtocolVersion,
TotalDifficulty = typedArgs.TotalDifficulty
});
bool isValid = _protocolValidator.DisconnectOnInvalid(Protocol.Eth, session, args);
if (isValid)
{
if (_syncPeers.TryAdd(session.SessionId, handler))
{
_syncManager.AddPeer(handler);
_transactionPool.AddPeer(handler);
}
else
{
session.InitiateDisconnect(DisconnectReason.ClientQuitting);
}
handler.ClientId = session.Node.ClientId;
if (_logger.IsTrace)
{
_logger.Trace($"Eth version {handler.ProtocolVersion} initialized, adding sync peer: {session.Node.Id}");
}
//Add/Update peer to the storage and to sync manager
_peerStorage.UpdateNodes(new[] { new NetworkNode(session.Node.Id, session.Node.Host, session.Node.Port, _stats.GetOrAdd(session.Node).NewPersistedNodeReputation) });
}
if (_logger.IsTrace)
{
_logger.Trace($"ETH Protocol Initialized: {session.RemoteNodeId}");
}
};
}