public async Task DialPeerAsync_GoodPeer_ShouldBeInPool()
{
IpEndPointHelper.TryParse(NetworkTestConstants.GoodPeerEndpoint, out var endpoint);
// two different hosts with the same pubkey.
var added = await _networkServer.ConnectAsync(endpoint);
added.ShouldBeTrue();
_peerPool.FindPeerByEndpoint(endpoint).ShouldNotBeNull();
}
public void AddedPeer_IsFindable_ByAddressAndPubkey()
{
var peer = CreatePeer();
_peerPool.TryAddPeer(peer);
_peerPool.PeerCount.ShouldBe(1);
_peerPool.FindPeerByEndpoint(peer.RemoteEndpoint).ShouldNotBeNull();
_peerPool.FindPeerByPublicKey(peer.Info.Pubkey).ShouldNotBeNull();
}
public async Task Connect_InboundPeerIsLater_Test()
{
var peer = CreatePeerAndAddToPeerPool();
peer.UpdateLastReceivedHandshake(new Handshake
{
HandshakeData = new HandshakeData
{
LastIrreversibleBlockHeight = 1,
BestChainHash = HashHelper.ComputeFrom("BestChainHash"),
BestChainHeight = 10,
Time = TimestampHelper.GetUtcNow().AddMinutes(1)
}
});
AElfPeerEndpointHelper.TryParse(NetworkTestConstants.GoodPeerEndpoint, out var endpoint);
var added = await _connectionService.ConnectAsync(endpoint);
added.ShouldBeTrue();
var currentPeer = _peerPool.FindPeerByEndpoint(endpoint);
currentPeer.ShouldNotBeNull();
AElfPeerEndpointHelper.TryParse(NetworkTestConstants.FakeIpEndpoint, out endpoint);
currentPeer = _peerPool.FindPeerByEndpoint(endpoint);
currentPeer.ShouldBeNull();
}
private async Task <GrpcPeer> GetDialedPeerWithEndpointAsync(DnsEndPoint endpoint)
{
var peer = _peerPool.FindPeerByEndpoint(endpoint);
if (peer != null)
{
if (peer.IsInvalid)
{
_peerPool.RemovePeer(peer.Info.Pubkey);
await peer.DisconnectAsync(false);
}
else
{
Logger.LogWarning($"Peer with endpoint {endpoint} is already in the pool.");
return(null);
}
}
if (_peerPool.IsPeerBlackListed(endpoint.Host))
{
Logger.LogDebug($"Peer with endpoint {endpoint} is blacklisted.");
return(null);
}
var dialedPeer = await _peerDialer.DialPeerAsync(endpoint);
if (dialedPeer == null)
{
Logger.LogDebug($"Error dialing {endpoint}.");
return(null);
}
return(dialedPeer);
}
public void QueryPeers_Test()
{
var commonHost = "12.34.56.67";
string commonPort = "1900";
string commonEndpoint = commonHost + ":" + commonPort;
var peer1 = CreatePeer(commonEndpoint);
_peerPool.TryAddPeer(peer1);
var peer2 = CreatePeer(commonEndpoint);
_peerPool.TryAddPeer(peer2);
var peer3 = CreatePeer("12.34.56.64:1900");
_peerPool.TryAddPeer(peer3);
var peer4 = CreatePeer("12.34.56.61:1900", isReady: false);
_peerPool.TryAddPeer(peer4);
var peers = _peerPool.GetPeers();
peers.Count.ShouldBe(3);
peers = _peerPool.GetPeers(true);
peers.Count.ShouldBe(4);
peers = _peerPool.GetPeersByHost(commonHost);
peers.Count.ShouldBe(2);
peers.ShouldContain(peer1);
peers.ShouldContain(peer2);
var peer = _peerPool.FindPeerByEndpoint(peer3.RemoteEndpoint);
peer.ShouldBe(peer3);
peer = _peerPool.FindPeerByPublicKey(peer3.Info.Pubkey);
peer.ShouldBe(peer3);
}
public async Task ProcessPeerDiscoveryJob_Test()
{
var node1 = new NodeInfo
{
Endpoint = "192.168.100.1:8001",
Pubkey = ByteString.CopyFromUtf8("node1")
};
await _peerDiscoveryService.AddNodeAsync(node1);
await RunDiscoveryWorkerAsync();
var endpointString = "192.168.100.100:8003";
var nodeList = await _peerDiscoveryService.GetNodesAsync(10);
nodeList.Nodes.Count.ShouldBe(1);
nodeList.Nodes[0].Endpoint.ShouldBe(endpointString);
nodeList.Nodes[0].Pubkey.ShouldBe(ByteString.CopyFromUtf8(endpointString));
AElfPeerEndpointHelper.TryParse(endpointString, out var aelEndpoint);
var peer = _peerPool.FindPeerByEndpoint(aelEndpoint);
peer.ShouldNotBeNull();
}
public async Task <bool> RemovePeerByEndpointAsync(string endpoint, int removalSeconds = NetworkConstants.DefaultPeerRemovalSeconds)
{
if (!AElfPeerEndpointHelper.TryParse(endpoint, out DnsEndPoint aelfPeerEndpoint))
{
return(false);
}
var peer = _peerPool.FindPeerByEndpoint(aelfPeerEndpoint);
if (!await TryRemovePeerAsync(peer, removalSeconds))
{
Logger.LogWarning($"Remove peer failed. Peer address: {endpoint}");
return(false);
}
return(true);
}
public async Task <bool> RemovePeerAsync(string address)
{
if (!AElfPeerEndpointHelper.TryParse(address, out DnsEndPoint endpoint))
{
return(false);
}
var peer = _peerPool.FindPeerByEndpoint(endpoint);
if (peer == null)
{
Logger.LogWarning($"Could not find peer at address {address}");
return(false);
}
await _networkServer.DisconnectAsync(peer);
return(true);
}
private async Task <GrpcPeer> GetDialedPeerWithEndpointAsync(DnsEndPoint endpoint)
{
if (_peerPool.FindPeerByEndpoint(endpoint) != null)
{
Logger.LogWarning($"Peer with endpoint {endpoint} is already in the pool.");
return(null);
}
if (_peerPool.IsPeerBlackListed(endpoint.Host))
{
Logger.LogWarning($"Peer with endpoint {endpoint} is blacklisted.");
return(null);
}
var dialedPeer = await _peerDialer.DialPeerAsync(endpoint);
if (dialedPeer == null)
{
Logger.LogWarning($"Error dialing {endpoint}.");
return(null);
}
return(dialedPeer);
}
internal async Task DoReconnectionJobAsync()
{
CheckNtpClockDrift();
await _networkService.CheckPeersHealthAsync();
var peersToConnect = _reconnectionService.GetPeersReadyForReconnection(TimestampHelper.GetUtcNow());
if (peersToConnect.Count <= 0)
{
return;
}
foreach (var peerToConnect in peersToConnect)
{
string peerEndpoint = peerToConnect.Endpoint;
if (!AElfPeerEndpointHelper.TryParse(peerEndpoint, out var parsed))
{
if (!_reconnectionService.CancelReconnection(peerEndpoint))
{
Logger.LogWarning($"Invalid {peerEndpoint}.");
}
continue;
}
// check that we haven't already reconnected to this node
if (_peerPool.FindPeerByEndpoint(parsed) != null)
{
Logger.LogDebug($"Peer {peerEndpoint} already in the pool, no need to reconnect.");
if (!_reconnectionService.CancelReconnection(peerEndpoint))
{
Logger.LogDebug($"Could not find to {peerEndpoint}.");
}
continue;
}
Logger.LogDebug($"Starting reconnection to {peerToConnect.Endpoint}.");
var connected = false;
try
{
connected = await _networkService.AddPeerAsync(peerEndpoint);
}
catch (Exception ex)
{
// down the stack the AddPeerAsync rethrows any exception,
// in order to continue this job, Exception has to be catched for now.
Logger.LogInformation(ex, $"Could not re-connect to {peerEndpoint}.");
}
if (connected)
{
Logger.LogDebug($"Reconnection to {peerEndpoint} succeeded.");
if (!_reconnectionService.CancelReconnection(peerEndpoint))
{
Logger.LogDebug($"Could not find {peerEndpoint}.");
}
}
else
{
var timeExtension = _networkOptions.PeerReconnectionPeriod * (int)Math.Pow(2, ++peerToConnect.RetryCount);
peerToConnect.NextAttempt = TimestampHelper.GetUtcNow().AddMilliseconds(timeExtension);
// if the option is set, verify that the next attempt does not exceed
// the maximum reconnection time.
if (_networkOptions.MaximumReconnectionTime != 0)
{
var maxReconnectionDate = peerToConnect.DisconnectionTime +
TimestampHelper.DurationFromMilliseconds(_networkOptions.MaximumReconnectionTime);
if (peerToConnect.NextAttempt > maxReconnectionDate)
{
_reconnectionService.CancelReconnection(peerEndpoint);
Logger.LogDebug($"Maximum reconnection time reached {peerEndpoint}, " +
$"next was {peerToConnect.NextAttempt}.");
continue;
}
}
Logger.LogDebug($"Could not connect to {peerEndpoint}, next attempt {peerToConnect.NextAttempt}, " +
$"current retries {peerToConnect.RetryCount}.");
}
}
void CheckNtpClockDrift()
{
try
{
_networkService.CheckNtpDrift();
}
catch (Exception)
{
// swallow any exception, we are not interested in anything else than valid checks.
}
}
}
/// <summary>
/// Connects to a node with the given ip address and adds it to the node's peer pool.
/// </summary>
/// <param name="endpoint">the ip address of the distant node</param>
/// <returns>True if the connection was successful, false otherwise</returns>
public async Task <bool> ConnectAsync(DnsEndPoint endpoint)
{
Logger.LogDebug($"Attempting to reach {endpoint}.");
if (_peerPool.FindPeerByEndpoint(endpoint) != null)
{
Logger.LogWarning($"Peer with endpoint {endpoint} is already in the pool.");
return(false);
}
if (_peerPool.IsPeerBlackListed(endpoint.Host))
{
Logger.LogWarning($"Peer with endpoint {endpoint} is blacklisted.");
return(false);
}
var dialedPeer = await _peerDialer.DialPeerAsync(endpoint);
if (dialedPeer == null)
{
Logger.LogWarning($"Error dialing {endpoint}.");
return(false);
}
var inboundPeer = _peerPool.FindPeerByPublicKey(dialedPeer.Info.Pubkey) as GrpcPeer;
/* A connection already exists, this can happen when both peers dial each other at the same time. To make
* sure both sides close the same connection, they both decide based on the times of the handshakes.
* Scenario steps, chronologically:
* 1) P1 (hsk_time: t1) --> dials P2 --and-- P1 <-- P2 dials (hsk_time: t2)
* 2) P2 receives P1s dial with t1 (in the hsk) and add to the pool
* 3) P1 receives P2s dial with and adds to pool
* 4) both dials finish and find that the pool already contains the dialed node.
* To resolve this situation, both peers will choose the connection that was initiated the earliest,
* so either P1s dial or P2s. */
GrpcPeer currentPeer = dialedPeer;
if (inboundPeer != null)
{
Logger.LogWarning("Duplicate peer connection detected: " +
$"{inboundPeer} ({inboundPeer.LastReceivedHandshakeTime}) " +
$"vs {dialedPeer} ({dialedPeer.LastSentHandshakeTime}).");
if (inboundPeer.LastReceivedHandshakeTime > dialedPeer.LastSentHandshakeTime)
{
// we started the dial first, replace the inbound connection with the dialed
if (!_peerPool.TryReplace(inboundPeer.Info.Pubkey, inboundPeer, dialedPeer))
{
Logger.LogWarning("Replacing the inbound connection failed.");
}
await inboundPeer.DisconnectAsync(false);
Logger.LogWarning($"Replaced the inbound connection with the dialed peer {inboundPeer} .");
}
else
{
// keep the inbound connection
await dialedPeer.DisconnectAsync(false);
currentPeer = inboundPeer;
Logger.LogWarning($"Disconnected dialed peer {dialedPeer}.");
}
}
else
{
if (!_peerPool.TryAddPeer(dialedPeer))
{
Logger.LogWarning($"Peer add to the failed {dialedPeer.Info.Pubkey}.");
await dialedPeer.DisconnectAsync(false);
return(false);
}
Logger.LogDebug($"Added to pool {dialedPeer.RemoteEndpoint} - {dialedPeer.Info.Pubkey}.");
}
try
{
await currentPeer.ConfirmHandshakeAsync();
}
catch (Exception e)
{
Logger.LogError(e, $"Confirm handshake error. Peer: {currentPeer.Info.Pubkey}.");
_peerPool.RemovePeer(currentPeer.Info.Pubkey);
await currentPeer.DisconnectAsync(false);
throw;
}
currentPeer.IsConnected = true;
currentPeer.SyncState = SyncState.Syncing;
Logger.LogInformation($"Connected to: {currentPeer.RemoteEndpoint} - {currentPeer.Info.Pubkey.Substring(0, 45)}" +
$" - in-token {currentPeer.InboundSessionId?.ToHex()}, out-token {currentPeer.OutboundSessionId?.ToHex()}" +
$" - LIB height {currentPeer.LastKnownLibHeight}" +
$" - best chain [{currentPeer.CurrentBlockHeight}, {currentPeer.CurrentBlockHash}]");
FireConnectionEvent(currentPeer);
return(true);
}
/// <summary>
/// Connects to a node with the given ip address and adds it to the node's peer pool.
/// </summary>
/// <param name="endpoint">the ip address of the distant node</param>
/// <returns>True if the connection was successful, false otherwise</returns>
public async Task <bool> ConnectAsync(IPEndPoint endpoint)
{
Logger.LogTrace($"Attempting to reach {endpoint}.");
if (_peerPool.FindPeerByEndpoint(endpoint) != null)
{
Logger.LogWarning($"Peer {endpoint} is already in the pool.");
return(false);
}
GrpcPeer peer;
try
{
// create the connection to the distant node
peer = await _peerDialer.DialPeerAsync(endpoint);
}
catch (PeerDialException ex)
{
Logger.LogError(ex, $"Dial exception {endpoint}:");
return(false);
}
var peerPubkey = peer.Info.Pubkey;
if (!_peerPool.TryAddPeer(peer))
{
Logger.LogWarning($"Peer {peerPubkey} is already in the pool.");
await peer.DisconnectAsync(false);
return(false);
}
Handshake peerHandshake;
try
{
peerHandshake = await peer.DoHandshakeAsync(await _handshakeProvider.GetHandshakeAsync());
}
catch (NetworkException ex)
{
Logger.LogError(ex, $"Handshake failed to {endpoint} - {peerPubkey}.");
await DisconnectAsync(peer);
return(false);
}
HandshakeError handshakeError = ValidateHandshake(peerHandshake, peerPubkey);
if (handshakeError != HandshakeError.HandshakeOk)
{
Logger.LogWarning($"Invalid handshake [{handshakeError}] from {endpoint} - {peerPubkey}");
await DisconnectAsync(peer);
return(false);
}
Logger.LogTrace($"Connected to {peer} - LIB height {peer.LastKnownLibHeight}, " +
$"best chain [{peer.CurrentBlockHeight}, {peer.CurrentBlockHash}].");
FireConnectionEvent(peer);
return(true);
}
internal async Task DoReconnectionJobAsync()
{
CheckNtpClockDrift();
await _networkService.SendHealthChecksAsync();
var peersToConnect = _reconnectionService.GetPeersReadyForReconnection(TimestampHelper.GetUtcNow());
if (peersToConnect.Count <= 0)
{
Logger.LogDebug("No peers to reconnect.");
return;
}
foreach (var peerToConnect in peersToConnect)
{
string peerEndpoint = peerToConnect.Endpoint;
if (!AElfPeerEndpointHelper.TryParse(peerEndpoint, out var parsed))
{
if (!_reconnectionService.CancelReconnection(peerEndpoint))
{
Logger.LogWarning($"Invalid {peerEndpoint}.");
}
continue;
}
// check that we haven't already reconnected to this node
if (_peerPool.FindPeerByEndpoint(parsed) != null)
{
Logger.LogDebug($"Peer {peerEndpoint} already in the pool, no need to reconnect.");
if (!_reconnectionService.CancelReconnection(peerEndpoint))
{
Logger.LogDebug($"Could not find to {peerEndpoint}.");
}
continue;
}
Logger.LogDebug($"Starting reconnection to {peerToConnect.Endpoint}.");
var connected = false;
try
{
connected = await _networkService.AddPeerAsync(peerEndpoint);
}
catch (Exception ex)
{
// down the stack the AddPeerAsync rethrows any exception,
// in order to continue this job, Exception has to be catched for now.
Logger.LogError(ex, $"Could not re-connect to {peerEndpoint}.");
}
if (connected)
{
Logger.LogDebug($"Reconnection to {peerEndpoint} succeeded.");
if (!_reconnectionService.CancelReconnection(peerEndpoint))
{
Logger.LogDebug($"Could not find {peerEndpoint}.");
}
}
else
{
peerToConnect.NextAttempt =
TimestampHelper.GetUtcNow().AddMilliseconds(_networkOptions.PeerReconnectionPeriod);
Logger.LogDebug($"Could not connect to {peerEndpoint}, next attempt {peerToConnect.NextAttempt}.");
}
}
void CheckNtpClockDrift()
{
try
{
_networkService.CheckNtpDrift();
}
catch (Exception)
{
// swallow any exception, we are not interested in anything else than valid checks.
}
}
}
internal async Task DoReconnectionJobAsync()
{
await _networkService.SendHealthChecksAsync();
var peersToConnect = _reconnectionService.GetPeersReadyForReconnection(TimestampHelper.GetUtcNow());
if (peersToConnect.Count <= 0)
{
Logger.LogDebug("No peers to reconnect.");
return;
}
foreach (var peerToConnect in peersToConnect)
{
string peerEndpoint = peerToConnect.Endpoint;
// check that we haven't already reconnected to this node
if (_peerPool.FindPeerByEndpoint(IpEndPointHelper.Parse(peerEndpoint)) != null)
{
Logger.LogDebug($"Peer {peerEndpoint} already in the pool, no need to reconnect.");
if (!_reconnectionService.CancelReconnection(peerEndpoint))
{
Logger.LogDebug($"Could not find to {peerEndpoint}.");
}
continue;
}
Logger.LogDebug($"Starting reconnection to {peerToConnect.Endpoint}.");
var connected = false;
try
{
connected = await _networkService.AddPeerAsync(peerEndpoint);
}
catch (Exception ex)
{
// todo consider different handling of the exception in dialer
// down the stack the AddPeerAsync rethrows any exception,
// in order to continue this job, Exception has to be catched for now.
Logger.LogError(ex, $"Could not re-connect to {peerEndpoint}.");
}
if (connected)
{
Logger.LogDebug($"Reconnection to {peerEndpoint} succeeded.");
if (!_reconnectionService.CancelReconnection(peerEndpoint))
{
Logger.LogDebug($"Could not find {peerEndpoint}.");
}
}
else
{
peerToConnect.NextAttempt =
TimestampHelper.GetUtcNow().AddMilliseconds(_networkOptions.PeerReconnectionPeriod);
Logger.LogDebug($"Could not connect to {peerEndpoint}, next attempt {peerToConnect.NextAttempt}.");
}
}
}
