public void StartEvictionProcess(INodeLifecycleManager evictionCandidate, INodeLifecycleManager replacementCandidate)
{
if (_logger.IsTrace)
{
_logger.Trace($"Starting eviction process, evictionCandidate: {evictionCandidate.ManagedNode}, replacementCandidate: {replacementCandidate.ManagedNode}");
}
EvictionPair newPair = new EvictionPair
{
EvictionCandidate = evictionCandidate,
ReplacementCandidate = replacementCandidate,
};
EvictionPair pair = _evictionPairs.GetOrAdd(evictionCandidate.ManagedNode.IdHash, newPair);
if (pair != newPair)
{
//existing eviction in process
//TODO add queue for further evictions
if (_logger.IsTrace)
{
_logger.Trace($"Existing eviction in process, evictionCandidate: {evictionCandidate.ManagedNode}, replacementCandidate: {replacementCandidate.ManagedNode}");
}
return;
}
evictionCandidate.OnStateChanged += OnStateChange;
evictionCandidate.StartEvictionProcess();
}
private void UpdateState(NodeLifecycleState newState)
{
if (newState == NodeLifecycleState.New)
{
//if node is just discovered we send ping to confirm it is active
SendPing();
}
else if (newState == NodeLifecycleState.Active)
{
//TODO && !ManagedNode.IsDicoveryNode - should we exclude discovery nodes
//received pong first time
if (State == NodeLifecycleState.New)
{
NodeAddResult result = _nodeTable.AddNode(ManagedNode);
if (result.ResultType == NodeAddResultType.Full)
{
INodeLifecycleManager evictionCandidate = _discoveryManager.GetNodeLifecycleManager(result.EvictionCandidate.Node);
if (evictionCandidate != null)
{
_evictionManager.StartEvictionProcess(evictionCandidate, this);
}
}
}
}
else if (newState == NodeLifecycleState.EvictCandidate)
{
SendPing();
}
State = newState;
OnStateChanged?.Invoke(this, State);
}
private void AddPersistedNodes(CancellationToken cancellationToken)
{
if (!_discoveryConfig.IsDiscoveryNodesPersistenceOn)
{
return;
}
NetworkNode[] nodes = _discoveryStorage.GetPersistedNodes();
foreach (NetworkNode networkNode in nodes)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
Node node;
try
{
node = new Node(networkNode.NodeId, networkNode.Host, networkNode.Port);
}
catch (Exception)
{
if (_logger.IsDebug)
{
_logger.Error($"ERROR/DEBUG peer could not be loaded for {networkNode.NodeId}@{networkNode.Host}:{networkNode.Port}");
}
continue;
}
INodeLifecycleManager manager = _discoveryManager.GetNodeLifecycleManager(node, true);
if (manager == null)
{
if (_logger.IsDebug)
{
_logger.Debug($"Skiping persisted node {networkNode.NodeId}@{networkNode.Host}:{networkNode.Port}, manager couldnt be created");
}
continue;
;
}
manager.NodeStats.CurrentPersistedNodeReputation = networkNode.Reputation;
if (_logger.IsTrace)
{
_logger.Trace($"Adding persisted node {networkNode.NodeId}@{networkNode.Host}:{networkNode.Port}");
}
}
if (_logger.IsDebug)
{
_logger.Debug($"Added persisted discovery nodes: {nodes.Length}");
}
}
private INodeLifecycleManager CreateLifecycleManager(Node node)
{
INodeLifecycleManager manager = Substitute.For <INodeLifecycleManager>();
manager.ManagedNode.Returns(node);
manager.NodeStats.Returns(new NodeStats(node, _statsConfig, NullLogManager.Instance)
{
CurrentPersistedNodeReputation = node.Port
});
return(manager);
}
private INodeLifecycleManager CreateLifecycleManager(Node node)
{
INodeLifecycleManager manager = Substitute.For <INodeLifecycleManager>();
manager.ManagedNode.Returns(node);
manager.NodeStats.Returns(new NodeStatsLight(node)
{
CurrentPersistedNodeReputation = node.Port
});
return(manager);
}
public void MemoryTest()
{
//receiving pong to have a node in the system
for (int a = 0; a < 255; a++)
{
for (int b = 0; b < 255; b++)
{
INodeLifecycleManager manager = _discoveryManager.GetNodeLifecycleManager(new Node($"{a}.{b}.1.1", 8000));
manager.SendPingAsync();
_discoveryManager.OnIncomingMessage(new PongMessage {
FarAddress = new IPEndPoint(IPAddress.Parse($"{a}.{b}.1.1"), _port), FarPublicKey = _publicKey
});
}
}
}
private void UpdateState(NodeLifecycleState newState)
{
if (newState == NodeLifecycleState.New)
{
//if node is just discovered we send ping to confirm it is active
#pragma warning disable 4014
SendPingAsync();
#pragma warning restore 4014
}
else if (newState == NodeLifecycleState.Active)
{
//TODO && !ManagedNode.IsDiscoveryNode - should we exclude discovery nodes
//received pong first time
if (State == NodeLifecycleState.New)
{
NodeAddResult result = _nodeTable.AddNode(ManagedNode);
if (result.ResultType == NodeAddResultType.Full)
{
INodeLifecycleManager evictionCandidate = _discoveryManager.GetNodeLifecycleManager(result.EvictionCandidate.Node);
if (evictionCandidate != null)
{
_evictionManager.StartEvictionProcess(evictionCandidate, this);
}
}
}
}
else if (newState == NodeLifecycleState.EvictCandidate)
{
if (State == NodeLifecycleState.EvictCandidate)
{
throw new InvalidOperationException("Cannot start more than one eviction process on same node.");
}
if (DateTime.UtcNow - _lastPingSent > TimeSpan.FromSeconds(5))
{
#pragma warning disable 4014
SendPingAsync();
#pragma warning restore 4014
}
else
{
OnStateChanged?.Invoke(this, NodeLifecycleState.Active);
}
}
State = newState;
OnStateChanged?.Invoke(this, State);
}
private async Task <Result> SendFindNode(Node destinationNode, byte[] searchedNodeId, CancellationToken cancellationToken)
{
try
{
INodeLifecycleManager nodeManager = _discoveryManager.GetNodeLifecycleManager(destinationNode);
nodeManager?.SendFindNode(searchedNodeId ?? _masterNode.Id.Bytes);
return(await _discoveryManager.WasMessageReceived(destinationNode.IdHash, MessageType.Neighbors, _discoveryConfig.SendNodeTimeout)
? Result.Success
: Result.Fail($"Did not receive Neighbors response in time from: {destinationNode.Host}"));
}
catch (OperationCanceledException)
{
return(Result.Fail("Cancelled"));
}
}
private async Task <bool> InitializeBootnodes(CancellationToken cancellationToken)
{
NetworkNode[] bootnodes = NetworkNode.ParseNodes(_discoveryConfig.Bootnodes, _logger);
if (!bootnodes.Any())
{
if (_logger.IsWarn)
{
_logger.Warn("No bootnodes specified in configuration");
}
return(true);
}
List <INodeLifecycleManager> managers = new List <INodeLifecycleManager>();
for (int i = 0; i < bootnodes.Length; i++)
{
NetworkNode bootnode = bootnodes[i];
Node node = bootnode.NodeId == null
? new Node(bootnode.Host, bootnode.Port)
: new Node(bootnode.NodeId, bootnode.Host, bootnode.Port, true);
INodeLifecycleManager manager = _discoveryManager.GetNodeLifecycleManager(node);
if (manager != null)
{
managers.Add(manager);
}
else
{
_logger.Warn($"Bootnode config contains self: {bootnode.NodeId}");
}
}
//Wait for pong message to come back from Boot nodes
int maxWaitTime = _discoveryConfig.BootnodePongTimeout;
int itemTime = maxWaitTime / 100;
for (int i = 0; i < 100; i++)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (managers.Any(x => x.State == NodeLifecycleState.Active))
{
break;
}
if (_discoveryManager.GetOrAddNodeLifecycleManagers(x => x.State == NodeLifecycleState.Active).Any())
{
if (_logger.IsTrace)
{
_logger.Trace("Was not able to connect to any of the bootnodes, but successfully connected to at least one persisted node.");
}
break;
}
if (_logger.IsTrace)
{
_logger.Trace($"Waiting {itemTime} ms for bootnodes to respond");
}
try
{
await Task.Delay(itemTime, cancellationToken);
}
catch (OperationCanceledException)
{
break;
}
}
int reachedNodeCounter = 0;
for (int i = 0; i < managers.Count; i++)
{
INodeLifecycleManager manager = managers[i];
if (manager.State != NodeLifecycleState.Active)
{
if (_logger.IsTrace)
{
_logger.Trace($"Could not reach bootnode: {manager.ManagedNode.Host}:{manager.ManagedNode.Port}");
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Reached bootnode: {manager.ManagedNode.Host}:{manager.ManagedNode.Port}");
}
reachedNodeCounter++;
}
}
if (_logger.IsInfo)
{
_logger.Info($"Connected to {reachedNodeCounter} bootnodes, {_discoveryManager.GetOrAddNodeLifecycleManagers(x => x.State == NodeLifecycleState.Active).Count} trusted/persisted nodes");
}
return(reachedNodeCounter > 0);
}
private void CloseEvictionProcess(INodeLifecycleManager evictionCandidate)
{
_evictionPairs.TryRemove(evictionCandidate.ManagedNode.IdHash, out EvictionPair _);
evictionCandidate.OnStateChanged -= OnStateChange;
}
public void EvictCandidateStateLostEvictionTest()
{
//adding 3 active nodes
List <INodeLifecycleManager> managers = new();
for (int i = 0; i < 3; i++)
{
string host = "192.168.1." + i;
Node node = new(_nodeIds[i], host, _port);
INodeLifecycleManager?manager = _discoveryManager.GetNodeLifecycleManager(node);
if (manager is null)
{
throw new Exception("Manager is null");
}
managers.Add(manager);
Assert.AreEqual(NodeLifecycleState.New, manager.State);
PongMsg msg = new (_nodeIds[i], GetExpirationTime(), new byte[32]);
msg.FarAddress = new IPEndPoint(IPAddress.Parse(_host), _port);
_discoveryManager.OnIncomingMsg(msg);
Assert.AreEqual(NodeLifecycleState.Active, manager.State);
}
//table should contain 3 active nodes
IEnumerable <Node> closestNodes = _nodeTable.GetClosestNodes().ToArray();
for (int i = 0; i < 3; i++)
{
Assert.IsTrue(closestNodes.Count(x => x.Host == managers[0].ManagedNode.Host) == 1);
}
//adding 4th node - table can store only 3, eviction process should start
Node candidateNode = new(_nodeIds[3], _host, _port);
INodeLifecycleManager?candidateManager = _discoveryManager.GetNodeLifecycleManager(candidateNode);
Assert.AreEqual(NodeLifecycleState.New, candidateManager?.State);
PongMsg pongMsg = new (_nodeIds[3], GetExpirationTime(), new byte[32]);
pongMsg.FarAddress = new IPEndPoint(IPAddress.Parse(_host), _port);
_discoveryManager.OnIncomingMsg(pongMsg);
//await Task.Delay(10);
Assert.That(() => candidateManager?.State, Is.EqualTo(NodeLifecycleState.Active).After(10, 5));
//Assert.AreEqual(NodeLifecycleState.Active, candidateManager.State);
INodeLifecycleManager evictionCandidate = managers.First(x => x.State == NodeLifecycleState.EvictCandidate);
//await Task.Delay(300);
//3th node should be evicted, 4th node should be added to the table
//Assert.AreEqual(NodeLifecycleState.Active, candidateManager.State);
Assert.That(() => candidateManager?.State, Is.EqualTo(NodeLifecycleState.Active).After(300, 50));
//Assert.AreEqual(NodeLifecycleState.Unreachable, evictionCandidate.State);
Assert.That(() => evictionCandidate.State, Is.EqualTo(NodeLifecycleState.Unreachable).After(300, 50));
closestNodes = _nodeTable.GetClosestNodes();
Assert.That(() => managers.Where(x => x.State == NodeLifecycleState.Active).All(x => closestNodes.Any(y => y.Host == x.ManagedNode.Host)), Is.True.After(300, 50));
Assert.That(() => closestNodes.Count(x => x.Host == evictionCandidate.ManagedNode.Host) == 0, Is.True.After(300, 50));
Assert.That(() => closestNodes.Count(x => x.Host == candidateNode.Host) == 1, Is.True.After(300, 50));
//Assert.IsTrue(managers.Where(x => x.State == NodeLifecycleState.Active).All(x => closestNodes.Any(y => y.Host == x.ManagedNode.Host)));
//Assert.IsTrue(closestNodes.Count(x => x.Host == evictionCandidate.ManagedNode.Host) == 0);
//Assert.IsTrue(closestNodes.Count(x => x.Host == candidateNode.Host) == 1);
}
public Peer(INodeLifecycleManager manager)
{
Node = manager.ManagedNode;
NodeLifecycleManager = manager;
NodeStats = manager.NodeStats;
}
public NodeEventArgs(INodeLifecycleManager manager)
{
Manager = manager;
}
private async Task <bool> InitializeBootnodes()
{
var bootNodes = _configurationProvider.BootNodes;
if (bootNodes == null || !bootNodes.Any())
{
if (_logger.IsWarnEnabled)
{
_logger.Warn("No bootnodes specified in configuration");
}
return(false);
}
var managers = new INodeLifecycleManager[bootNodes.Length];
for (var i = 0; i < bootNodes.Length; i++)
{
var bootnode = bootNodes[i];
var node = bootnode.NodeId == null
? _nodeFactory.CreateNode(bootnode.Host, bootnode.Port)
: _nodeFactory.CreateNode(new NodeId(new PublicKey(new Hex(bootnode.NodeId))), bootnode.Host, bootnode.Port, true);
var manager = _discoveryManager.GetNodeLifecycleManager(node);
managers[i] = manager;
}
//Wait for pong message to come back from Boot nodes
var maxWaitTime = _configurationProvider.BootNodePongTimeout;
var itemTime = maxWaitTime / 100;
for (var i = 0; i < 100; i++)
{
if (managers.Any(x => x.State == NodeLifecycleState.Active))
{
break;
}
if (_logger.IsDebugEnabled)
{
_logger.Debug($"Waiting {itemTime} ms for bootnodes to respond");
}
await Task.Delay(1000);
}
var reachedNodeCounter = 0;
for (var i = 0; i < managers.Length; i++)
{
var manager = managers[i];
if (manager.State != NodeLifecycleState.Active)
{
if (_logger.IsWarnEnabled)
{
_logger.Warn($"Cannot reach bootnode: {manager.ManagedNode.Host}:{manager.ManagedNode.Port}");
}
}
else
{
reachedNodeCounter++;
}
}
return(reachedNodeCounter > 0);
}
public EvictionPair(INodeLifecycleManager evictionCandidate, INodeLifecycleManager replacementCandidate)
{
EvictionCandidate = evictionCandidate;
ReplacementCandidate = replacementCandidate;
}
