public ExtendedBlockPullerBehavior(IBlockPuller blockPuller, IInitialBlockDownloadState ibdState, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory)
{
this.ShouldThrowAtRequestBlocksAsync = false;
this.RecalculateQualityScoreWasCalled = false;
this.RequestedHashes = new List <uint256>();
this.underlyingBehavior = new BlockPullerBehavior(blockPuller, ibdState, dateTimeProvider, loggerFactory);
}
/// <summary>
/// Decreases the quality score of the peer node.
/// <para>This function is called when something goes wrong with the peer.</para>
/// <para>If the score reaches the minimal value, the tasks assigned for the node are released.</para>
/// </summary>
/// <param name="chainedBlock">Block the node wanted to download, but something went wrong during the process.</param>
protected void OnStalling(ChainedBlock chainedBlock)
{
this.logger.LogTrace("({0}:'{1}')", nameof(chainedBlock), chainedBlock.HashBlock);
BlockPullerBehavior behavior = null;
lock (this.lockObject)
{
this.assignedBlockTasks.TryGetValue(chainedBlock.HashBlock, out behavior);
}
if (behavior != null)
{
double penalty = this.peerQuality.CalculateNextBlockTimeoutQualityPenalty();
this.logger.LogTrace("Block '{0}' assigned to peer {1:x}, penalty is {2}.", chainedBlock.HashBlock, behavior.GetHashCode(), penalty);
behavior.UpdateQualityScore(penalty);
if (Math.Abs(behavior.QualityScore - QualityScore.MinScore) < 0.00001)
{
behavior.ReleaseAll(false);
this.AssignPendingVectors();
}
}
else
{
this.logger.LogTrace("Block '{0}' not assigned to any peer.", chainedBlock.HashBlock);
this.AssignPendingVectors();
}
this.logger.LogTrace("(-)");
}
/// <summary>
/// Releases all pending block download tasks assigned to a peer.
/// </summary>
/// <param name="peer">Peer to have all its pending download task released.</param>
/// <param name="disconnectPeer">If set to <c>true</c> the peer is considered as disconnected and should be prevented from being assigned additional work.</param>
/// <exception cref="InvalidOperationException">Thrown in case of data inconsistency between synchronized structures, which should never happen.</exception>
internal void ReleaseAllPeerDownloadTaskAssignments(BlockPullerBehavior peer, bool disconnectPeer)
{
this.logger.LogTrace("({0}:'{1:x}',{2}:{3})", nameof(peer), peer.GetHashCode(), nameof(disconnectPeer), disconnectPeer);
lock (this.lockObject)
{
// Prevent the peer to get any more work from now on if it was disconnected.
if (disconnectPeer)
{
peer.Disconnected = true;
}
Dictionary <uint256, DownloadAssignment> peerPendingDownloads;
if (this.peersPendingDownloads.TryGetValue(peer, out peerPendingDownloads))
{
this.logger.LogTrace("Releasing {0} pending downloads of peer '{1:x}'.", peerPendingDownloads.Count, peer.GetHashCode());
// Make a fresh copy of items in peerPendingDownloads to avoid modification of the collection.
foreach (uint256 blockHash in peerPendingDownloads.Keys.ToList())
{
if (!this.ReleaseDownloadTaskAssignmentLocked(peerPendingDownloads, blockHash))
{
this.logger.LogCritical("Data structures inconsistency, please notify the devs.");
throw new InvalidOperationException("Data structures inconsistency, please notify the devs.");
}
}
this.peersPendingDownloads.Remove(peer);
}
}
this.logger.LogTrace("(-)");
}
public BlockPullerBehaviorTests()
{
var puller = new Mock <IBlockPuller>();
var ibdState = new Mock <IInitialBlockDownloadState>();
ibdState.Setup(x => x.IsInitialBlockDownload()).Returns(() => true);
var loggerFactory = ExtendedLoggerFactory.Create();
this.behavior = new BlockPullerBehavior(puller.Object, ibdState.Object, DateTimeProvider.Default, loggerFactory);
}
/// <summary>
/// Obtains a number of tasks assigned to a peer.
/// </summary>
/// <param name="peer">Peer to get number of assigned tasks for.</param>
/// <returns>Number of tasks assigned to <paramref name="peer"/>.</returns>
internal int GetPendingDownloadsCount(BlockPullerBehavior peer)
{
int res = 0;
lock (this.lockObject)
{
Dictionary <uint256, DownloadAssignment> peerPendingDownloads;
if (this.peersPendingDownloads.TryGetValue(peer, out peerPendingDownloads))
{
res = peerPendingDownloads.Count;
}
}
return(res);
}
public BlockPullerBehaviorTests()
{
var puller = new Mock <IBlockPuller>();
var ibdState = new Mock <IInitialBlockDownloadState>();
ibdState.Setup(x => x.IsInitialBlockDownload()).Returns(() => true);
var loggerFactory = new ExtendedLoggerFactory();
loggerFactory.AddConsoleWithFilters();
this.behavior = new BlockPullerBehavior(puller.Object, ibdState.Object, loggerFactory);
}
public string GetStats()
{
StringBuilder builder = new StringBuilder();
lock (this.downloads)
{
PerformanceSnapshot diffTotal = new PerformanceSnapshot(0, 0);
builder.AppendLine("=======Connections=======");
foreach (INetworkPeer peer in this.ConnectedPeers)
{
PerformanceSnapshot newSnapshot = peer.Counter.Snapshot();
PerformanceSnapshot lastSnapshot = null;
if (this.downloads.TryGetValue(peer, out lastSnapshot))
{
BlockPullerBehavior behavior = peer.Behaviors.OfType <BlockPullerBehavior>()
.FirstOrDefault(b => b.Puller.GetType() == typeof(LookaheadBlockPuller));
PerformanceSnapshot diff = newSnapshot - lastSnapshot;
diffTotal = new PerformanceSnapshot(diff.TotalReadBytes + diffTotal.TotalReadBytes, diff.TotalWrittenBytes + diffTotal.TotalWrittenBytes)
{
Start = diff.Start, Taken = diff.Taken
};
builder.Append((peer.RemoteSocketAddress + ":" + peer.RemoteSocketPort).PadRight(LoggingConfiguration.ColumnLength * 2) + "R:" + this.ToKBSec(diff.ReadenBytesPerSecond) + "\tW:" + this.ToKBSec(diff.WrittenBytesPerSecond));
if (behavior != null)
{
int intQuality = (int)behavior.QualityScore;
builder.Append("\tQualityScore: " + intQuality + (intQuality < 10 ? "\t" : "") + "\tPendingBlocks: " + behavior.PendingDownloadsCount);
}
builder.AppendLine();
}
this.downloads.AddOrReplace(peer, newSnapshot);
}
builder.AppendLine("=================");
builder.AppendLine("Total".PadRight(LoggingConfiguration.ColumnLength * 2) + "R:" + this.ToKBSec(diffTotal.ReadenBytesPerSecond) + "\tW:" + this.ToKBSec(diffTotal.WrittenBytesPerSecond));
builder.AppendLine("==========================");
//TODO: Hack, we should just clean nodes that are not connect anymore.
if (this.downloads.Count > 1000)
{
this.downloads.Clear();
}
}
return(builder.ToString());
}
/// <summary>
/// Adds download task to the peer's list of pending download tasks.
/// </summary>
/// <param name="peer">Peer to add task to.</param>
/// <param name="blockHash">Hash of the block being assigned to <paramref name="peer"/> for download.</param>
/// <remarks>The caller of this method is responsible for holding <see cref="lockObject"/>.</remarks>
private void AddPeerPendingDownloadLocked(BlockPullerBehavior peer, uint256 blockHash)
{
this.logger.LogTrace("({0}:'{1:x}',{2}:'{3}')", nameof(peer), peer.GetHashCode(), nameof(blockHash), blockHash);
Dictionary <uint256, DownloadAssignment> peerPendingDownloads;
if (!this.peersPendingDownloads.TryGetValue(peer, out peerPendingDownloads))
{
peerPendingDownloads = new Dictionary <uint256, DownloadAssignment>();
this.peersPendingDownloads.Add(peer, peerPendingDownloads);
}
DownloadAssignment downloadTask = new DownloadAssignment(blockHash);
peerPendingDownloads.Add(blockHash, downloadTask);
this.logger.LogTrace("(-)");
}
/// <summary>
/// Checks if the puller behavior is currently responsible for downloading specific block.
/// </summary>
/// <param name="peer">Peer's behavior to check the assignment for.</param>
/// <param name="blockHash">Hash of the block.</param>
/// <returns><c>true</c> if the <paramref name="peer"/> is currently responsible for downloading block with hash <paramref name="blockHash"/>.</returns>
public bool CheckBlockTaskAssignment(BlockPullerBehavior peer, uint256 blockHash)
{
this.logger.LogTrace("({0}:'{1:x}',{2}:'{3}')", nameof(peer), peer.GetHashCode(), nameof(blockHash), blockHash);
bool res = false;
lock (this.lockObject)
{
Dictionary <uint256, DownloadAssignment> peerPendingDownloads;
if (this.peersPendingDownloads.TryGetValue(peer, out peerPendingDownloads))
{
res = peerPendingDownloads.ContainsKey(blockHash);
}
}
this.logger.LogTrace("(-):{0}", res);
return(res);
}
/// <summary>
/// Assigns a download task to a specific peer.
/// </summary>
/// <param name="peer">Peer to be assigned the new task.</param>
/// <param name="blockHash">Hash of the block to download from <paramref name="peer"/>.</param>
/// <param name="peerDisconnected">If the function fails, this is set to <c>true</c> if the peer was marked as disconnected and thus unable to be assigned any more work.</param>
/// <returns>
/// <c>true</c> if the block was assigned to the peer, <c>false</c> in case the block has already been assigned to someone,
/// or if the peer is disconnected and should not be assigned any more work.
/// </returns>
internal bool AssignDownloadTaskToPeer(BlockPullerBehavior peer, uint256 blockHash, out bool peerDisconnected)
{
this.logger.LogTrace("({0}:'{1:x}',{2}:'{3}')", nameof(peer), peer.GetHashCode(), nameof(blockHash), blockHash);
bool res = false;
lock (this.lockObject)
{
peerDisconnected = peer.Disconnected;
if (!peerDisconnected && this.assignedBlockTasks.TryAdd(blockHash, peer))
{
this.AddPeerPendingDownloadLocked(peer, blockHash);
res = true;
}
}
this.logger.LogTrace("(-):{0}", res);
return(res);
}
/// <summary>
/// Assigns a pending download task to a specific peer.
/// </summary>
/// <param name="peer">Peer to be assigned the new task.</param>
/// <param name="blockHash">If the function succeeds, this is filled with the hash of the block that will be requested from <paramref name="peer"/>.</param>
/// <returns>
/// <c>true</c> if a download task was assigned to the peer, <c>false</c> otherwise,
/// which indicates that there was no pending task, or that the peer is disconnected and should not be assigned any more work.
/// </returns>
internal bool AssignPendingDownloadTaskToPeer(BlockPullerBehavior peer, out uint256 blockHash)
{
this.logger.LogTrace("({0}:'{1:x}')", nameof(peer), peer.GetHashCode());
blockHash = null;
lock (this.lockObject)
{
if (!peer.Disconnected && (this.pendingInventoryVectors.Count > 0))
{
blockHash = this.pendingInventoryVectors.Dequeue();
if (this.assignedBlockTasks.TryAdd(blockHash, peer))
{
this.AddPeerPendingDownloadLocked(peer, blockHash);
}
}
}
bool res = blockHash != null;
this.logger.LogTrace("(-):{0},*{1}='{2}'", res, nameof(blockHash), blockHash);
return(res);
}
/// <summary>
/// When a peer downloads a block, it notifies the puller about the block by calling this method.
/// <para>
/// The downloaded task is removed from the list of pending downloads
/// and it is also removed from the <see cref="assignedBlockTasks"/> - i.e. the task is no longer assigned to the peer.
/// And finally, it is added to the list of downloaded blocks, provided that the block is not present there already.
/// </para>
/// </summary>
/// <param name="peer">Peer that finished the download task.</param>
/// <param name="blockHash">Hash of the downloaded block.</param>
/// <param name="downloadedBlock">Description of the downloaded block.</param>
/// <returns>
/// <c>true</c> if the download task for the block was assigned to <paramref name="peer"/>
/// and the task was removed and added to the list of downloaded blocks.
/// <c>false</c> if the downloaded block has been assigned to another peer
/// or if the block was already on the list of downloaded blocks.
/// </returns>
internal bool DownloadTaskFinished(BlockPullerBehavior peer, uint256 blockHash, DownloadedBlock downloadedBlock)
{
this.logger.LogTrace("({0}:'{1:x}',{2}:'{3}',{4}.{5}:{6})", nameof(peer), peer.GetHashCode(), nameof(blockHash), blockHash, nameof(downloadedBlock), nameof(downloadedBlock.Length), downloadedBlock.Length);
bool error = false;
bool res = false;
double peerQualityAdjustment = 0;
lock (this.lockObject)
{
BlockPullerBehavior peerAssigned;
if (this.assignedBlockTasks.TryGetValue(blockHash, out peerAssigned))
{
Dictionary <uint256, DownloadAssignment> peerPendingDownloads;
if (this.peersPendingDownloads.TryGetValue(peer, out peerPendingDownloads))
{
if (peer == peerAssigned)
{
DownloadAssignment downloadTask = null;
peerPendingDownloads.TryGetValue(blockHash, out downloadTask);
if (this.assignedBlockTasks.Remove(blockHash) && peerPendingDownloads.Remove(blockHash))
{
// Task was assigned to this peer and was removed.
if (this.downloadedBlocks.TryAdd(blockHash, downloadedBlock))
{
long blockDownloadTime = downloadTask.Finish();
this.peerQuality.AddSample(peer, blockDownloadTime, downloadedBlock.Length);
peerQualityAdjustment = this.peerQuality.CalculateQualityAdjustment(blockDownloadTime, downloadedBlock.Length);
this.logger.LogTrace("Block '{0}' size '{1}' downloaded by peer '{2:x}' in {3} ms, peer's score will be adjusted by {4}.", blockHash, downloadedBlock.Length, peer.GetHashCode(), blockDownloadTime, peerQualityAdjustment);
res = true;
}
else
{
this.logger.LogTrace("Block '{0}' already present on the list of downloaded blocks.", blockHash);
}
}
else
{
// Task was assigned to this peer but the data are inconsistent.
error = true;
}
}
else
{
// Before this peer provided the block, it has been assigned to other peer, which is OK.
this.logger.LogTrace("Incoming block '{0}' is assigned to peer '{1:x}', not to '{2:x}'.", blockHash, peerAssigned.GetHashCode(), peer.GetHashCode());
}
}
else
{
// Peer's pending downloads were probably released, which is OK.
this.logger.LogTrace("Peer '{0:x}' has no assignments.", peer.GetHashCode());
}
}
else
{
// The task was probably assigned to other peer and that task completed before this peer provided the block, which is OK.
this.logger.LogTrace("Incoming block '{0}' is not pending.", blockHash);
}
}
if (error)
{
this.logger.LogCritical("Data structures inconsistency, please notify the devs.");
// TODO: This exception is going to be silently discarded by Node_MessageReceived.
throw new InvalidOperationException("Data structures inconsistency, please notify the devs.");
}
if (res)
{
peer.UpdateQualityScore(peerQualityAdjustment);
}
this.logger.LogTrace("(-):{0}", res);
return(res);
}
/// <summary>
/// Schedules downloading of one or more blocks that the node is missing from one or more peer nodes.
/// <para>
/// Node's quality score is being considered as a weight during the random distribution
/// of the download tasks among the nodes.
/// </para>
/// <para>
/// Nodes are only asked for blocks that they should have (according to our information
/// about how long their chains are).
/// </para>
/// </summary>
/// <param name="vectors">List of information about blocks to download mapped by their height. Must not be empty.</param>
/// <param name="minHeight">Minimum height of the chain that the target nodes has to have in order to be asked for one or more of the block to be downloaded from them.</param>
private void DistributeDownload(Dictionary <int, InventoryVector> vectors, int minHeight)
{
this.logger.LogTrace("({0}.{1}:{2},{3}:{4})", nameof(vectors), nameof(vectors.Count), vectors.Count, nameof(minHeight), minHeight);
// Count number of tasks assigned to each peer.
BlockPullerBehavior[] nodes = this.GetNodeBehaviors();
Dictionary <BlockPullerBehavior, int> assignedTasksCount = new Dictionary <BlockPullerBehavior, int>();
lock (this.lockObject)
{
foreach (BlockPullerBehavior behavior in nodes)
{
int taskCount = 0;
Dictionary <uint256, DownloadAssignment> peerPendingDownloads;
if (this.peersPendingDownloads.TryGetValue(behavior, out peerPendingDownloads))
{
taskCount = peerPendingDownloads.Keys.Count;
}
assignedTasksCount.Add(behavior, taskCount);
}
}
// Prefilter available peers so that we only work with peers that can be assigned any work.
// If there is a peer whose chain is so short that it can't provide any blocks we want, it is ignored.
List <PullerDownloadAssignments.PeerInformation> peerInformation = new List <PullerDownloadAssignments.PeerInformation>();
foreach (BlockPullerBehavior behavior in nodes)
{
int?peerHeight = behavior.ChainHeadersBehavior?.PendingTip?.Height;
if (peerHeight >= minHeight)
{
PullerDownloadAssignments.PeerInformation peerInfo = new PullerDownloadAssignments.PeerInformation
{
QualityScore = behavior.QualityScore,
PeerId = behavior,
ChainHeight = peerHeight.Value,
TasksAssignedCount = assignedTasksCount[behavior]
};
peerInformation.Add(peerInfo);
this.logger.LogTrace("Peer '{0:x}' available: quality {1}, height {2}.", peerInfo.PeerId.GetHashCode(), peerInfo.QualityScore, peerInfo.ChainHeight);
}
else
{
this.logger.LogTrace("Peer '{0:x}' filtered out: height {1}.", behavior.GetHashCode(), peerHeight);
}
}
// There are no available peers with long enough chains.
if (peerInformation.Count == 0)
{
lock (this.lockObject)
{
foreach (InventoryVector vector in vectors.Values)
{
this.pendingInventoryVectors.Enqueue(vector.Hash);
}
}
this.logger.LogTrace("(-)[NO_PEERS_LEFT]");
return;
}
List <int> requestedBlockHeights = vectors.Keys.ToList();
Dictionary <PullerDownloadAssignments.PeerInformation, List <int> > blocksAssignedToPeers = PullerDownloadAssignments.AssignBlocksToPeers(requestedBlockHeights, peerInformation);
// Go through the assignments and start download tasks.
foreach (KeyValuePair <PullerDownloadAssignments.PeerInformation, List <int> > kvp in blocksAssignedToPeers)
{
PullerDownloadAssignments.PeerInformation peer = kvp.Key;
List <int> blockHeightsToDownload = kvp.Value;
GetDataPayload getDataPayload = new GetDataPayload();
BlockPullerBehavior peerBehavior = (BlockPullerBehavior)peer.PeerId;
// Create GetDataPayload from the list of block heights this peer has been assigned.
bool peerDisconnected = false;
foreach (int blockHeight in blockHeightsToDownload)
{
InventoryVector inventoryVector = vectors[blockHeight];
if (this.AssignDownloadTaskToPeer(peerBehavior, inventoryVector.Hash, out peerDisconnected))
{
this.logger.LogTrace("Block '{0}/{1}' assigned to peer '{2:x}'", inventoryVector.Hash, blockHeight, peerBehavior.GetHashCode());
getDataPayload.Inventory.Add(inventoryVector);
}
else if (peerDisconnected)
{
// The peer was disconnected recently, we need to make sure that the blocks assigned to it go back to the pending list.
// This is done below.
this.logger.LogTrace("Peer '{0:x} has been disconnected.'", peerBehavior.GetHashCode());
break;
}
// else This block has been assigned to someone else already, no action required.
}
if (!peerDisconnected)
{
// If this node was assigned at least one download task, start the task.
if (getDataPayload.Inventory.Count > 0)
{
peerDisconnected = !peerBehavior.StartDownloadAsync(getDataPayload).GetAwaiter().GetResult();
}
}
if (peerDisconnected)
{
// Return blocks that were supposed to be assigned to the disconnected peer back to the pending list.
lock (this.lockObject)
{
foreach (int blockHeight in blockHeightsToDownload)
{
InventoryVector inventoryVector = vectors[blockHeight];
this.pendingInventoryVectors.Enqueue(inventoryVector.Hash);
}
}
}
}
this.logger.LogTrace("(-)");
}
