/// <summary>Recalculates quality score of a peer or all peers if given peer has the best upload speed.</summary>
/// <remarks>This method has to be protected by <see cref="peerLock"/>.</remarks>
/// <param name="pullerBehavior">The puller behavior of a peer which quality score should be recalculated.</param>
/// <param name="peerId">ID of a peer which behavior is passed.</param>
private void RecalculateQualityScoreLocked(IBlockPullerBehavior pullerBehavior, int peerId)
{
this.logger.LogTrace("({0}:{1})", nameof(peerId), peerId);
// Now decide if we need to recalculate quality score for all peers or just for this one.
long bestSpeed = this.pullerBehaviorsByPeerId.Max(x => x.Value.SpeedBytesPerSecond);
long adjustedBestSpeed = bestSpeed;
if (!this.isIbd && (adjustedBestSpeed > PeerSpeedLimitWhenNotInIbdBytesPerSec))
{
adjustedBestSpeed = PeerSpeedLimitWhenNotInIbdBytesPerSec;
}
if (pullerBehavior.SpeedBytesPerSecond != bestSpeed)
{
// This is not the best peer. Recalculate it's score only.
pullerBehavior.RecalculateQualityScore(adjustedBestSpeed);
}
else
{
this.logger.LogTrace("Peer ID {0} is the fastest peer. Recalculating quality score of all peers.", peerId);
// This is the best peer. Recalculate quality score for everyone.
foreach (IBlockPullerBehavior peerPullerBehavior in this.pullerBehaviorsByPeerId.Values)
{
peerPullerBehavior.RecalculateQualityScore(adjustedBestSpeed);
}
}
this.logger.LogTrace("(-)");
}
/// <inheritdoc />
public void PushBlock(uint256 blockHash, Block block, int peerId)
{
this.logger.LogTrace("({0}:'{1}',{2}:{3})", nameof(blockHash), blockHash, nameof(peerId), peerId);
AssignedDownload assignedDownload;
lock (this.assignedLock)
{
if (!this.assignedDownloadsByHash.TryGetValue(blockHash, out assignedDownload))
{
this.logger.LogTrace("(-)[BLOCK_NOT_REQUESTED]");
return;
}
this.logger.LogTrace("Assignment '{0}' for peer ID {1} was delivered by peer ID {2}.", blockHash, assignedDownload.PeerId, peerId);
if (assignedDownload.PeerId != peerId)
{
this.logger.LogTrace("(-)[WRONG_PEER_DELIVERED]");
return;
}
this.RemoveAssignedDownloadLocked(assignedDownload);
}
double deliveredInSeconds = (this.dateTimeProvider.GetUtcNow() - assignedDownload.AssignedTime).TotalSeconds;
this.logger.LogTrace("Peer {0} delivered block '{1}' in {2} seconds.", assignedDownload.PeerId, blockHash, deliveredInSeconds);
lock (this.peerLock)
{
// Add peer sample.
IBlockPullerBehavior pullerBehavior = this.pullerBehaviorsByPeerId[peerId];
pullerBehavior.AddSample(block.BlockSize.Value, deliveredInSeconds);
// Recalculate quality score.
this.RecalculateQualityScoreLocked(pullerBehavior, peerId);
}
lock (this.queueLock)
{
this.averageBlockSizeBytes.AddSample(block.BlockSize.Value);
this.RecalculateMaxBlocksBeingDownloadedLocked();
this.processQueuesSignal.Set();
}
this.onDownloadedCallback(blockHash, block);
this.logger.LogTrace("(-)");
}
/// <summary>
/// Adds new time of a block to the list of times of recently downloaded blocks.
/// </summary>
/// <param name="peer">Peer that downloaded the block.</param>
/// <param name="blockDownloadTimeMs">Time in milliseconds it took to download the block from the peer.</param>
/// <param name="blockSize">Size of the downloaded block in bytes.</param>
public void AddSample(IBlockPullerBehavior peer, long blockDownloadTimeMs, int blockSize)
{
this.logger.LogTrace($"({nameof(peer)}:{peer.GetHashCode():x},{nameof(blockDownloadTimeMs)}:{blockDownloadTimeMs},{nameof(blockSize)}:{blockSize})");
double timePerKb = 1024.0 * (double)blockDownloadTimeMs / (double)blockSize;
if (timePerKb < 0.00001)
{
timePerKb = 0.00001;
}
lock (this.lockObject)
{
// If we reached the maximum number of samples, we need to remove oldest sample.
if (this.samplesCount == this.samples.Length)
{
PeerSample oldSample = this.samples[this.samplesIndex];
this.samplesSum -= oldSample.timePerKb;
this.peerReferenceCounter[oldSample.peer]--;
if (this.peerReferenceCounter[oldSample.peer] == 0)
{
this.peerReferenceCounter.Remove(oldSample.peer);
}
}
else
{
this.samplesCount++;
}
// Add new sample to the mix.
this.samples[this.samplesIndex].timePerKb = timePerKb;
this.samples[this.samplesIndex].peer = peer;
this.samplesIndex = (this.samplesIndex + 1) % this.samples.Length;
if (this.peerReferenceCounter.ContainsKey(peer))
{
this.peerReferenceCounter[peer]++;
}
else
{
this.peerReferenceCounter.Add(peer, 1);
}
// Update the sum and the average with the latest data.
this.samplesSum += timePerKb;
this.AverageBlockTimePerKb = this.samplesSum / this.samplesCount;
}
this.logger.LogTrace($"(-):{nameof(this.AverageBlockTimePerKb)}={this.AverageBlockTimePerKb}");
}
/// <summary>
/// Adds new time of a block to the list of times of recently downloaded blocks.
/// </summary>
/// <param name="peer">Peer that downloaded the block.</param>
/// <param name="blockDownloadTimeMs">Time in milliseconds it took to download the block from the peer.</param>
/// <param name="blockSize">Size of the downloaded block in bytes.</param>
public void AddSample(IBlockPullerBehavior peer, long blockDownloadTimeMs, int blockSize)
{
this.logger.LogTrace("({0}:{1:x},{2}:{3},{4}:{5})", nameof(peer), peer.GetHashCode(), nameof(blockDownloadTimeMs), blockDownloadTimeMs, nameof(blockSize), blockSize);
double timePerKb = 1024.0 * (double)blockDownloadTimeMs / (double)blockSize;
if (timePerKb < 0.00001)
{
timePerKb = 0.00001;
}
lock (this.lockObject)
{
// Add new sample to the mix.
PeerSample newSample = new PeerSample();
newSample.TimePerKb = timePerKb;
newSample.Peer = peer;
if (this.peerReferenceCounter.ContainsKey(peer))
{
this.peerReferenceCounter[peer]++;
}
else
{
this.peerReferenceCounter.Add(peer, 1);
}
PeerSample oldSample;
if (this.samples.Add(newSample, out oldSample))
{
// If we reached the maximum number of samples, we need to remove oldest sample.
this.samplesSum -= oldSample.TimePerKb;
this.peerReferenceCounter[oldSample.Peer]--;
if (this.peerReferenceCounter[oldSample.Peer] == 0)
{
this.peerReferenceCounter.Remove(oldSample.Peer);
}
}
// Update the sum and the average with the latest data.
this.samplesSum += timePerKb;
this.AverageBlockTimePerKb = this.samplesSum / this.samples.Count;
}
this.logger.LogTrace("(-):{0}={1}", nameof(this.AverageBlockTimePerKb), this.AverageBlockTimePerKb);
}
public void RecalculateQualityScoreLocked(IBlockPullerBehavior pullerBehavior, int peerId)
{
this.puller.InvokeMethod("RecalculateQualityScoreLocked", pullerBehavior, peerId);
}
/// <summary>Checks if peers failed to deliver important blocks and penalizes them if they did.</summary>
private void CheckStalling()
{
this.logger.LogTrace("()");
int lastImportantHeight = this.chainState.ConsensusTip.Height + ImportantHeightMargin;
this.logger.LogTrace("Blocks up to height {0} are considered to be important.", lastImportantHeight);
var allReleasedAssignments = new List <Dictionary <int, List <ChainedHeader> > >();
lock (this.assignedLock)
{
LinkedListNode <AssignedDownload> current = this.assignedDownloadsSorted.First;
var peerIdsToReassignJobs = new HashSet <int>();
while (current != null)
{
// Since the headers in the linked list are sorted by height after we found first that is
// not important we can assume that the rest of them are not important.
if (current.Value.Header.Height > lastImportantHeight)
{
break;
}
double secondsPassed = (this.dateTimeProvider.GetUtcNow() - current.Value.AssignedTime).TotalSeconds;
// Peer failed to deliver important block.
int peerId = current.Value.PeerId;
current = current.Next;
if (secondsPassed < MaxSecondsToDeliverBlock)
{
continue;
}
// Peer already added to the collection of peers to release and reassign.
if (peerIdsToReassignJobs.Contains(peerId))
{
continue;
}
peerIdsToReassignJobs.Add(peerId);
int assignedCount = this.assignedHeadersByPeerId[peerId].Count;
this.logger.LogDebug("Peer {0} failed to deliver {1} blocks from which some were important.", peerId, assignedCount);
lock (this.peerLock)
{
IBlockPullerBehavior pullerBehavior = this.pullerBehaviorsByPeerId[peerId];
pullerBehavior.Penalize(secondsPassed, assignedCount);
this.RecalculateQualityScoreLocked(pullerBehavior, peerId);
}
}
// Release downloads for selected peers.
foreach (int peerId in peerIdsToReassignJobs)
{
Dictionary <int, List <ChainedHeader> > reassignedAssignmentsByJobId = this.ReleaseAssignmentsLocked(peerId);
allReleasedAssignments.Add(reassignedAssignmentsByJobId);
}
}
if (allReleasedAssignments.Count > 0)
{
lock (this.queueLock)
{
// Reassign all released jobs.
foreach (Dictionary <int, List <ChainedHeader> > released in allReleasedAssignments)
{
this.ReassignAssignmentsLocked(released);
}
// Trigger queue processing in case anything was reassigned.
this.processQueuesSignal.Set();
}
}
this.logger.LogTrace("(-)");
}
/// <summary>Distributes download job's headers to peers that can provide blocks represented by those headers.</summary>
/// <remarks>
/// If some of the blocks from the job can't be provided by any peer those headers will be added to a <param name="failedHashes">.</param>
/// <para>
/// Have to be locked by <see cref="queueLock"/>.
/// </para>
/// <para>
/// Node's quality score is being considered as a weight during the random distribution of the hashes to download among the nodes.
/// </para>
/// </remarks>
/// <param name="downloadJob">Download job to be partially of fully consumed.</param>
/// <param name="failedHashes">List of failed hashes which will be extended in case there is no peer to claim required hash.</param>
/// <param name="emptySlots">Number of empty slots. This is the maximum number of assignments that can be created.</param>
/// <returns>List of downloads that were distributed between the peers.</returns>
private List <AssignedDownload> DistributeHeadersLocked(DownloadJob downloadJob, List <uint256> failedHashes, int emptySlots)
{
this.logger.LogTrace("({0}.{1}:{2},{3}.{4}:{5},{6}:{7})", nameof(downloadJob.Headers), nameof(downloadJob.Headers.Count), downloadJob.Headers.Count,
nameof(failedHashes), nameof(failedHashes.Count), failedHashes.Count, nameof(emptySlots), emptySlots);
var newAssignments = new List <AssignedDownload>();
HashSet <IBlockPullerBehavior> peerBehaviors;
lock (this.peerLock)
{
peerBehaviors = new HashSet <IBlockPullerBehavior>(this.pullerBehaviorsByPeerId.Values);
}
bool jobFailed = false;
if (peerBehaviors.Count == 0)
{
this.logger.LogDebug("There are no peers that can participate in download job distribution! Job ID {0} failed.", downloadJob.Id);
jobFailed = true;
}
int lastSucceededIndex = -1;
for (int index = 0; (index < downloadJob.Headers.Count) && (index < emptySlots) && !jobFailed; index++)
{
ChainedHeader header = downloadJob.Headers[index];
while (!jobFailed)
{
// Weighted random selection based on the peer's quality score.
double sumOfQualityScores = peerBehaviors.Sum(x => x.QualityScore);
double scoreToReachPeer = this.random.NextDouble() * sumOfQualityScores;
IBlockPullerBehavior selectedBehavior = peerBehaviors.First();
foreach (IBlockPullerBehavior peerBehavior in peerBehaviors)
{
if (peerBehavior.QualityScore >= scoreToReachPeer)
{
selectedBehavior = peerBehavior;
break;
}
scoreToReachPeer -= peerBehavior.QualityScore;
}
INetworkPeer attachedPeer = selectedBehavior.AttachedPeer;
// Behavior's tip can't be null because we only have behaviors inserted in the behaviors structure after the tip is set.
if ((attachedPeer != null) && (selectedBehavior.Tip.FindAncestorOrSelf(header) != null))
{
int peerId = attachedPeer.Connection.Id;
// Assign to this peer.
newAssignments.Add(new AssignedDownload()
{
PeerId = peerId,
JobId = downloadJob.Id,
AssignedTime = this.dateTimeProvider.GetUtcNow(),
Header = header
});
lastSucceededIndex = index;
this.logger.LogTrace("Block '{0}' was assigned to peer ID {1}.", header.HashBlock, peerId);
break;
}
else
{
// Peer doesn't claim this header.
peerBehaviors.Remove(selectedBehavior);
if (peerBehaviors.Count != 0)
{
continue;
}
jobFailed = true;
this.logger.LogDebug("Job {0} failed because there is no peer claiming header '{1}'.", downloadJob.Id, header);
}
}
}
if (!jobFailed)
{
downloadJob.Headers.RemoveRange(0, lastSucceededIndex + 1);
}
else
{
int removeFrom = (lastSucceededIndex == -1) ? 0 : lastSucceededIndex + 1;
IEnumerable <uint256> failed = downloadJob.Headers.GetRange(removeFrom, downloadJob.Headers.Count - removeFrom).Select(x => x.HashBlock);
failedHashes.AddRange(failed);
downloadJob.Headers.Clear();
}
this.logger.LogTrace("(-):*.{0}={1},{2}.{3}={4}", nameof(newAssignments.Count), newAssignments.Count, nameof(failedHashes), nameof(failedHashes.Count), failedHashes.Count);
return(newAssignments);
}
