private void SyncBlock(Block block, ISyncPeer syncPeer)
{
if (_logger.IsTrace)
{
_logger.Trace($"{block}");
}
// we do not trust total difficulty from peers
// Parity sends invalid data here and it is equally expensive to validate and to set from null
block.Header.TotalDifficulty = null;
bool isKnownParent = _blockTree.IsKnownBlock(block.Number - 1, block.ParentHash);
if (isKnownParent)
{
if (!_blockValidator.ValidateSuggestedBlock(block))
{
string message = $"Peer {syncPeer?.Node:c} sent an invalid block";
if (_logger.IsDebug)
{
_logger.Debug(message);
}
lock (_recentlySuggested)
{
_recentlySuggested.Delete(block.Hash);
}
throw new EthSyncException(message);
}
AddBlockResult result = _blockTree.SuggestBlock(block, true);
if (_logger.IsTrace)
{
_logger.Trace($"{block.Hash} ({block.Number}) adding result is {result}");
}
}
// TODO: now it should be done by sync peer pool?
// // do not change to if..else
// // there are some rare cases when it did not work...
// // do not remember why
// if (result == AddBlockResult.UnknownParent)
// {
// _synchronizer.RequestSynchronization(SyncTriggerType.NewBlock);
// }
}
private void SyncBlock(Block block, ISyncPeer syncPeer)
{
if (_logger.IsTrace)
{
_logger.Trace($"{block}");
}
// we do not trust total difficulty from peers
// Parity sends invalid data here and it is equally expensive to validate and to set from null
block.Header.TotalDifficulty = null;
bool isKnownParent = _blockTree.IsKnownBlock(block.Number - 1, block.ParentHash);
if (isKnownParent)
{
if (!_blockValidator.ValidateSuggestedBlock(block))
{
string message = $"Peer {syncPeer?.Node:c} sent an invalid block";
if (_logger.IsDebug)
{
_logger.Debug(message);
}
lock (_recentlySuggested)
{
_recentlySuggested.Delete(block.Hash);
}
throw new EthSyncException(message);
}
AddBlockResult result = _blockTree.SuggestBlock(block);
if (_logger.IsTrace)
{
_logger.Trace($"{block.Hash} ({block.Number}) adding result is {result}");
}
}
}
public bool IsBeaconSyncHeadersFinished()
{
BlockHeader?lowestInsertedBeaconHeader = _blockTree.LowestInsertedBeaconHeader;
bool chainMerged =
((lowestInsertedBeaconHeader?.Number ?? 0) - 1) <= (_blockTree.BestSuggestedHeader?.Number ?? long.MaxValue) &&
lowestInsertedBeaconHeader != null &&
_blockTree.IsKnownBlock(lowestInsertedBeaconHeader.Number - 1, lowestInsertedBeaconHeader.ParentHash !);
bool finished = lowestInsertedBeaconHeader == null ||
lowestInsertedBeaconHeader.Number <= _syncConfig.PivotNumberParsed + 1 ||
chainMerged;
if (_logger.IsTrace)
{
_logger.Trace(
$"IsBeaconSyncHeadersFinished: {finished}," +
$" BeaconPivotExists: {_beaconPivot.BeaconPivotExists()}," +
$" LowestInsertedBeaconHeaderNumber: {_blockTree.LowestInsertedBeaconHeader?.Number}," +
$" BestSuggestedHeader: {_blockTree.BestSuggestedHeader?.Number}," +
$" ChainMerged: {chainMerged}," +
$" BeaconPivot: {_beaconPivot.PivotNumber}," +
$" BeaconPivotDestinationNumber: {_beaconPivot.PivotDestinationNumber}");
}
return(finished);
}
public async Task <long> DownloadBlocks(PeerInfo bestPeer, int numberOfLatestBlocksToBeIgnored, CancellationToken cancellation, BlockDownloaderOptions options = BlockDownloaderOptions.Process)
{
IReceiptsRecovery receiptsRecovery = new ReceiptsRecovery();
if (bestPeer == null)
{
string message = $"Not expecting best peer to be null inside the {nameof(BlockDownloader)}";
if (_logger.IsError)
{
_logger.Error(message);
}
throw new ArgumentNullException(message);
}
bool downloadReceipts = (options & BlockDownloaderOptions.DownloadReceipts) == BlockDownloaderOptions.DownloadReceipts;
bool shouldProcess = (options & BlockDownloaderOptions.Process) == BlockDownloaderOptions.Process;
bool shouldMoveToMain = (options & BlockDownloaderOptions.MoveToMain) == BlockDownloaderOptions.MoveToMain;
int blocksSynced = 0;
int ancestorLookupLevel = 0;
long currentNumber = Math.Max(0, Math.Min(_blockTree.BestKnownNumber, bestPeer.HeadNumber - 1));
// pivot number - 6 for uncle validation
// long currentNumber = Math.Max(Math.Max(0, pivotNumber - 6), Math.Min(_blockTree.BestKnownNumber, bestPeer.HeadNumber - 1));
while (bestPeer.TotalDifficulty > (_blockTree.BestSuggestedHeader?.TotalDifficulty ?? 0) && currentNumber <= bestPeer.HeadNumber)
{
if (_logger.IsDebug)
{
_logger.Debug($"Continue full sync with {bestPeer} (our best {_blockTree.BestKnownNumber})");
}
long blocksLeft = bestPeer.HeadNumber - currentNumber - numberOfLatestBlocksToBeIgnored;
int headersToRequest = (int)Math.Min(blocksLeft + 1, _syncBatchSize.Current);
if (headersToRequest <= 1)
{
break;
}
headersToRequest = Math.Min(headersToRequest, bestPeer.MaxHeadersPerRequest());
if (_logger.IsTrace)
{
_logger.Trace($"Full sync request {currentNumber}+{headersToRequest} to peer {bestPeer} with {bestPeer.HeadNumber} blocks. Got {currentNumber} and asking for {headersToRequest} more.");
}
if (cancellation.IsCancellationRequested)
{
return(blocksSynced); // check before every heavy operation
}
BlockHeader[] headers = await RequestHeaders(bestPeer, cancellation, currentNumber, headersToRequest);
BlockDownloadContext context = new BlockDownloadContext(_specProvider, bestPeer, headers, downloadReceipts, receiptsRecovery);
if (cancellation.IsCancellationRequested)
{
return(blocksSynced); // check before every heavy operation
}
await RequestBodies(bestPeer, cancellation, context);
if (downloadReceipts)
{
if (cancellation.IsCancellationRequested)
{
return(blocksSynced); // check before every heavy operation
}
await RequestReceipts(bestPeer, cancellation, context);
}
_sinceLastTimeout++;
if (_sinceLastTimeout > 2)
{
_syncBatchSize.Expand();
}
Block[] blocks = context.Blocks;
Block blockZero = blocks[0];
if (context.FullBlocksCount > 0)
{
bool parentIsKnown = _blockTree.IsKnownBlock(blockZero.Number - 1, blockZero.ParentHash);
if (!parentIsKnown)
{
ancestorLookupLevel++;
if (ancestorLookupLevel >= _ancestorJumps.Length)
{
if (_logger.IsWarn)
{
_logger.Warn($"Could not find common ancestor with {bestPeer}");
}
throw new EthSynchronizationException("Peer with inconsistent chain in sync");
}
int ancestorJump = _ancestorJumps[ancestorLookupLevel] - _ancestorJumps[ancestorLookupLevel - 1];
currentNumber = currentNumber >= ancestorJump ? (currentNumber - ancestorJump) : 0L;
continue;
}
}
ancestorLookupLevel = 0;
for (int blockIndex = 0; blockIndex < context.FullBlocksCount; blockIndex++)
{
if (cancellation.IsCancellationRequested)
{
if (_logger.IsTrace)
{
_logger.Trace("Peer sync cancelled");
}
break;
}
Block currentBlock = blocks[blockIndex];
if (_logger.IsTrace)
{
_logger.Trace($"Received {currentBlock} from {bestPeer}");
}
// can move this to block tree now?
if (!_blockValidator.ValidateSuggestedBlock(currentBlock))
{
throw new EthSynchronizationException($"{bestPeer} sent an invalid block {currentBlock.ToString(Block.Format.Short)}.");
}
if (HandleAddResult(bestPeer, currentBlock.Header, blockIndex == 0, _blockTree.SuggestBlock(currentBlock, shouldProcess)))
{
if (downloadReceipts)
{
for (int receiptIndex = 0; receiptIndex < (context.ReceiptsForBlocks[blockIndex]?.Length ?? 0); receiptIndex++)
{
_receiptStorage.Add(context.ReceiptsForBlocks[blockIndex][receiptIndex], true);
}
}
blocksSynced++;
}
if (shouldMoveToMain)
{
_blockTree.UpdateMainChain(new[] { currentBlock }, false);
}
currentNumber += 1;
}
if (blocksSynced > 0)
{
_syncReport.FullSyncBlocksDownloaded.Update(_blockTree.BestSuggestedHeader?.Number ?? 0);
_syncReport.FullSyncBlocksKnown = bestPeer.HeadNumber;
}
else
{
break;
}
}
return(blocksSynced);
}
public bool IsKnownBlock(long number, Keccak blockHash)
{
return(_blockTree.IsKnownBlock(number, blockHash));
}
public void Does_not_request_peer_refresh_on_known_hints()
{
_blockTree.IsKnownBlock(1, TestItem.KeccakA).ReturnsForAnyArgs(true);
_syncServer.HintBlock(TestItem.KeccakA, 1, _nodeWhoSentTheBlock);
_peerPool.DidNotReceiveWithAnyArgs().RefreshTotalDifficulty(null, null);
}
public bool IsKnownBlock(long number, Keccak blockHash) => _wrapped.IsKnownBlock(number, blockHash);
public bool IsKnownBlock(long number, Keccak blockHash)
{
return(_wrapped.IsKnownBlock(number, blockHash));
}
public async Task <long> DownloadBlocks(PeerInfo bestPeer, int newBlocksToSkip, CancellationToken cancellation, bool shouldProcess = true)
{
if (bestPeer == null)
{
string message = $"Not expecting best peer to be null inside the {nameof(BlockDownloader)}";
_logger.Error(message);
throw new ArgumentNullException(message);
}
int blocksSynced = 0;
int ancestorLookupLevel = 0;
long currentNumber = Math.Max(0, Math.Min(_blockTree.BestKnownNumber, bestPeer.HeadNumber - 1));
while (bestPeer.TotalDifficulty > (_blockTree.BestSuggestedHeader?.TotalDifficulty ?? 0) && currentNumber <= bestPeer.HeadNumber)
{
if (_logger.IsDebug)
{
_logger.Debug($"Continue full sync with {bestPeer} (our best {_blockTree.BestKnownNumber})");
}
if (ancestorLookupLevel > MaxReorganizationLength)
{
if (_logger.IsWarn)
{
_logger.Warn($"Could not find common ancestor with {bestPeer}");
}
throw new EthSynchronizationException("Peer with inconsistent chain in sync");
}
long blocksLeft = bestPeer.HeadNumber - currentNumber - newBlocksToSkip;
int blocksToRequest = (int)BigInteger.Min(blocksLeft + 1, _syncBatchSize.Current);
if (blocksToRequest <= 1)
{
break;
}
if (_logger.IsTrace)
{
_logger.Trace($"Full sync request {currentNumber}+{blocksToRequest} to peer {bestPeer} with {bestPeer.HeadNumber} blocks. Got {currentNumber} and asking for {blocksToRequest} more.");
}
var headers = await RequestHeaders(bestPeer, cancellation, currentNumber, blocksToRequest);
List <Keccak> hashes = new List <Keccak>();
Dictionary <Keccak, BlockHeader> headersByHash = new Dictionary <Keccak, BlockHeader>();
for (int i = 1; i < headers.Length; i++)
{
if (headers[i] == null)
{
break;
}
hashes.Add(headers[i].Hash);
headersByHash[headers[i].Hash] = headers[i];
}
Task <BlockBody[]> bodiesTask = bestPeer.SyncPeer.GetBlocks(hashes.ToArray(), cancellation);
await bodiesTask.ContinueWith(t =>
{
if (t.IsFaulted)
{
_sinceLastTimeout = 0;
if (t.Exception?.InnerException is TimeoutException ||
(t.Exception?.InnerExceptions.Any(x => x is TimeoutException) ?? false) ||
(t.Exception?.InnerExceptions.Any(x => x.InnerException is TimeoutException) ?? false))
{
if (_logger.IsTrace)
{
_logger.Error("Failed to retrieve bodies when synchronizing (Timeout)", bodiesTask.Exception);
}
_syncBatchSize.Shrink();
}
else
{
if (_logger.IsError)
{
_logger.Error("Failed to retrieve bodies when synchronizing", bodiesTask.Exception);
}
}
throw new EthSynchronizationException("Bodies task faulted.", bodiesTask.Exception);
}
});
if (bodiesTask.IsCanceled)
{
return(blocksSynced);
}
BlockBody[] bodies = bodiesTask.Result;
Block[] blocks = new Block[bodies.Length];
for (int i = 0; i < bodies.Length; i++)
{
BlockBody body = bodies[i];
if (body == null)
{
// TODO: this is how it used to be... I do not want to touch it without extensive testing
throw new EthSynchronizationException($"{bestPeer} sent an empty body for {blocks[i].ToString(Block.Format.Short)}.");
}
blocks[i] = new Block(null, body);
}
_sinceLastTimeout++;
if (_sinceLastTimeout > 2)
{
_syncBatchSize.Expand();
}
for (int i = 0; i < blocks.Length; i++)
{
blocks[i].Header = headersByHash[hashes[i]];
}
if (blocks.Length > 0)
{
bool parentIsKnown = _blockTree.IsKnownBlock(blocks[0].Number - 1, blocks[0].ParentHash);
if (!parentIsKnown)
{
ancestorLookupLevel += _syncBatchSize.Current;
currentNumber = currentNumber >= _syncBatchSize.Current ? (currentNumber - _syncBatchSize.Current) : 0L;
continue;
}
}
for (int i = 0; i < blocks.Length; i++)
{
if (cancellation.IsCancellationRequested)
{
if (_logger.IsTrace)
{
_logger.Trace("Peer sync cancelled");
}
break;
}
if (_logger.IsTrace)
{
_logger.Trace($"Received {blocks[i]} from {bestPeer}");
}
// can move this to block tree now?
if (!_blockValidator.ValidateSuggestedBlock(blocks[i]))
{
throw new EthSynchronizationException($"{bestPeer} sent an invalid block {blocks[i].ToString(Block.Format.Short)}.");
}
if (HandleAddResult(blocks[i].Header, i == 0, _blockTree.SuggestBlock(blocks[i], shouldProcess)))
{
blocksSynced++;
}
currentNumber = currentNumber + 1;
}
if (blocksSynced > 0)
{
_syncStats.Update(_blockTree.BestSuggestedHeader?.Number ?? 0, bestPeer.HeadNumber, 1);
}
}
return(blocksSynced);
}
public void AddNewBlock(Block block, Node nodeWhoSentTheBlock)
{
if (block.Number < _pivotNumber)
{
return;
}
if (block.TotalDifficulty == null)
{
throw new InvalidOperationException("Cannot add a block with unknown total difficulty");
}
_pool.TryFind(nodeWhoSentTheBlock.Id, out PeerInfo peerInfo);
if (peerInfo == null)
{
string errorMessage = $"Received a new block from an unknown peer {nodeWhoSentTheBlock:c} {nodeWhoSentTheBlock.Id} {_pool.PeerCount}";
if (_logger.IsDebug)
{
_logger.Debug(errorMessage);
}
return;
}
if ((block.TotalDifficulty ?? 0) > peerInfo.TotalDifficulty)
{
if (_logger.IsTrace)
{
_logger.Trace($"ADD NEW BLOCK Updating header of {peerInfo} from {peerInfo.HeadNumber} {peerInfo.TotalDifficulty} to {block.Number} {block.TotalDifficulty}");
}
peerInfo.HeadNumber = block.Number;
peerInfo.HeadHash = block.Hash;
peerInfo.TotalDifficulty = block.TotalDifficulty ?? peerInfo.TotalDifficulty;
}
if ((block.TotalDifficulty ?? 0) < _blockTree.BestSuggestedHeader.TotalDifficulty)
{
return;
}
lock (_recentlySuggested)
{
if (_recentlySuggested.Get(block.Hash) != null)
{
return;
}
_recentlySuggested.Set(block.Hash, _dummyValue);
}
if (block.Number > _blockTree.BestKnownNumber + 8)
{
// ignore blocks when syncing in a simple non-locking way
_synchronizer.RequestSynchronization(SyncTriggerType.NewDistantBlock);
return;
}
if (block.Number < (_blockTree.Head?.Number ?? 0) - 512)
{
return;
}
if (_logger.IsTrace)
{
_logger.Trace($"Adding new block {block.ToString(Block.Format.Short)}) from {nodeWhoSentTheBlock:c}");
}
_sealValidator.HintValidationRange(_sealValidatorUserGuid, block.Number - 128, block.Number + 1024);
if (!_sealValidator.ValidateSeal(block.Header, true))
{
if (_logger.IsDebug)
{
_logger.Debug($"Peer {peerInfo.SyncPeer?.Node:c} sent a block with an invalid seal");
}
throw new EthSynchronizationException("Peer sent a block with an invalid seal");
}
if (block.Number <= _blockTree.BestKnownNumber + 1)
{
if (_logger.IsInfo)
{
string authorString = block.Author == null ? null : "sealed by " + (KnownAddresses.GoerliValidators.ContainsKey(block.Author) ? KnownAddresses.GoerliValidators[block.Author] : block.Author?.ToString());
if (authorString == null)
{
authorString = block.Beneficiary == null ? string.Empty : "mined by " + (KnownAddresses.KnownMiners.ContainsKey(block.Beneficiary) ? KnownAddresses.KnownMiners[block.Beneficiary] : block.Beneficiary?.ToString());
}
if (_logger.IsInfo)
{
_logger.Info($"Discovered a new block {string.Empty.PadLeft(9 - block.Number.ToString().Length, ' ')}{block.ToString(Block.Format.HashNumberAndTx)} {authorString}, sent by {nodeWhoSentTheBlock:s}");
}
}
if (_logger.IsTrace)
{
_logger.Trace($"{block}");
}
if (_synchronizer.SyncMode == SyncMode.Full)
{
AddBlockResult result = AddBlockResult.UnknownParent;
bool isKnownParent = _blockTree.IsKnownBlock(block.Number - 1, block.ParentHash);
if (isKnownParent)
{
if (!_blockValidator.ValidateSuggestedBlock(block))
{
if (_logger.IsDebug)
{
_logger.Debug($"Peer {peerInfo.SyncPeer?.Node:c} sent an invalid block");
}
throw new EthSynchronizationException("Peer sent an invalid block");
}
result = _blockTree.SuggestBlock(block, true);
if (_logger.IsTrace)
{
_logger.Trace($"{block.Hash} ({block.Number}) adding result is {result}");
}
}
if (result == AddBlockResult.UnknownParent)
{
_synchronizer.RequestSynchronization(SyncTriggerType.Reorganization);
}
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Received a block {block.Hash} ({block.Number}) from {nodeWhoSentTheBlock} - need to resync");
}
_synchronizer.RequestSynchronization(SyncTriggerType.NewNearBlock);
}
}
