private AddBlockResult InsertHeader(BlockHeader header)
{
if (header.IsGenesis)
{
return(AddBlockResult.AlreadyKnown);
}
AddBlockResult insertOutcome = _blockTree.Insert(header);
if (insertOutcome == AddBlockResult.Added || insertOutcome == AddBlockResult.AlreadyKnown)
{
ulong nextHeaderDiff = 0;
_nextHeaderHash = header.ParentHash;
if (_expectedDifficultyOverride?.TryGetValue(header.Number, out nextHeaderDiff) == true)
{
_nextHeaderDiff = nextHeaderDiff;
}
else
{
_nextHeaderDiff = (header.TotalDifficulty ?? 0) - header.Difficulty;
}
}
return(insertOutcome);
}
public ProcessingTestContext Suggested(Block block)
{
AddBlockResult result = _blockTree.SuggestBlock(block);
if (result != AddBlockResult.Added)
{
Console.WriteLine($"Finished waiting for {block.ToString(Block.Format.Short)} as block was ignored");
_resetEvent.Set();
}
return(this);
}
private async Task ProcessBlock(Block block)
{
try
{
if (!_blockValidator.ValidateSuggestedBlock(block))
{
if (_logger.IsInfo)
{
_logger.Info($"Invalid block {block}");
}
return;
}
AddBlockResult result = await _blockTree.SuggestBlockAsync(block);
if (result != AddBlockResult.Added && result != AddBlockResult.AlreadyKnown)
{
if (_logger.IsError)
{
_logger.Error($"Cannot add block {block} to the blockTree, add result {result}");
}
return;
}
try
{
if (_tracer.Trace(block, NullBlockTracer.Instance) is null)
{
return;
}
}
catch (Exception ex)
{
if (_logger.IsError)
{
_logger.Error($"Failed to process block {block}", ex);
}
return;
}
if (_logger.IsInfo)
{
_logger.Info(
$"HIVE suggested {block.ToString(Block.Format.Short)}, now best suggested header {_blockTree.BestSuggestedHeader}, head {_blockTree.Head?.Header?.ToString(BlockHeader.Format.Short)}");
}
await WaitForBlockProcessing(_resetEvent);
}
catch (Exception e)
{
_logger.Error($"HIVE Invalid block: {block.Hash}, ignoring. ", e);
}
}
public ProcessingTestContext Suggested(BlockHeader block)
{
AddBlockResult result = _blockTree.SuggestHeader(block);
if (result != AddBlockResult.Added)
{
_logger.Info($"Finished waiting for {block.ToString(BlockHeader.Format.Short)} as block was ignored");
_resetEvent.Set();
}
return(this);
}
private bool HandleAddResult(BlockHeader block, bool isFirstInBatch, AddBlockResult addResult)
{
switch (addResult)
{
// this generally should not happen as there is a consistency check before
case AddBlockResult.UnknownParent:
{
if (_logger.IsTrace)
{
_logger.Trace($"Block/header {block.Number} ignored (unknown parent)");
}
if (isFirstInBatch)
{
const string message = "Peer sent orphaned blocks/headers inside the batch";
_logger.Error(message);
throw new EthSynchronizationException(message);
}
else
{
const string message = "Peer sent an inconsistent batch of blocks/headers";
_logger.Error(message);
throw new EthSynchronizationException(message);
}
}
case AddBlockResult.CannotAccept:
throw new EthSynchronizationException("Block tree rejected block/header");
case AddBlockResult.InvalidBlock:
throw new EthSynchronizationException("Peer sent an invalid block/header");
case AddBlockResult.Added:
if (_logger.IsTrace)
{
_logger.Trace($"Block/header {block.Number} suggested for processing");
}
return(true);
case AddBlockResult.AlreadyKnown:
if (_logger.IsTrace)
{
_logger.Trace($"Block/header {block.Number} skipped - already known");
}
return(false);
default:
throw new NotImplementedException($"Unknown {nameof(AddBlockResult)} {addResult}");
}
}
public void Can_start_insert_pivot_block_with_correct_pointers()
{
(BlockTree notSyncedTree, BlockTree syncedTree) = BuildBlockTrees(10, 20);
Block?beaconBlock = syncedTree.FindBlock(14, BlockTreeLookupOptions.None);
BlockTreeInsertHeaderOptions insertHeaderOption = BlockTreeInsertHeaderOptions.BeaconBlockInsert;
AddBlockResult insertResult = notSyncedTree.Insert(beaconBlock, BlockTreeInsertBlockOptions.SaveHeader, insertHeaderOption);
Assert.AreEqual(AddBlockResult.Added, insertResult);
Assert.AreEqual(9, notSyncedTree.BestKnownNumber);
Assert.AreEqual(9, notSyncedTree.BestSuggestedHeader !.Number);
Assert.AreEqual(9, notSyncedTree.Head !.Number);
Assert.AreEqual(9, notSyncedTree.BestSuggestedBody !.Number);
Assert.AreEqual(14, notSyncedTree.BestKnownBeaconNumber);
Assert.AreEqual(14, notSyncedTree.BestSuggestedBeaconHeader !.Number);
}
public void Can_insert_beacon_headers()
{
(BlockTree notSyncedTree, BlockTree syncedTree) = BuildBlockTrees(10, 20);
Block?beaconBlock = syncedTree.FindBlock(14, BlockTreeLookupOptions.None);
BlockTreeInsertHeaderOptions headerOptions = BlockTreeInsertHeaderOptions.BeaconBlockInsert;
AddBlockResult insertResult = notSyncedTree.Insert(beaconBlock, BlockTreeInsertBlockOptions.SaveHeader, headerOptions);
for (int i = 13; i > 9; --i)
{
BlockHeader? beaconHeader = syncedTree.FindHeader(i, BlockTreeLookupOptions.None);
AddBlockResult insertOutcome = notSyncedTree.Insert(beaconHeader !, headerOptions);
Assert.AreEqual(insertOutcome, insertResult);
}
}
private AddBlockResult InsertHeader(BlockHeader header)
{
if (header.IsGenesis)
{
return(AddBlockResult.AlreadyKnown);
}
AddBlockResult insertOutcome = _blockTree.Insert(header);
if (insertOutcome == AddBlockResult.Added || insertOutcome == AddBlockResult.AlreadyKnown)
{
_nextHeaderHash = header.ParentHash;
_nextHeaderDiff = (header.TotalDifficulty ?? 0) - header.Difficulty;
}
return(insertOutcome);
}
private void SyncBlock(Block block, ISyncPeer syncPeer)
{
if (_logger.IsTrace)
{
_logger.Trace($"{block}");
}
// we do not trust total difficulty from peers
// Parity sends invalid data here and it is equally expensive to validate and to set from null
block.Header.TotalDifficulty = null;
bool isKnownParent = _blockTree.IsKnownBlock(block.Number - 1, block.ParentHash);
if (isKnownParent)
{
if (!_blockValidator.ValidateSuggestedBlock(block))
{
string message = $"Peer {syncPeer?.Node:c} sent an invalid block";
if (_logger.IsDebug)
{
_logger.Debug(message);
}
lock (_recentlySuggested)
{
_recentlySuggested.Delete(block.Hash);
}
throw new EthSyncException(message);
}
AddBlockResult result = _blockTree.SuggestBlock(block, true);
if (_logger.IsTrace)
{
_logger.Trace($"{block.Hash} ({block.Number}) adding result is {result}");
}
}
// TODO: now it should be done by sync peer pool?
// // do not change to if..else
// // there are some rare cases when it did not work...
// // do not remember why
// if (result == AddBlockResult.UnknownParent)
// {
// _synchronizer.RequestSynchronization(SyncTriggerType.NewBlock);
// }
}
public void After_removing_invalid_block_will_not_accept_it_again()
{
MemDb blocksDb = new MemDb();
MemDb blockInfosDb = new MemDb();
BlockTree tree = new BlockTree(blocksDb, blockInfosDb, MainNetSpecProvider.Instance, NullTransactionPool.Instance, LimboLogs.Instance);
Block block0 = Build.A.Block.WithNumber(0).WithDifficulty(1).TestObject;
Block block1 = Build.A.Block.WithNumber(1).WithDifficulty(2).WithParent(block0).TestObject;
Block block2 = Build.A.Block.WithNumber(2).WithDifficulty(3).WithParent(block1).TestObject;
Block block3 = Build.A.Block.WithNumber(3).WithDifficulty(4).WithParent(block2).TestObject;
tree.SuggestBlock(block0);
tree.SuggestBlock(block1);
tree.SuggestBlock(block2);
tree.SuggestBlock(block3);
tree.DeleteInvalidBlock(block1);
AddBlockResult result = tree.SuggestBlock(block1);
Assert.AreEqual(AddBlockResult.InvalidBlock, result);
}
public BlockTreeBuilder OfChainLength(out Block headBlock, int chainLength, int splitVariant = 0, int splitFrom = 0, params Address[] blockBeneficiaries)
{
Block current = _genesisBlock;
headBlock = _genesisBlock;
bool skipGenesis = TestObjectInternal.Genesis != null;
for (int i = 0; i < chainLength; i++)
{
Address beneficiary = blockBeneficiaries.Length == 0 ? Address.Zero : blockBeneficiaries[i % blockBeneficiaries.Length];
headBlock = current;
if (_onlyHeaders)
{
if (!(current.IsGenesis && skipGenesis))
{
TestObjectInternal.SuggestHeader(current.Header);
}
Block parent = current;
current = CreateBlock(splitVariant, splitFrom, i, parent, beneficiary);
}
else
{
if (!(current.IsGenesis && skipGenesis))
{
AddBlockResult result = TestObjectInternal.SuggestBlock(current);
Assert.AreEqual(AddBlockResult.Added, result, $"Adding {current.ToString(Block.Format.Short)} at split variant {splitVariant}");
TestObjectInternal.UpdateMainChain(current);
}
Block parent = current;
current = CreateBlock(splitVariant, splitFrom, i, parent, beneficiary);
}
}
return(this);
}
private AddBlockResult InsertHeader(BlockHeader header)
{
if (header.IsGenesis)
{
return(AddBlockResult.AlreadyKnown);
}
AddBlockResult insertOutcome = _blockTree.Insert(header);
if (insertOutcome == AddBlockResult.Added || insertOutcome == AddBlockResult.AlreadyKnown)
{
_nextHeaderHash = header.ParentHash;
_nextHeaderDiff = (header.TotalDifficulty ?? 0) - header.Difficulty;
long parentNumber = header.Number - 1;
if (_dependencies.TryRemove(parentNumber, out HeadersSyncBatch batch))
{
InsertHeaders(batch);
}
}
return(insertOutcome);
}
private void SyncBlock(Block block, ISyncPeer syncPeer)
{
if (_logger.IsTrace)
{
_logger.Trace($"{block}");
}
// we do not trust total difficulty from peers
// Parity sends invalid data here and it is equally expensive to validate and to set from null
block.Header.TotalDifficulty = null;
bool isKnownParent = _blockTree.IsKnownBlock(block.Number - 1, block.ParentHash);
if (isKnownParent)
{
if (!_blockValidator.ValidateSuggestedBlock(block))
{
string message = $"Peer {syncPeer?.Node:c} sent an invalid block";
if (_logger.IsDebug)
{
_logger.Debug(message);
}
lock (_recentlySuggested)
{
_recentlySuggested.Delete(block.Hash);
}
throw new EthSyncException(message);
}
AddBlockResult result = _blockTree.SuggestBlock(block);
if (_logger.IsTrace)
{
_logger.Trace($"{block.Hash} ({block.Number}) adding result is {result}");
}
}
}
public BlockTreeBuilder OfChainLength(out Block headBlock, int chainLength, int splitVariant = 0)
{
Block current = _genesisBlock;
headBlock = _genesisBlock;
bool skipGenesis = TestObjectInternal.Head != null;
for (int i = 0; i < chainLength; i++)
{
headBlock = current;
if (_onlyHeaders)
{
if (!(current.IsGenesis && skipGenesis))
{
TestObjectInternal.SuggestHeader(current.Header);
}
current = Build.A.Block.WithNumber(i + 1).WithParent(current).WithDifficulty(BlockHeaderBuilder.DefaultDifficulty - (ulong)splitVariant).TestObject;
}
else
{
if (!(current.IsGenesis && skipGenesis))
{
AddBlockResult result = TestObjectInternal.SuggestBlock(current);
Assert.AreEqual(AddBlockResult.Added, result, $"Adding {current.ToString(Block.Format.Short)} at split variant {splitVariant}");
TestObjectInternal.UpdateMainChain(current);
}
current = Build.A.Block.WithNumber(i + 1).WithParent(current).WithDifficulty(BlockHeaderBuilder.DefaultDifficulty - (ulong)splitVariant).TestObject;
}
}
return(this);
}
public void Can_fill_beacon_headers_gap()
{
(BlockTree notSyncedTree, BlockTree syncedTree) = BuildBlockTrees(10, 20);
Block?beaconBlock = syncedTree.FindBlock(14, BlockTreeLookupOptions.None);
BlockTreeInsertHeaderOptions headerOptions = BlockTreeInsertHeaderOptions.BeaconBlockInsert;
AddBlockResult insertResult = notSyncedTree.Insert(beaconBlock, BlockTreeInsertBlockOptions.SaveHeader, headerOptions);
for (int i = 13; i > 9; --i)
{
BlockHeader? beaconHeader = syncedTree.FindHeader(i, BlockTreeLookupOptions.None);
AddBlockResult insertOutcome = notSyncedTree.Insert(beaconHeader !, headerOptions);
Assert.AreEqual(AddBlockResult.Added, insertOutcome);
}
for (int i = 10; i < 14; ++i)
{
Block? block = syncedTree.FindBlock(i, BlockTreeLookupOptions.None);
AddBlockResult insertOutcome = notSyncedTree.SuggestBlock(block !);
Assert.AreEqual(AddBlockResult.Added, insertOutcome);
}
Assert.AreEqual(13, notSyncedTree.BestSuggestedBody !.Number);
}
private int InsertHeaders(HeadersSyncBatch batch)
{
if (batch.Response.Length > batch.RequestSize)
{
if (_logger.IsDebug)
{
_logger.Debug($"Peer sent too long response ({batch.Response.Length}) to {batch}");
}
_syncPeerPool.ReportBreachOfProtocol(batch.ResponseSourcePeer, $"response too long ({batch.Response.Length})");
_pending.Enqueue(batch);
return(0);
}
long addedLast = batch.StartNumber - 1;
long addedEarliest = batch.EndNumber + 1;
int skippedAtTheEnd = 0;
for (int i = batch.Response.Length - 1; i >= 0; i--)
{
BlockHeader header = batch.Response[i];
if (header == null)
{
skippedAtTheEnd++;
continue;
}
if (header.Number != batch.StartNumber + i)
{
_syncPeerPool.ReportBreachOfProtocol(batch.ResponseSourcePeer, "inconsistent headers batch");
break;
}
bool isFirst = i == batch.Response.Length - 1 - skippedAtTheEnd;
if (isFirst)
{
BlockHeader lowestInserted = _blockTree.LowestInsertedHeader;
// response does not carry expected data
if (header.Number == lowestInserted?.Number && header.Hash != lowestInserted?.Hash)
{
if (batch.ResponseSourcePeer != null)
{
if (_logger.IsDebug)
{
_logger.Debug($"{batch} - reporting INVALID hash");
}
_syncPeerPool.ReportBreachOfProtocol(batch.ResponseSourcePeer, "first hash inconsistent with request");
}
break;
}
// response needs to be cached until predecessors arrive
if (header.Hash != _nextHeaderHash)
{
if (header.Number == (_blockTree.LowestInsertedHeader?.Number ?? _pivotNumber + 1) - 1)
{
if (_logger.IsDebug)
{
_logger.Debug($"{batch} - ended up IGNORED - different branch - number {header.Number} was {header.Hash} while expected {_nextHeaderHash}");
}
_syncPeerPool.ReportBreachOfProtocol(batch.ResponseSourcePeer, "headers - different branch");
break;
}
if (header.Number == _blockTree.LowestInsertedHeader?.Number)
{
if (_logger.IsDebug)
{
_logger.Debug($"{batch} - ended up IGNORED - different branch");
}
_syncPeerPool.ReportBreachOfProtocol(batch.ResponseSourcePeer, "headers - different branch");
break;
}
if (_dependencies.ContainsKey(header.Number))
{
_pending.Enqueue(batch);
throw new InvalidOperationException($"Only one header dependency expected ({batch})");
}
for (int j = 0; j < batch.Response.Length; j++)
{
BlockHeader current = batch.Response[j];
if (batch.Response[j] != null)
{
addedEarliest = Math.Min(addedEarliest, current.Number);
addedLast = Math.Max(addedLast, current.Number);
}
else
{
break;
}
}
HeadersSyncBatch dependentBatch = BuildDependentBatch(batch, addedLast, addedEarliest);
_dependencies[header.Number] = dependentBatch;
if (_logger.IsDebug)
{
_logger.Debug($"{batch} -> DEPENDENCY {dependentBatch}");
}
// but we cannot do anything with it yet
break;
}
}
else
{
if (header.Hash != batch.Response[i + 1]?.ParentHash)
{
if (batch.ResponseSourcePeer != null)
{
if (_logger.IsDebug)
{
_logger.Debug($"{batch} - reporting INVALID inconsistent");
}
_syncPeerPool.ReportBreachOfProtocol(batch.ResponseSourcePeer, "headers - response not matching request");
}
break;
}
}
header.TotalDifficulty = _nextHeaderDiff;
AddBlockResult addBlockResult = InsertHeader(header);
if (addBlockResult == AddBlockResult.InvalidBlock)
{
if (batch.ResponseSourcePeer != null)
{
if (_logger.IsDebug)
{
_logger.Debug($"{batch} - reporting INVALID bad block");
}
_syncPeerPool.ReportBreachOfProtocol(batch.ResponseSourcePeer, $"invalid header {header.ToString(BlockHeader.Format.Short)}");
}
break;
}
addedEarliest = Math.Min(addedEarliest, header.Number);
addedLast = Math.Max(addedLast, header.Number);
}
int added = (int)(addedLast - addedEarliest + 1);
int leftFillerSize = (int)(addedEarliest - batch.StartNumber);
int rightFillerSize = (int)(batch.EndNumber - addedLast);
if (added + leftFillerSize + rightFillerSize != batch.RequestSize)
{
throw new Exception($"Added {added} + left {leftFillerSize} + right {rightFillerSize} != request size {batch.RequestSize} in {batch}");
}
added = Math.Max(0, added);
if (added < batch.RequestSize)
{
if (added <= 0)
{
batch.Response = null;
_pending.Enqueue(batch);
}
else
{
if (leftFillerSize > 0)
{
HeadersSyncBatch leftFiller = BuildLeftFiller(batch, leftFillerSize);
_pending.Enqueue(leftFiller);
if (_logger.IsDebug)
{
_logger.Debug($"{batch} -> FILLER {leftFiller}");
}
}
if (rightFillerSize > 0)
{
HeadersSyncBatch rightFiller = BuildRightFiller(batch, rightFillerSize);
_pending.Enqueue(rightFiller);
if (_logger.IsDebug)
{
_logger.Debug($"{batch} -> FILLER {rightFiller}");
}
}
}
}
if (added == 0)
{
if (batch.ResponseSourcePeer != null)
{
if (_logger.IsDebug)
{
_logger.Debug($"{batch} - reporting no progress");
}
_syncPeerPool.ReportNoSyncProgress(batch.ResponseSourcePeer, AllocationContexts.Headers);
}
}
if (_blockTree.LowestInsertedHeader != null)
{
_syncReport.FastBlocksHeaders.Update(_pivotNumber - _blockTree.LowestInsertedHeader.Number + 1);
}
if (_logger.IsDebug)
{
_logger.Debug($"LOWEST_INSERTED {_blockTree.LowestInsertedHeader?.Number} | HANDLED {batch}");
}
_syncReport.HeadersInQueue.Update(HeadersInQueue);
return(added);
}
public void AddNewBlock(Block block, NodeId receivedFrom)
{
lock (_isSyncingLock)
{
if (_isSyncing)
{
_logger.Info($"Ignoring new block {block.Hash} while syncing");
return;
}
}
// TODO: validation
if (_logger.IsInfoEnabled)
{
_logger.Info($"Adding new block {block.Hash} ({block.Number}) from {receivedFrom}");
}
bool getValueResult = _peers.TryGetValue(receivedFrom, out PeerInfo peerInfo);
if (!getValueResult)
{
_logger.Error($"Try get value failed on {nameof(PeerInfo)} {receivedFrom}");
}
if (peerInfo == null && _logger.IsErrorEnabled)
{
_logger.Error($"{receivedFrom} peer info is null");
}
if (_logger.IsDebugEnabled)
{
_logger.Debug($"Using {peerInfo}");
}
if (peerInfo == null)
{
string errorMessage = $"unknown synchronization peer {receivedFrom}";
_logger.Error(errorMessage);
throw new InvalidOperationException(errorMessage);
}
peerInfo.NumberAvailable = BigInteger.Max(block.Number, peerInfo.NumberAvailable);
if (block.Number <= BlockTree.BestSuggested.Number + 1)
{
if (_logger.IsDebugEnabled)
{
_logger.Debug($"Suggesting a block {block.Hash} ({block.Number}) from {receivedFrom} with {block.Transactions.Length} transactions");
}
if (_logger.IsTraceEnabled)
{
_logger.Trace("SUGGESTING BLOCK:");
_logger.Trace($"{block}");
}
AddBlockResult result = BlockTree.SuggestBlock(block);
if (result == AddBlockResult.UnknownParent)
{
RunSync();
}
else
{
peerInfo.NumberReceived = block.Number;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"{block.Hash} ({block.Number}) adding result is {result}");
}
}
else if (block.Number > BlockTree.BestSuggested.Number + 1)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Received a block {block.Hash} ({block.Number}) from {receivedFrom} - need to resync");
}
RunSync();
}
else
{
Debug.Fail("above should be covering everything");
}
}
private async Task PeerSyncAsync(CancellationToken token, PeerInfo peerInfo)
{
bool wasCancelled = false;
ISynchronizationPeer peer = peerInfo.Peer;
BigInteger bestNumber = BlockTree.BestSuggested.Number;
const int maxLookup = 64;
int ancestorLookupLevel = 0;
bool isCommonAncestorKnown = false;
while (peerInfo.NumberAvailable > bestNumber && peerInfo.NumberReceived <= peerInfo.NumberAvailable)
{
if (ancestorLookupLevel > maxLookup)
{
throw new InvalidOperationException("Cannot find ancestor"); // TODO: remodel this after full sync test is added
}
if (token.IsCancellationRequested)
{
token.ThrowIfCancellationRequested();
}
if (!isCommonAncestorKnown)
{
// TODO: cases when many peers used for sync and one peer finished sync and then we need resync - we should start from common point and not NumberReceived that may be far in the past
_logger.Info($"Finding common ancestor for {peerInfo.Peer.NodeId}");
isCommonAncestorKnown = true;
}
BigInteger blocksLeft = peerInfo.NumberAvailable - peerInfo.NumberReceived;
// TODO: fault handling on tasks
int blocksToRequest = (int)BigInteger.Min(blocksLeft + 1, BatchSize);
if (_logger.IsDebugEnabled)
{
_logger.Debug($"Sync request to peer with {peerInfo.NumberAvailable} blocks. Got {peerInfo.NumberReceived} and asking for {blocksToRequest} more.");
}
Task <BlockHeader[]> headersTask = peer.GetBlockHeaders(peerInfo.NumberReceived, blocksToRequest, 0, token);
_currentSyncTask = headersTask;
BlockHeader[] headers = await headersTask;
if (_currentSyncTask.IsCanceled)
{
wasCancelled = true;
break;
}
if (_currentSyncTask.IsFaulted)
{
_logger.Error("Failed to retrieve headers when synchronizing", _currentSyncTask.Exception);
throw _currentSyncTask.Exception;
}
if (token.IsCancellationRequested)
{
token.ThrowIfCancellationRequested();
}
List <Keccak> hashes = new List <Keccak>();
Dictionary <Keccak, BlockHeader> headersByHash = new Dictionary <Keccak, BlockHeader>();
for (int i = 1; i < headers.Length; i++)
{
hashes.Add(headers[i].Hash);
headersByHash[headers[i].Hash] = headers[i];
}
Task <Block[]> bodiesTask = peer.GetBlocks(hashes.ToArray(), token);
_currentSyncTask = bodiesTask;
Block[] blocks = await bodiesTask;
if (_currentSyncTask.IsCanceled)
{
wasCancelled = true;
break;
}
if (_currentSyncTask.IsFaulted)
{
_logger.Error("Failed to retrieve bodies when synchronizing", _currentSyncTask.Exception);
throw _currentSyncTask.Exception;
}
ancestorLookupLevel = 0;
for (int i = 0; i < blocks.Length; i++)
{
if (token.IsCancellationRequested)
{
token.ThrowIfCancellationRequested();
}
blocks[i].Header = headersByHash[hashes[i]];
}
if (blocks.Length > 0)
{
Block parent = BlockTree.FindParent(blocks[0]);
if (parent == null)
{
ancestorLookupLevel += BatchSize;
peerInfo.NumberReceived -= BatchSize;
continue;
}
}
// Parity 1.11 non canonical blocks when testing on 27/06
for (int i = 0; i < blocks.Length; i++)
{
if (i != 0 && blocks[i].ParentHash != blocks[i - 1].Hash)
{
throw new EthSynchronizationException("Peer send an inconsistent block list");
}
}
for (int i = 0; i < blocks.Length; i++)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Received {blocks[i]} from {peer.NodeId}");
}
if (_blockValidator.ValidateSuggestedBlock(blocks[i]))
{
AddBlockResult addResult = BlockTree.SuggestBlock(blocks[i]);
if (addResult == AddBlockResult.UnknownParent)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Block {blocks[i].Number} ignored (unknown parent)");
}
if (i == 0)
{
if (_logger.IsWarnEnabled)
{
_logger.Warn("Resyncing split");
}
peerInfo.NumberReceived -= 1;
var syncTask = Task.Run(() => PeerSyncAsync(_syncCancellationTokenSource.Token, peerInfo), _syncCancellationTokenSource.Token);
await syncTask;
}
else
{
const string message = "Peer sent an inconsistent batch of block headers";
_logger.Error(message);
throw new EthSynchronizationException(message);
}
}
if (_logger.IsDebugEnabled)
{
_logger.Debug($"Block {blocks[i].Number} suggested for processing");
}
}
else
{
if (_logger.IsWarnEnabled)
{
_logger.Warn($"Block {blocks[i].Number} skipped (validation failed)");
}
}
}
peerInfo.NumberReceived = blocks[blocks.Length - 1].Number;
bestNumber = BlockTree.BestSuggested.Number;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Stopping sync processes with Node: {peerInfo.Peer.NodeId}, wasCancelled: {wasCancelled}");
}
if (!wasCancelled)
{
peerInfo.IsSynced = true;
Synced?.Invoke(this, new SyncEventArgs(peerInfo.Peer));
}
}
public void AddNewBlock(Block block, Node nodeWhoSentTheBlock)
{
if (block.TotalDifficulty == null)
{
throw new InvalidOperationException("Cannot add a block with unknown total difficulty");
}
_pool.TryFind(nodeWhoSentTheBlock.Id, out PeerInfo peerInfo);
if (peerInfo == null)
{
string errorMessage = $"Received a new block from an unknown peer {nodeWhoSentTheBlock:c} {nodeWhoSentTheBlock.Id} {_pool.PeerCount}";
if (_logger.IsDebug)
{
_logger.Debug(errorMessage);
}
return;
}
if ((block.TotalDifficulty ?? 0) > peerInfo.TotalDifficulty)
{
if (_logger.IsTrace)
{
_logger.Trace($"ADD NEW BLOCK Updating header of {peerInfo} from {peerInfo.HeadNumber} {peerInfo.TotalDifficulty} to {block.Number} {block.TotalDifficulty}");
}
peerInfo.HeadNumber = block.Number;
peerInfo.HeadHash = block.Hash;
peerInfo.TotalDifficulty = block.TotalDifficulty ?? peerInfo.TotalDifficulty;
}
if ((block.TotalDifficulty ?? 0) < _blockTree.BestSuggestedHeader.TotalDifficulty)
{
return;
}
lock (_recentlySuggested)
{
if (_recentlySuggested.Get(block.Hash) != null)
{
return;
}
_recentlySuggested.Set(block.Hash, _dummyValue);
}
if (block.Number > _blockTree.BestKnownNumber + 8)
{
// ignore blocks when syncing in a simple non-locking way
_synchronizer.RequestSynchronization(SyncTriggerType.NewDistantBlock);
return;
}
if (_logger.IsTrace)
{
_logger.Trace($"Adding new block {block.ToString(Block.Format.Short)}) from {nodeWhoSentTheBlock:c}");
}
if (!_sealValidator.ValidateSeal(block.Header))
{
throw new EthSynchronizationException("Peer sent a block with an invalid seal");
}
if (block.Number <= _blockTree.BestKnownNumber + 1)
{
if (_logger.IsInfo)
{
string authorString = block.Author == null ? string.Empty : "sealed by " + (KnownAddresses.GoerliValidators.ContainsKey(block.Author) ? KnownAddresses.GoerliValidators[block.Author] : block.Author?.ToString());
if (_logger.IsInfo)
{
_logger.Info($"Discovered a new block {string.Empty.PadLeft(9 - block.Number.ToString().Length, ' ')}{block.ToString(Block.Format.HashNumberAndTx)} {authorString}, sent by {nodeWhoSentTheBlock:s}");
}
}
if (_logger.IsTrace)
{
_logger.Trace($"{block}");
}
if (_synchronizer.SyncMode == SyncMode.Full)
{
AddBlockResult result = _blockTree.SuggestBlock(block);
if (_logger.IsTrace)
{
_logger.Trace($"{block.Hash} ({block.Number}) adding result is {result}");
}
if (result == AddBlockResult.UnknownParent)
{
_synchronizer.RequestSynchronization(SyncTriggerType.Reorganization);
}
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Received a block {block.Hash} ({block.Number}) from {nodeWhoSentTheBlock} - need to resync");
}
_synchronizer.RequestSynchronization(SyncTriggerType.NewNearBlock);
}
}
public void AddNewBlock(Block block, NodeId receivedFrom)
{
// TODO: validation
_peers.TryGetValue(receivedFrom, out PeerInfo peerInfo);
if (peerInfo == null)
{
string errorMessage = $"Received a new block from an unknown peer {receivedFrom}";
_logger.Error(errorMessage);
return;
}
peerInfo.NumberAvailable = UInt256.Max(block.Number, peerInfo.NumberAvailable);
// peerInfo.Difficulty = UInt256.Max(block.Difficulty, peerInfo.Difficulty);
lock (_isSyncingLock)
{
if (_isSyncing)
{
if (_logger.IsTrace)
{
_logger.Trace($"Ignoring new block {block.Hash} while syncing");
}
return;
}
}
if (_logger.IsTrace)
{
_logger.Trace($"Adding new block {block.Hash} ({block.Number}) from {receivedFrom}");
}
if (block.Number <= _blockTree.BestSuggested.Number + 1)
{
if (_logger.IsTrace)
{
_logger.Trace($"Suggesting a block {block.Hash} ({block.Number}) from {receivedFrom} with {block.Transactions.Length} transactions");
}
if (_logger.IsTrace)
{
_logger.Trace($"{block}");
}
AddBlockResult result = _blockTree.SuggestBlock(block);
if (_logger.IsInfo)
{
_logger.Info($"{block.Hash} ({block.Number}) adding result is {result}");
}
if (result == AddBlockResult.UnknownParent)
{
/* here we want to cover scenario when our peer is reorganizing and sends us a head block
* from a new branch and we need to sync previous blocks as we do not know this block's parent */
RequestSync();
}
else
{
peerInfo.NumberReceived = block.Number;
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Received a block {block.Hash} ({block.Number}) from {receivedFrom} - need to resync");
}
RequestSync();
}
}
private async Task SynchronizeWithPeerAsync(PeerInfo peerInfo, CancellationToken peerSyncToken)
{
bool wasCanceled = false;
ISynchronizationPeer peer = peerInfo.Peer;
BigInteger bestNumber = _blockTree.BestSuggested.Number;
// UInt256 bestDifficulty = _blockTree.BestSuggested.Difficulty;
const int maxLookup = 64;
int ancestorLookupLevel = 0;
bool isCommonAncestorKnown = false;
while (peerInfo.NumberAvailable > bestNumber && peerInfo.NumberReceived <= peerInfo.NumberAvailable)
{
if (_logger.IsTrace)
{
_logger.Trace($"Continue syncing with {peerInfo} (our best {bestNumber})");
}
if (ancestorLookupLevel > maxLookup)
{
if (_logger.IsWarn)
{
_logger.Warn($"Could not find common ancestor with {peerInfo.Peer.NodeId}");
}
throw new EthSynchronizationException("Peer with inconsistent chain in sync");
}
if (peerSyncToken.IsCancellationRequested)
{
peerSyncToken.ThrowIfCancellationRequested();
}
if (!isCommonAncestorKnown)
{
// TODO: cases when many peers used for sync and one peer finished sync and then we need resync - we should start from common point and not NumberReceived that may be far in the past
_logger.Trace($"Finding common ancestor for {peerInfo.Peer.NodeId}");
isCommonAncestorKnown = true;
}
BigInteger blocksLeft = peerInfo.NumberAvailable - peerInfo.NumberReceived;
int blocksToRequest = (int)BigInteger.Min(blocksLeft + 1, BatchSize);
if (_logger.IsTrace)
{
_logger.Trace($"Sync request to peer with {peerInfo.NumberAvailable} blocks. Got {peerInfo.NumberReceived} and asking for {blocksToRequest} more.");
}
Task <BlockHeader[]> headersTask = peer.GetBlockHeaders(peerInfo.NumberReceived, blocksToRequest, 0, peerSyncToken);
_currentSyncTask = headersTask;
BlockHeader[] headers = await headersTask;
if (_currentSyncTask.IsCanceled)
{
wasCanceled = true;
break;
}
if (_currentSyncTask.IsFaulted)
{
if (_currentSyncTask.Exception.InnerExceptions.Any(x => x.InnerException is TimeoutException))
{
if (_logger.IsTrace)
{
_logger.Error("Failed to retrieve headers when synchronizing (Timeout)", _currentSyncTask.Exception);
}
}
else
{
if (_logger.IsError)
{
_logger.Error("Failed to retrieve headers when synchronizing", _currentSyncTask.Exception);
}
}
throw _currentSyncTask.Exception;
}
if (peerSyncToken.IsCancellationRequested)
{
peerSyncToken.ThrowIfCancellationRequested();
}
List <Keccak> hashes = new List <Keccak>();
Dictionary <Keccak, BlockHeader> headersByHash = new Dictionary <Keccak, BlockHeader>();
for (int i = 1; i < headers.Length; i++)
{
hashes.Add(headers[i].Hash);
headersByHash[headers[i].Hash] = headers[i];
}
Task <Block[]> bodiesTask = peer.GetBlocks(hashes.ToArray(), peerSyncToken);
_currentSyncTask = bodiesTask;
Block[] blocks = await bodiesTask;
if (_currentSyncTask.IsCanceled)
{
wasCanceled = true;
break;
}
if (_currentSyncTask.IsFaulted)
{
if (_currentSyncTask.Exception.InnerExceptions.Any(x => x.InnerException is TimeoutException))
{
if (_logger.IsTrace)
{
_logger.Error("Failed to retrieve bodies when synchronizing (Timeout)", _currentSyncTask.Exception);
}
}
else
{
if (_logger.IsError)
{
_logger.Error("Failed to retrieve bodies when synchronizing", _currentSyncTask.Exception);
}
}
throw _currentSyncTask.Exception;
}
for (int i = 0; i < blocks.Length; i++)
{
if (peerSyncToken.IsCancellationRequested)
{
peerSyncToken.ThrowIfCancellationRequested();
}
blocks[i].Header = headersByHash[hashes[i]];
}
if (blocks.Length > 0)
{
Block parent = _blockTree.FindParent(blocks[0]);
if (parent == null)
{
ancestorLookupLevel += BatchSize;
peerInfo.NumberReceived = peerInfo.NumberReceived >= BatchSize ? peerInfo.NumberReceived - BatchSize : 0;
continue;
}
}
// Parity 1.11 non canonical blocks when testing on 27/06
for (int i = 0; i < blocks.Length; i++)
{
if (i != 0 && blocks[i].ParentHash != blocks[i - 1].Hash)
{
throw new EthSynchronizationException("Peer sent an inconsistent block list");
}
}
for (int i = 0; i < blocks.Length; i++)
{
if (_logger.IsTrace)
{
_logger.Trace($"Received {blocks[i]} from {peer.NodeId}");
}
if (_blockValidator.ValidateSuggestedBlock(blocks[i]))
{
AddBlockResult addResult = _blockTree.SuggestBlock(blocks[i]);
if (addResult == AddBlockResult.UnknownParent)
{
if (_logger.IsTrace)
{
_logger.Trace($"Block {blocks[i].Number} ignored (unknown parent)");
}
if (i == 0)
{
const string message = "Peer sent orphaned blocks";
_logger.Error(message);
throw new EthSynchronizationException(message);
// if (_logger.IsTrace) _logger.Trace("Resyncing split");
// peerInfo.NumberReceived -= 1;
// var syncTask =
// Task.Run(() => SynchronizeWithPeerAsync(peerInfo, _peerSyncCancellationTokenSource.Token),
// _peerSyncCancellationTokenSource.Token);
// await syncTask;
}
else
{
const string message = "Peer sent an inconsistent batch of block headers";
_logger.Error(message);
throw new EthSynchronizationException(message);
}
}
if (_logger.IsTrace)
{
_logger.Trace($"Block {blocks[i].Number} suggested for processing");
}
}
else
{
if (_logger.IsWarn)
{
_logger.Warn($"Block {blocks[i].Number} skipped (validation failed)");
}
}
}
peerInfo.NumberReceived = blocks[blocks.Length - 1].Number;
bestNumber = _blockTree.BestSuggested.Number;
}
if (_logger.IsTrace)
{
_logger.Trace($"Stopping sync processes with Node: {peerInfo.Peer.NodeId}, wasCancelled: {wasCanceled}");
}
}
protected override AddBlockResult InsertToBlockTree(BlockHeader header)
{
if (_chainMerged)
{
if (_logger.IsTrace)
{
_logger.Trace(
"Chain already merged, skipping header insert");
}
return(AddBlockResult.AlreadyKnown);
}
if (_logger.IsTrace)
{
_logger.Trace(
$"Adding new header in beacon headers sync {header.ToString(BlockHeader.Format.FullHashAndNumber)}");
}
BlockTreeInsertHeaderOptions headerOptions = BlockTreeInsertHeaderOptions.BeaconHeaderInsert;
if (_nextHeaderDiff is null)
{
headerOptions |= BlockTreeInsertHeaderOptions.TotalDifficultyNotNeeded;
}
// Found existing block in the block tree
if (_blockTree.IsKnownBlock(header.Number, header.GetOrCalculateHash()))
{
_chainMerged = true;
if (_logger.IsTrace)
{
_logger.Trace(
$"Found header to join dangling beacon chain {header.ToString(BlockHeader.Format.FullHashAndNumber)}");
}
return(AddBlockResult.AlreadyKnown);
}
_invalidChainTracker.SetChildParent(header.Hash !, header.ParentHash !);
AddBlockResult insertOutcome = _blockTree.Insert(header, headerOptions);
if (insertOutcome == AddBlockResult.Added || insertOutcome == AddBlockResult.AlreadyKnown)
{
_nextHeaderHash = header.ParentHash !;
if (_expectedDifficultyOverride?.TryGetValue(header.Number, out ulong nextHeaderDiff) == true)
{
_nextHeaderDiff = nextHeaderDiff;
}
else
{
_nextHeaderDiff = header.TotalDifficulty != null && header.TotalDifficulty >= header.Difficulty
? header.TotalDifficulty - header.Difficulty
: null;
}
}
if (_logger.IsTrace)
{
_logger.Trace(
$"New header {header.ToString(BlockHeader.Format.FullHashAndNumber)} in beacon headers sync. InsertOutcome: {insertOutcome}");
}
return(insertOutcome);
}
