private void AddDependency(Keccak dependency, DependentItem dependentItem) { lock (_dependencies) { if (!_dependencies.ContainsKey(dependency)) { _dependencies[dependency] = new HashSet <DependentItem>(DependentItemComparer.Instance); } _dependencies[dependency].Add(dependentItem); _tempDb[dependentItem.SyncItem.Hash.Bytes] = dependentItem.Value; } }
private void HandleTrieNode(StateSyncItem currentStateSyncItem, byte[] currentResponseItem, ref int invalidNodes) { NodeDataType nodeDataType = currentStateSyncItem.NodeDataType; TrieNode trieNode = new TrieNode(NodeType.Unknown, currentResponseItem); trieNode.ResolveNode(null); switch (trieNode.NodeType) { case NodeType.Unknown: invalidNodes++; if (_logger.IsError) { _logger.Error($"Node {currentStateSyncItem.Hash} resolved to {nameof(NodeType.Unknown)}"); } break; case NodeType.Branch: DependentItem dependentBranch = new DependentItem(currentStateSyncItem, currentResponseItem, 0); // children may have the same hashes (e.g. a set of accounts with the same code at different addresses) HashSet <Keccak> alreadyProcessedChildHashes = new HashSet <Keccak>(); for (int childIndex = 15; childIndex >= 0; childIndex--) { Keccak childHash = trieNode.GetChildHash(childIndex); if (childHash != null && alreadyProcessedChildHashes.Contains(childHash)) { continue; } alreadyProcessedChildHashes.Add(childHash); if (childHash != null) { AddNodeResult addChildResult = AddNodeToPending(new StateSyncItem(childHash, nodeDataType, currentStateSyncItem.Level + 1, CalculateRightness(trieNode.NodeType, currentStateSyncItem, childIndex)) { BranchChildIndex = (short)childIndex, ParentBranchChildIndex = currentStateSyncItem.BranchChildIndex }, dependentBranch, "branch child"); if (addChildResult != AddNodeResult.AlreadySaved) { dependentBranch.Counter++; } else { _syncProgress.ReportSynced(currentStateSyncItem.Level + 1, currentStateSyncItem.BranchChildIndex, childIndex, currentStateSyncItem.NodeDataType, NodeProgressState.AlreadySaved); } } else { _syncProgress.ReportSynced(currentStateSyncItem.Level + 1, currentStateSyncItem.BranchChildIndex, childIndex, currentStateSyncItem.NodeDataType, NodeProgressState.Empty); } } if (dependentBranch.Counter == 0) { SaveNode(currentStateSyncItem, currentResponseItem); } break; case NodeType.Extension: Keccak next = trieNode[0].Keccak; if (next != null) { DependentItem dependentItem = new DependentItem(currentStateSyncItem, currentResponseItem, 1); AddNodeResult addResult = AddNodeToPending(new StateSyncItem(next, nodeDataType, currentStateSyncItem.Level + trieNode.Path.Length, CalculateRightness(trieNode.NodeType, currentStateSyncItem, 0)) { ParentBranchChildIndex = currentStateSyncItem.BranchChildIndex }, dependentItem, "extension child"); if (addResult == AddNodeResult.AlreadySaved) { SaveNode(currentStateSyncItem, currentResponseItem); } } else { /* this happens when we have a short RLP format of the node * that would not be stored as Keccak but full RLP*/ SaveNode(currentStateSyncItem, currentResponseItem); } break; case NodeType.Leaf: if (nodeDataType == NodeDataType.State) { _pendingItems.MaxStateLevel = 64; DependentItem dependentItem = new DependentItem(currentStateSyncItem, currentResponseItem, 0, true); (Keccak codeHash, Keccak storageRoot) = AccountDecoder.DecodeHashesOnly(new RlpStream(trieNode.Value)); if (codeHash != Keccak.OfAnEmptyString) { // prepare a branch without the code DB // this only protects against being same as storage root? if (codeHash == storageRoot) { lock (_codesSameAsNodes) { _codesSameAsNodes.Add(codeHash); } } else { AddNodeResult addCodeResult = AddNodeToPending(new StateSyncItem(codeHash, NodeDataType.Code, 0, currentStateSyncItem.Rightness), dependentItem, "code"); if (addCodeResult != AddNodeResult.AlreadySaved) { dependentItem.Counter++; } } } if (storageRoot != Keccak.EmptyTreeHash) { AddNodeResult addStorageNodeResult = AddNodeToPending(new StateSyncItem(storageRoot, NodeDataType.Storage, 0, currentStateSyncItem.Rightness), dependentItem, "storage"); if (addStorageNodeResult != AddNodeResult.AlreadySaved) { dependentItem.Counter++; } } if (dependentItem.Counter == 0) { Interlocked.Increment(ref _data.SavedAccounts); SaveNode(currentStateSyncItem, currentResponseItem); } } else { _pendingItems.MaxStorageLevel = 64; SaveNode(currentStateSyncItem, currentResponseItem); } break; default: if (_logger.IsError) { _logger.Error($"Unknown value {currentStateSyncItem.NodeDataType} of {nameof(NodeDataType)} at {currentStateSyncItem.Hash}"); } invalidNodes++; break; } }
private AddNodeResult AddNodeToPending(StateSyncItem syncItem, DependentItem dependentItem, string reason, bool missing = false) { if (!missing) { if (syncItem.Level <= 2) { _syncProgress.ReportSynced(syncItem.Level, syncItem.ParentBranchChildIndex, syncItem.BranchChildIndex, syncItem.NodeDataType, NodeProgressState.Requested); } if (_alreadySaved.Get(syncItem.Hash)) { Interlocked.Increment(ref _data.CheckWasCached); if (_logger.IsTrace) { _logger.Trace($"Node already in the DB - skipping {syncItem.Hash}"); } return(AddNodeResult.AlreadySaved); } object lockToTake = syncItem.NodeDataType == NodeDataType.Code ? _codeDbLock : _stateDbLock; lock (lockToTake) { IDb dbToCheck = syncItem.NodeDataType == NodeDataType.Code ? _codeDb : _stateDb; Interlocked.Increment(ref _data.DbChecks); bool keyExists = dbToCheck.KeyExists(syncItem.Hash); if (keyExists) { if (_logger.IsTrace) { _logger.Trace($"Node already in the DB - skipping {syncItem.Hash}"); } _alreadySaved.Set(syncItem.Hash); Interlocked.Increment(ref _data.StateWasThere); return(AddNodeResult.AlreadySaved); } Interlocked.Increment(ref _data.StateWasNotThere); } bool isAlreadyRequested; lock (_dependencies) { isAlreadyRequested = _dependencies.ContainsKey(syncItem.Hash); if (dependentItem != null) { if (_logger.IsTrace) { _logger.Trace($"Adding dependency {syncItem.Hash} -> {dependentItem.SyncItem.Hash}"); } AddDependency(syncItem.Hash, dependentItem); } } /* same items can have same hashes and we only need them once * there is an issue when we have an item, we add it to dependencies, then we request it and the request times out * and we never request it again because it was already on the dependencies list */ if (isAlreadyRequested) { Interlocked.Increment(ref _data.CheckWasInDependencies); if (_logger.IsTrace) { _logger.Trace($"Node already requested - skipping {syncItem.Hash}"); } return(AddNodeResult.AlreadyRequested); } } _pendingItems.PushToSelectedStream(syncItem, _syncProgress.LastProgress); if (_logger.IsTrace) { _logger.Trace($"Added a node {syncItem.Hash} - {reason}"); } return(AddNodeResult.Added); }