public bool refreshSignatures(Block b, bool forceRefresh = false, RemoteEndpoint endpoint = null) { if (!forceRefresh) { // we refuse to change sig numbers older than 4 blocks ulong sigLockHeight = getLastBlockNum() > 5 ? getLastBlockNum() - 3 : 1; if (b.blockNum <= sigLockHeight) { return(false); } } Block updatestorage_block = null; int beforeSigs = 0; int afterSigs = 0; lock (blocks) { int idx = blocks.FindIndex(x => x.blockNum == b.blockNum && x.blockChecksum.SequenceEqual(b.blockChecksum)); if (idx > 0) { if (blocks[idx].compacted) { Logging.error("Trying to refresh signatures on compacted block {0}", blocks[idx].blockNum); return(false); } byte[] beforeSigsChecksum = blocks[idx].calculateSignatureChecksum(); beforeSigs = blocks[idx].getFrozenSignatureCount(); var added_sigs = blocks[idx].addSignaturesFrom(b); if (added_sigs != null && Node.isMasterNode()) { foreach (var sig in added_sigs) { Node.inventoryCache.setProcessedFlag(InventoryItemTypes.blockSignature, InventoryItemSignature.getHash(sig[1], b.blockChecksum), true); SignatureProtocolMessages.broadcastBlockSignature(sig[0], sig[1], b.blockNum, b.blockChecksum, endpoint, null); } } if (forceRefresh) { if (!b.verifyBlockProposer()) { Logging.error("Error verifying block proposer while force refreshing signatures on block {0} ({1})", b.blockNum, Crypto.hashToString(b.blockChecksum)); return(false); } blocks[idx].setFrozenSignatures(b.signatures); afterSigs = b.signatures.Count; } else { afterSigs = blocks[idx].signatures.Count; } byte[] afterSigsChecksum = blocks[idx].calculateSignatureChecksum(); if (!beforeSigsChecksum.SequenceEqual(afterSigsChecksum)) { updatestorage_block = blocks[idx]; } } } // Check if the block needs to be refreshed if (updatestorage_block != null) { updateBlock(updatestorage_block); Logging.info("Refreshed block #{0}: Updated signatures {1} -> {2}", b.blockNum, beforeSigs, afterSigs); return(true); } return(false); }
static void handleInventory2(byte[] data, RemoteEndpoint endpoint) { using (MemoryStream m = new MemoryStream(data)) { using (BinaryReader reader = new BinaryReader(m)) { ulong item_count = reader.ReadIxiVarUInt(); if (item_count > (ulong)CoreConfig.maxInventoryItems) { Logging.warn("Received {0} inventory items, max items is {1}", item_count, CoreConfig.maxInventoryItems); item_count = (ulong)CoreConfig.maxInventoryItems; } ulong last_block_height = IxianHandler.getLastBlockHeight(); Dictionary <ulong, List <InventoryItemSignature> > sig_lists = new Dictionary <ulong, List <InventoryItemSignature> >(); List <InventoryItemKeepAlive> ka_list = new List <InventoryItemKeepAlive>(); List <byte[]> tx_list = new List <byte[]>(); bool request_next_block = false; for (ulong i = 0; i < item_count; i++) { ulong len = reader.ReadIxiVarUInt(); byte[] item_bytes = reader.ReadBytes((int)len); InventoryItem item = InventoryCache.decodeInventoryItem(item_bytes); if (item.type == InventoryItemTypes.transaction) { PendingTransactions.increaseReceivedCount(item.hash, endpoint.presence.wallet); } PendingInventoryItem pii = Node.inventoryCache.add(item, endpoint); if (!pii.processed && pii.lastRequested == 0) { // first time we're seeing this inventory item switch (item.type) { case InventoryItemTypes.keepAlive: ka_list.Add((InventoryItemKeepAlive)item); pii.lastRequested = Clock.getTimestamp(); break; case InventoryItemTypes.transaction: tx_list.Add(item.hash); pii.lastRequested = Clock.getTimestamp(); break; case InventoryItemTypes.blockSignature: var iis = (InventoryItemSignature)item; if (iis.blockNum < last_block_height - 5 && iis.blockNum > last_block_height + 6) { continue; } if (!sig_lists.ContainsKey(iis.blockNum)) { sig_lists.Add(iis.blockNum, new List <InventoryItemSignature>()); } sig_lists[iis.blockNum].Add(iis); pii.lastRequested = Clock.getTimestamp(); break; case InventoryItemTypes.block: var iib = ((InventoryItemBlock)item); if (iib.blockNum <= last_block_height) { Node.inventoryCache.processInventoryItem(pii); } else { pii.lastRequested = Clock.getTimestamp(); request_next_block = true; if (iib.blockNum > endpoint.blockHeight) { endpoint.blockHeight = iib.blockNum; } if (iib.blockNum > Node.blockProcessor.highestNetworkBlockNum) { Node.blockProcessor.highestNetworkBlockNum = iib.blockNum; } } break; default: Node.inventoryCache.processInventoryItem(pii); break; } } } PresenceProtocolMessages.broadcastGetKeepAlives(ka_list, endpoint); if (Node.blockSync.synchronizing) { return; } TransactionProtocolMessages.broadcastGetTransactions(tx_list, 0, endpoint); if (request_next_block) { byte include_tx = 2; if (Node.isMasterNode()) { include_tx = 0; } BlockProtocolMessages.broadcastGetBlock(last_block_height + 1, null, endpoint, include_tx, true); } foreach (var sig_list in sig_lists) { SignatureProtocolMessages.broadcastGetSignatures(sig_list.Key, sig_list.Value, endpoint); } } } }
private void rollForward() { bool sleep = false; ulong lowestBlockNum = getLowestBlockNum(); ulong syncToBlock = syncTargetBlockNum; if (Node.blockChain.Count > 5) { lock (pendingBlocks) { pendingBlocks.RemoveAll(x => x.blockNum < Node.blockChain.getLastBlockNum() - 5); } } lock (pendingBlocks) { // Loop until we have no more pending blocks do { ulong next_to_apply = lowestBlockNum; if (Node.blockChain.Count > 0) { next_to_apply = Node.blockChain.getLastBlockNum() + 1; } if (next_to_apply > syncToBlock) { // we have everything, clear pending blocks and break pendingBlocks.Clear(); lock (requestedBlockTimes) { requestedBlockTimes.Clear(); } break; } Block b = pendingBlocks.Find(x => x.blockNum == next_to_apply); if (b == null) { lock (requestedBlockTimes) { if (requestBlockAgain(next_to_apply)) { // the node isn't connected yet, wait a while sleep = true; } } break; } b = new Block(b); if (Node.blockChain.Count == 0 && b.blockNum > 1) { Block tmp_b = Node.blockChain.getBlock(b.blockNum - 1, true, true); if (tmp_b != null) { Node.blockChain.setLastBlockVersion(tmp_b.version); } else { Node.blockChain.setLastBlockVersion(b.version); } } if (b.version > Block.maxVersion) { Logging.error("Received block {0} with a version higher than this node can handle, discarding the block.", b.blockNum); pendingBlocks.RemoveAll(x => x.blockNum == b.blockNum); Node.blockProcessor.networkUpgraded = true; sleep = true; break; } else { Node.blockProcessor.networkUpgraded = false; } if (next_to_apply > 5) { ulong targetBlock = next_to_apply - 5; Block tb = pendingBlocks.Find(x => x.blockNum == targetBlock); if (tb != null) { Block local_block = Node.blockChain.getBlock(tb.blockNum); if (local_block != null && tb.blockChecksum.SequenceEqual(local_block.blockChecksum) && Node.blockProcessor.verifyBlockBasic(tb) == BlockVerifyStatus.Valid) { if (Node.blockProcessor.verifyBlockSignatures(tb, null)) { Node.blockChain.refreshSignatures(tb, true); } else { Logging.warn("Target block " + tb.blockNum + " does not have the required consensus."); } } pendingBlocks.RemoveAll(x => x.blockNum == tb.blockNum); } } try { Logging.info("Sync: Applying block #{0}/{1}.", b.blockNum, syncToBlock); bool ignoreWalletState = true; if (b.blockNum > wsSyncConfirmedBlockNum || Config.fullStorageDataVerification) { ignoreWalletState = false; } b.powField = null; // wallet state is correct as of wsConfirmedBlockNumber, so before that we call // verify with a parameter to ignore WS tests, but do all the others BlockVerifyStatus b_status = BlockVerifyStatus.Valid; if (b.fromLocalStorage) { // TODO TODO improve this section with NodeStorage.getTransactionsInBlock once rocksdb switch happens bool missing = false; foreach (byte[] txid in b.transactions) { if (!running) { break; } Transaction t = TransactionPool.getUnappliedTransaction(txid); if (t == null) { t = Node.storage.getTransaction(txid, b.blockNum); if (t != null) { t.applied = 0; TransactionPool.addTransaction(t, true, null, Config.fullStorageDataVerification); } else { CoreProtocolMessage.broadcastGetTransaction(txid, b.blockNum); missing = true; } } } if (missing) { Logging.info("Requesting missing transactions for block {0}", b.blockNum); Thread.Sleep(100); break; } } if (b.blockNum > wsSyncConfirmedBlockNum || b.fromLocalStorage == false || Config.fullStorageDataVerification) { b_status = Node.blockProcessor.verifyBlock(b, ignoreWalletState); } else { b_status = Node.blockProcessor.verifyBlockBasic(b, false); } if (b_status == BlockVerifyStatus.Indeterminate) { if (lastProcessedBlockNum < b.blockNum) { lastProcessedBlockNum = b.blockNum; lastProcessedBlockTime = Clock.getTimestamp(); } if (Clock.getTimestamp() - lastProcessedBlockTime > ConsensusConfig.blockGenerationInterval * 2) { Logging.info("Sync: Discarding indeterminate block #{0}, due to timeout...", b.blockNum); Node.blockProcessor.blacklistBlock(b); pendingBlocks.RemoveAll(x => x.blockNum == b.blockNum); requestBlockAgain(b.blockNum); } else { Logging.info("Sync: Waiting for missing transactions from block #{0}...", b.blockNum); } Thread.Sleep(100); return; } if (b_status != BlockVerifyStatus.Valid) { Logging.warn("Block #{0} {1} is invalid. Discarding and requesting a new one.", b.blockNum, Crypto.hashToString(b.blockChecksum)); Node.blockProcessor.blacklistBlock(b); pendingBlocks.RemoveAll(x => x.blockNum == b.blockNum); requestBlockAgain(b.blockNum); if (b_status == BlockVerifyStatus.PotentiallyForkedBlock && b.blockNum + 7 > lastBlockToReadFromStorage) { Node.blockProcessor.handleForkedFlag(); } return; } if (!b.fromLocalStorage && !Node.blockProcessor.verifyBlockSignatures(b, null) && Node.blockChain.Count > 16) { Logging.warn("Block #{0} {1} doesn't have the required consensus. Discarding and requesting a new one.", b.blockNum, Crypto.hashToString(b.blockChecksum)); pendingBlocks.RemoveAll(x => x.blockNum == b.blockNum); requestBlockAgain(b.blockNum); return; } bool sigFreezeCheck = Node.blockProcessor.verifySignatureFreezeChecksum(b, null); // Apply transactions when rolling forward from a recover file without a synced WS if (b.blockNum > wsSyncConfirmedBlockNum) { if (Node.blockChain.Count <= 5 || sigFreezeCheck) { Node.walletState.beginTransaction(b.blockNum, false); bool applied = false; try { applied = Node.blockProcessor.applyAcceptedBlock(b); if (applied) { Node.walletState.commitTransaction(b.blockNum); } }catch (Exception e) { Logging.error("Error occured during block sync, while applying/commiting transactions: " + e); } if (!applied) { Logging.warn("Error applying Block #{0} {1}. Reverting, discarding and requesting a new one.", b.blockNum, Crypto.hashToString(b.blockChecksum)); Node.walletState.revertTransaction(b.blockNum); Node.blockChain.revertBlockTransactions(b); Node.blockProcessor.blacklistBlock(b); pendingBlocks.RemoveAll(x => x.blockNum == b.blockNum); requestBlockAgain(b.blockNum); return; } else { if (b.version >= BlockVer.v5 && b.lastSuperBlockChecksum == null) { // skip WS checksum check } else { if (b.lastSuperBlockChecksum != null) { byte[] wsChecksum = Node.walletState.calculateWalletStateChecksum(); if (wsChecksum == null || !wsChecksum.SequenceEqual(b.walletStateChecksum)) { Logging.error("After applying block #{0}, walletStateChecksum is incorrect!. Block's WS: {1}, actual WS: {2}", b.blockNum, Crypto.hashToString(b.walletStateChecksum), Crypto.hashToString(wsChecksum)); Node.walletState.revertTransaction(b.blockNum); Node.blockChain.revertBlockTransactions(b); Node.blockProcessor.blacklistBlock(b); pendingBlocks.RemoveAll(x => x.blockNum == b.blockNum); requestBlockAgain(b.blockNum); return; } } } if (b.blockNum % Config.saveWalletStateEveryBlock == 0) { DLT.Meta.WalletStateStorage.saveWalletState(b.blockNum); } } } } else { if (syncToBlock == b.blockNum) { if (b.version >= BlockVer.v5 && b.lastSuperBlockChecksum == null) { // skip WS checksum check } else { byte[] wsChecksum = Node.walletState.calculateWalletStateChecksum(); if (wsChecksum == null || !wsChecksum.SequenceEqual(b.walletStateChecksum)) { Logging.warn("Block #{0} is last and has an invalid WSChecksum. Discarding and requesting a new one.", b.blockNum); Node.blockProcessor.blacklistBlock(b); pendingBlocks.RemoveAll(x => x.blockNum == b.blockNum); requestBlockAgain(b.blockNum); return; } } } } if (Node.blockChain.Count <= 5 || sigFreezeCheck) { //Logging.info(String.Format("Appending block #{0} to blockChain.", b.blockNum)); if (b.blockNum <= wsSyncConfirmedBlockNum) { if (!TransactionPool.setAppliedFlagToTransactionsFromBlock(b)) { Node.blockChain.revertBlockTransactions(b); Node.blockProcessor.blacklistBlock(b); pendingBlocks.RemoveAll(x => x.blockNum == b.blockNum); requestBlockAgain(b.blockNum); return; } } if (b.blockNum > 12 && b.blockNum + 5 >= IxianHandler.getHighestKnownNetworkBlockHeight()) { if (Node.isMasterNode() && b.blockNum > 7) { byte[][] signature_data = b.applySignature(); // applySignature() will return signature_data, if signature was applied and null, if signature was already present from before if (signature_data != null) { Node.inventoryCache.setProcessedFlag(InventoryItemTypes.blockSignature, InventoryItemSignature.getHash(signature_data[1], b.blockChecksum), true); // ProtocolMessage.broadcastNewBlock(localNewBlock); SignatureProtocolMessages.broadcastBlockSignature(signature_data[0], signature_data[1], b.blockNum, b.blockChecksum, null, null); } } } Node.blockChain.appendBlock(b, !b.fromLocalStorage); } else if (Node.blockChain.Count > 5 && !sigFreezeCheck) { if (CoreConfig.preventNetworkOperations || Config.recoverFromFile) { var last_block = lastBlocks.Find(x => x.blockNum == b.blockNum - 5); if (last_block != null) { pendingBlocks.Add(last_block); } return; } // invalid sigfreeze, waiting for the correct block Logging.warn("Block #{0} {1} doesn't have the correct sigfreezed block. Discarding and requesting a new one.", b.blockNum, Crypto.hashToString(b.blockChecksum)); pendingBlocks.RemoveAll(x => x.blockNum == b.blockNum); requestBlockAgain(b.blockNum); return; } } catch (Exception e) { Logging.error("Exception occured while syncing block #{0}: {1}", b.blockNum, e); } if (Config.enableChainReorgTest) { if (!Node.blockProcessor.chainReorgTest(b.blockNum)) { pendingBlocks.RemoveAll(x => x.blockNum + 6 < b.blockNum); } } else { pendingBlocks.RemoveAll(x => x.blockNum == b.blockNum); } Node.blockProcessor.cleanupBlockBlacklist(); } while (pendingBlocks.Count > 0 && running); } if (!sleep && Node.blockChain.getLastBlockNum() >= syncToBlock) { if (verifyLastBlock()) { sleep = false; } else { sleep = true; } } if (sleep) { Thread.Sleep(500); } }
// Unified protocol message parsing public static void parseProtocolMessage(ProtocolMessageCode code, byte[] data, RemoteEndpoint endpoint) { if (endpoint == null) { Logging.error("Endpoint was null. parseProtocolMessage"); return; } try { switch (code) { case ProtocolMessageCode.hello: handleHello(data, endpoint); break; case ProtocolMessageCode.helloData: handleHelloData(data, endpoint); break; case ProtocolMessageCode.getBlock: BlockProtocolMessages.handleGetBlock(data, endpoint); break; case ProtocolMessageCode.getBalance: WalletStateProtocolMessages.handleGetBalance(data, endpoint); break; case ProtocolMessageCode.getTransaction: TransactionProtocolMessages.handleGetTransaction(data, endpoint); break; case ProtocolMessageCode.getTransaction2: TransactionProtocolMessages.handleGetTransaction2(data, endpoint); break; case ProtocolMessageCode.getTransaction3: TransactionProtocolMessages.handleGetTransaction3(data, endpoint); break; case ProtocolMessageCode.newTransaction: case ProtocolMessageCode.transactionData: TransactionProtocolMessages.handleTransactionData(data, endpoint); break; case ProtocolMessageCode.bye: CoreProtocolMessage.processBye(data, endpoint); break; case ProtocolMessageCode.newBlock: case ProtocolMessageCode.blockData: BlockProtocolMessages.handleBlockData(data, endpoint); break; case ProtocolMessageCode.syncWalletState: WalletStateProtocolMessages.handleSyncWalletState(data, endpoint); break; case ProtocolMessageCode.walletState: WalletStateProtocolMessages.handleWalletState(data, endpoint); break; case ProtocolMessageCode.getWalletStateChunk: WalletStateProtocolMessages.handleGetWalletStateChunk(data, endpoint); break; case ProtocolMessageCode.walletStateChunk: WalletStateProtocolMessages.handleWalletStateChunk(data, endpoint); break; case ProtocolMessageCode.updatePresence: PresenceProtocolMessages.handleUpdatePresence(data, endpoint); break; case ProtocolMessageCode.keepAlivePresence: PresenceProtocolMessages.handleKeepAlivePresence(data, endpoint); break; case ProtocolMessageCode.getPresence: PresenceProtocolMessages.handleGetPresence(data, endpoint); break; case ProtocolMessageCode.getPresence2: PresenceProtocolMessages.handleGetPresence2(data, endpoint); break; case ProtocolMessageCode.getKeepAlives: PresenceProtocolMessages.handleGetKeepAlives(data, endpoint); break; case ProtocolMessageCode.keepAlivesChunk: PresenceProtocolMessages.handleKeepAlivesChunk(data, endpoint); break; // return 10 random presences of the selected type case ProtocolMessageCode.getRandomPresences: PresenceProtocolMessages.handleGetRandomPresences(data, endpoint); break; case ProtocolMessageCode.getUnappliedTransactions: TransactionProtocolMessages.handleGetUnappliedTransactions(data, endpoint); break; case ProtocolMessageCode.blockTransactionsChunk: BlockProtocolMessages.handleBlockTransactionsChunk(data, endpoint); break; case ProtocolMessageCode.attachEvent: NetworkEvents.handleAttachEventMessage(data, endpoint); break; case ProtocolMessageCode.detachEvent: NetworkEvents.handleDetachEventMessage(data, endpoint); break; case ProtocolMessageCode.blockSignature: SignatureProtocolMessages.handleBlockSignature(data, endpoint); break; case ProtocolMessageCode.getBlockSignatures: { using (MemoryStream m = new MemoryStream(data)) { using (BinaryReader reader = new BinaryReader(m)) { ulong block_num = reader.ReadUInt64(); int checksum_len = reader.ReadInt32(); byte[] checksum = reader.ReadBytes(checksum_len); SignatureProtocolMessages.handleGetBlockSignatures(block_num, checksum, endpoint); } } } break; case ProtocolMessageCode.blockSignatures: SignatureProtocolMessages.handleSigfreezedBlockSignatures(data, endpoint); break; case ProtocolMessageCode.getNextSuperBlock: BlockProtocolMessages.handleGetNextSuperBlock(data, endpoint); break; case ProtocolMessageCode.getBlockHeaders: BlockProtocolMessages.handleGetBlockHeaders(data, endpoint); break; case ProtocolMessageCode.getPIT: BlockProtocolMessages.handleGetPIT(data, endpoint); break; case ProtocolMessageCode.inventory: handleInventory(data, endpoint); break; case ProtocolMessageCode.inventory2: handleInventory2(data, endpoint); break; case ProtocolMessageCode.getSignatures: SignatureProtocolMessages.handleGetSignatures(data, endpoint); break; case ProtocolMessageCode.signaturesChunk: SignatureProtocolMessages.handleSignaturesChunk(data, endpoint); break; case ProtocolMessageCode.getTransactions: TransactionProtocolMessages.handleGetTransactions(data, endpoint); break; case ProtocolMessageCode.getTransactions2: TransactionProtocolMessages.handleGetTransactions2(data, endpoint); break; case ProtocolMessageCode.transactionsChunk: TransactionProtocolMessages.handleTransactionsChunk(data, endpoint); break; case ProtocolMessageCode.transactionsChunk2: TransactionProtocolMessages.handleTransactionsChunk2(data, endpoint); break; case ProtocolMessageCode.getBlockHeaders2: BlockProtocolMessages.handleGetBlockHeaders2(data, endpoint); break; case ProtocolMessageCode.getPIT2: BlockProtocolMessages.handleGetPIT2(data, endpoint); break; case ProtocolMessageCode.getBlock2: BlockProtocolMessages.handleGetBlock2(data, endpoint); break; case ProtocolMessageCode.getBlock3: BlockProtocolMessages.handleGetBlock3(data, endpoint); break; case ProtocolMessageCode.getBalance2: WalletStateProtocolMessages.handleGetBalance2(data, endpoint); break; case ProtocolMessageCode.getBlockSignatures2: { using (MemoryStream m = new MemoryStream(data)) { using (BinaryReader reader = new BinaryReader(m)) { ulong block_num = reader.ReadIxiVarUInt(); int checksum_len = (int)reader.ReadIxiVarUInt(); byte[] checksum = reader.ReadBytes(checksum_len); SignatureProtocolMessages.handleGetBlockSignatures2(block_num, checksum, endpoint); } } } break; case ProtocolMessageCode.blockSignature2: SignatureProtocolMessages.handleBlockSignature2(data, endpoint); break; default: break; } } catch (Exception e) { Logging.error("Error parsing network message. Details: {0}", e.ToString()); } }