// Methods for how to add transactions to a block. // Add transactions based on feerate including unconfirmed ancestors // Increments nPackagesSelected / nDescendantsUpdated with corresponding // statistics from the package selection (for logging statistics). // This transaction selection algorithm orders the mempool based // on feerate of a transaction including all unconfirmed ancestors. // Since we don't remove transactions from the mempool as we select them // for block inclusion, we need an alternate method of updating the feerate // of a transaction with its not-yet-selected ancestors as we go. // This is accomplished by walking the in-mempool descendants of selected // transactions and storing a temporary modified state in mapModifiedTxs. // Each time through the loop, we compare the best transaction in // mapModifiedTxs with the next transaction in the mempool to decide what // transaction package to work on next. protected virtual void AddTransactions(int nPackagesSelected, int nDescendantsUpdated) { // mapModifiedTx will store sorted packages after they are modified // because some of their txs are already in the block var mapModifiedTx = new Dictionary <uint256, TxMemPoolModifiedEntry>(); //var mapModifiedTxRes = this.mempoolScheduler.ReadAsync(() => mempool.MapTx.Values).GetAwaiter().GetResult(); // mapModifiedTxRes.Select(s => new TxMemPoolModifiedEntry(s)).OrderBy(o => o, new CompareModifiedEntry()); // Keep track of entries that failed inclusion, to avoid duplicate work TxMempool.SetEntries failedTx = new TxMempool.SetEntries(); // Start by adding all descendants of previously added txs to mapModifiedTx // and modifying them for their already included ancestors UpdatePackagesForAdded(inBlock, mapModifiedTx); var ancestorScoreList = this.mempoolScheduler.ReadAsync(() => mempool.MapTx.AncestorScore).GetAwaiter().GetResult().ToList(); TxMempoolEntry iter; // Limit the number of attempts to add transactions to the block when it is // close to full; this is just a simple heuristic to finish quickly if the // mempool has a lot of entries. int MAX_CONSECUTIVE_FAILURES = 1000; int nConsecutiveFailed = 0; while (ancestorScoreList.Any() || mapModifiedTx.Any()) { var mi = ancestorScoreList.FirstOrDefault(); if (mi != null) { // Skip entries in mapTx that are already in a block or are present // in mapModifiedTx (which implies that the mapTx ancestor state is // stale due to ancestor inclusion in the block) // Also skip transactions that we've already failed to add. This can happen if // we consider a transaction in mapModifiedTx and it fails: we can then // potentially consider it again while walking mapTx. It's currently // guaranteed to fail again, but as a belt-and-suspenders check we put it in // failedTx and avoid re-evaluation, since the re-evaluation would be using // cached size/sigops/fee values that are not actually correct. // First try to find a new transaction in mapTx to evaluate. if (mapModifiedTx.ContainsKey(mi.TransactionHash) || inBlock.Contains(mi) || failedTx.Contains(mi)) { ancestorScoreList.Remove(mi); continue; } } // Now that mi is not stale, determine which transaction to evaluate: // the next entry from mapTx, or the best from mapModifiedTx? bool fUsingModified = false; TxMemPoolModifiedEntry modit; var compare = new CompareModifiedEntry(); if (mi == null) { modit = mapModifiedTx.Values.OrderByDescending(o => o, compare).First(); iter = modit.iter; fUsingModified = true; } else { // Try to compare the mapTx entry to the mapModifiedTx entry iter = mi; modit = mapModifiedTx.Values.OrderByDescending(o => o, compare).FirstOrDefault(); if (modit != null && compare.Compare(modit, new TxMemPoolModifiedEntry(iter)) > 0) { // The best entry in mapModifiedTx has higher score // than the one from mapTx. // Switch which transaction (package) to consider iter = modit.iter; fUsingModified = true; } else { // Either no entry in mapModifiedTx, or it's worse than mapTx. // Increment mi for the next loop iteration. ancestorScoreList.Remove(iter); } } // We skip mapTx entries that are inBlock, and mapModifiedTx shouldn't // contain anything that is inBlock. Guard.Assert(!inBlock.Contains(iter)); var packageSize = iter.SizeWithAncestors; var packageFees = iter.ModFeesWithAncestors; var packageSigOpsCost = iter.SizeWithAncestors; if (fUsingModified) { packageSize = modit.SizeWithAncestors; packageFees = modit.ModFeesWithAncestors; packageSigOpsCost = modit.SigOpCostWithAncestors; } if (packageFees < blockMinFeeRate.GetFee((int)packageSize)) { // Everything else we might consider has a lower fee rate return; } if (!TestPackage(packageSize, packageSigOpsCost)) { if (fUsingModified) { // Since we always look at the best entry in mapModifiedTx, // we must erase failed entries so that we can consider the // next best entry on the next loop iteration mapModifiedTx.Remove(modit.iter.TransactionHash); failedTx.Add(iter); } ++nConsecutiveFailed; if (nConsecutiveFailed > MAX_CONSECUTIVE_FAILURES && blockWeight > blockMaxWeight - 4000) { // Give up if we're close to full and haven't succeeded in a while break; } continue; } TxMempool.SetEntries ancestors = new TxMempool.SetEntries(); long nNoLimit = long.MaxValue; string dummy; mempool.CalculateMemPoolAncestors(iter, ancestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, out dummy, false); OnlyUnconfirmed(ancestors); ancestors.Add(iter); // Test if all tx's are Final if (!TestPackageTransactions(ancestors)) { if (fUsingModified) { mapModifiedTx.Remove(modit.iter.TransactionHash); failedTx.Add(iter); } continue; } // This transaction will make it in; reset the failed counter. nConsecutiveFailed = 0; // Package can be added. Sort the entries in a valid order. // Sort package by ancestor count // If a transaction A depends on transaction B, then A's ancestor count // must be greater than B's. So this is sufficient to validly order the // transactions for block inclusion. var sortedEntries = ancestors.ToList().OrderBy(o => o, new CompareTxIterByAncestorCount()).ToList(); foreach (var sortedEntry in sortedEntries) { AddToBlock(sortedEntry); // Erase from the modified set, if present mapModifiedTx.Remove(sortedEntry.TransactionHash); } ++nPackagesSelected; // Update transactions that depend on each of these nDescendantsUpdated += UpdatePackagesForAdded(ancestors, mapModifiedTx); } }
public void MempoolIndexingTest() { NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet); var pool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings); var entry = new TestMemPoolEntryHelper(); /* 3rd highest fee */ var tx1 = new Transaction(); tx1.AddOutput(new TxOut(new Money(10 * Money.COIN), new Script(OpcodeType.OP_11, OpcodeType.OP_EQUAL))); pool.AddUnchecked(tx1.GetHash(), entry.Fee(new Money(10000L)).Priority(10.0).FromTx(tx1)); /* highest fee */ var tx2 = new Transaction(); tx2.AddOutput(new TxOut(new Money(2 * Money.COIN), new Script(OpcodeType.OP_11, OpcodeType.OP_EQUAL))); pool.AddUnchecked(tx2.GetHash(), entry.Fee(new Money(20000L)).Priority(9.0).FromTx(tx2)); /* lowest fee */ var tx3 = new Transaction(); tx3.AddOutput(new TxOut(new Money(5 * Money.COIN), new Script(OpcodeType.OP_11, OpcodeType.OP_EQUAL))); pool.AddUnchecked(tx3.GetHash(), entry.Fee(new Money(0L)).Priority(100.0).FromTx(tx3)); /* 2nd highest fee */ var tx4 = new Transaction(); tx4.AddOutput(new TxOut(new Money(6 * Money.COIN), new Script(OpcodeType.OP_11, OpcodeType.OP_EQUAL))); pool.AddUnchecked(tx4.GetHash(), entry.Fee(new Money(15000L)).Priority(1.0).FromTx(tx4)); /* equal fee rate to tx1, but newer */ var tx5 = new Transaction(); tx5.AddOutput(new TxOut(new Money(11 * Money.COIN), new Script(OpcodeType.OP_11, OpcodeType.OP_EQUAL))); pool.AddUnchecked(tx5.GetHash(), entry.Fee(new Money(10000L)).Priority(10.0).Time(1).FromTx(tx5)); // assert size Assert.Equal(5, pool.Size); var sortedOrder = new List <string>(5); sortedOrder.Insert(0, tx3.GetHash().ToString()); // 0 sortedOrder.Insert(1, tx5.GetHash().ToString()); // 10000 sortedOrder.Insert(2, tx1.GetHash().ToString()); // 10000 sortedOrder.Insert(3, tx4.GetHash().ToString()); // 15000 sortedOrder.Insert(4, tx2.GetHash().ToString()); // 20000 this.CheckSort(pool, pool.MapTx.DescendantScore.ToList(), sortedOrder); /* low fee but with high fee child */ /* tx6 -> tx7 -> tx8, tx9 -> tx10 */ var tx6 = new Transaction(); tx6.AddOutput(new TxOut(new Money(20 * Money.COIN), new Script(OpcodeType.OP_11, OpcodeType.OP_EQUAL))); pool.AddUnchecked(tx6.GetHash(), entry.Fee(new Money(0L)).FromTx(tx6)); // assert size Assert.Equal(6, pool.Size); // Check that at this point, tx6 is sorted low sortedOrder.Insert(0, tx6.GetHash().ToString()); this.CheckSort(pool, pool.MapTx.DescendantScore.ToList(), sortedOrder); var setAncestors = new TxMempool.SetEntries(); setAncestors.Add(pool.MapTx.TryGet(tx6.GetHash())); var tx7 = new Transaction(); tx7.AddInput(new TxIn(new OutPoint(tx6.GetHash(), 0), new Script(OpcodeType.OP_11))); tx7.AddOutput(new TxOut(new Money(10 * Money.COIN), new Script(OpcodeType.OP_11, OpcodeType.OP_EQUAL))); tx7.AddOutput(new TxOut(new Money(1 * Money.COIN), new Script(OpcodeType.OP_11, OpcodeType.OP_EQUAL))); var setAncestorsCalculated = new TxMempool.SetEntries(); string dummy; Assert.True(pool.CalculateMemPoolAncestors(entry.Fee(2000000L).FromTx(tx7), setAncestorsCalculated, 100, 1000000, 1000, 1000000, out dummy)); Assert.True(setAncestorsCalculated.Equals(setAncestors)); pool.AddUnchecked(tx7.GetHash(), entry.FromTx(tx7), setAncestors); Assert.Equal(7, pool.Size); // Now tx6 should be sorted higher (high fee child): tx7, tx6, tx2, ... sortedOrder.RemoveAt(0); sortedOrder.Add(tx6.GetHash().ToString()); sortedOrder.Add(tx7.GetHash().ToString()); this.CheckSort(pool, pool.MapTx.DescendantScore.ToList(), sortedOrder); /* low fee child of tx7 */ var tx8 = new Transaction(); tx8.AddInput(new TxIn(new OutPoint(tx7.GetHash(), 0), new Script(OpcodeType.OP_11))); tx8.AddOutput(new TxOut(new Money(10 * Money.COIN), new Script(OpcodeType.OP_11, OpcodeType.OP_EQUAL))); setAncestors.Add(pool.MapTx.TryGet(tx7.GetHash())); pool.AddUnchecked(tx8.GetHash(), entry.Fee(0L).Time(2).FromTx(tx8), setAncestors); // Now tx8 should be sorted low, but tx6/tx both high sortedOrder.Insert(0, tx8.GetHash().ToString()); this.CheckSort(pool, pool.MapTx.DescendantScore.ToList(), sortedOrder); /* low fee child of tx7 */ var tx9 = new Transaction(); tx9.AddInput(new TxIn(new OutPoint(tx7.GetHash(), 1), new Script(OpcodeType.OP_11))); tx9.AddOutput(new TxOut(new Money(1 * Money.COIN), new Script(OpcodeType.OP_11, OpcodeType.OP_EQUAL))); pool.AddUnchecked(tx9.GetHash(), entry.Fee(0L).Time(3).FromTx(tx9), setAncestors); // tx9 should be sorted low Assert.Equal(9, pool.Size); sortedOrder.Insert(0, tx9.GetHash().ToString()); this.CheckSort(pool, pool.MapTx.DescendantScore.ToList(), sortedOrder); List <string> snapshotOrder = sortedOrder.ToList(); setAncestors.Add(pool.MapTx.TryGet(tx8.GetHash())); setAncestors.Add(pool.MapTx.TryGet(tx9.GetHash())); /* tx10 depends on tx8 and tx9 and has a high fee*/ var tx10 = new Transaction(); tx10.AddInput(new TxIn(new OutPoint(tx8.GetHash(), 0), new Script(OpcodeType.OP_11))); tx10.AddInput(new TxIn(new OutPoint(tx9.GetHash(), 0), new Script(OpcodeType.OP_11))); tx10.AddOutput(new TxOut(new Money(10 * Money.COIN), new Script(OpcodeType.OP_11, OpcodeType.OP_EQUAL))); setAncestorsCalculated.Clear(); Assert.True(pool.CalculateMemPoolAncestors(entry.Fee(200000L).Time(4).FromTx(tx10), setAncestorsCalculated, 100, 1000000, 1000, 1000000, out dummy)); Assert.True(setAncestorsCalculated.Equals(setAncestors)); pool.AddUnchecked(tx10.GetHash(), entry.FromTx(tx10), setAncestors); /** * tx8 and tx9 should both now be sorted higher * Final order after tx10 is added: * * tx3 = 0 (1) * tx5 = 10000 (1) * tx1 = 10000 (1) * tx4 = 15000 (1) * tx2 = 20000 (1) * tx9 = 200k (2 txs) * tx8 = 200k (2 txs) * tx10 = 200k (1 tx) * tx6 = 2.2M (5 txs) * tx7 = 2.2M (4 txs) */ sortedOrder.RemoveRange(0, 2); // take out tx9, tx8 from the beginning sortedOrder.Insert(5, tx9.GetHash().ToString()); sortedOrder.Insert(6, tx8.GetHash().ToString()); sortedOrder.Insert(7, tx10.GetHash().ToString()); // tx10 is just before tx6 this.CheckSort(pool, pool.MapTx.DescendantScore.ToList(), sortedOrder); // there should be 10 transactions in the mempool Assert.Equal(10, pool.Size); // Now try removing tx10 and verify the sort order returns to normal pool.RemoveRecursive(pool.MapTx.TryGet(tx10.GetHash()).Transaction); this.CheckSort(pool, pool.MapTx.DescendantScore.ToList(), snapshotOrder); pool.RemoveRecursive(pool.MapTx.TryGet(tx9.GetHash()).Transaction); pool.RemoveRecursive(pool.MapTx.TryGet(tx8.GetHash()).Transaction); /* Now check the sort on the mining score index. * Final order should be: * * tx7 (2M) * tx2 (20k) * tx4 (15000) * tx1/tx5 (10000) * tx3/6 (0) * (Ties resolved by hash) */ sortedOrder.Clear(); sortedOrder.Add(tx7.GetHash().ToString()); sortedOrder.Add(tx2.GetHash().ToString()); sortedOrder.Add(tx4.GetHash().ToString()); if (tx1.GetHash() < tx5.GetHash()) { sortedOrder.Add(tx5.GetHash().ToString()); sortedOrder.Add(tx1.GetHash().ToString()); } else { sortedOrder.Add(tx1.GetHash().ToString()); sortedOrder.Add(tx5.GetHash().ToString()); } if (tx3.GetHash() < tx6.GetHash()) { sortedOrder.Add(tx6.GetHash().ToString()); sortedOrder.Add(tx3.GetHash().ToString()); } else { sortedOrder.Add(tx3.GetHash().ToString()); sortedOrder.Add(tx6.GetHash().ToString()); } this.CheckSort(pool, pool.MapTx.MiningScore.ToList(), sortedOrder); }