public void When_two_branches_with_two_same_children_change_one_and_change_back_next_block()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Set(_keyB, _longLeaf1);
patriciaTree.Set(_keyC, _longLeaf1);
patriciaTree.Set(_keyD, _longLeaf1);
patriciaTree.UpdateRootHash();
patriciaTree.Commit(0);
patriciaTree.Set(_keyA, _longLeaf3);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.UpdateRootHash();
patriciaTree.Commit(1);
memDb.Keys.Should().HaveCount(5);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(_keyA).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(_keyB).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(_keyC).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(_keyD).Should().BeEquivalentTo(_longLeaf1);
}
public void Connect_extension_with_extension()
{
/* to test this case we need something like this initially */
/* R
* E - - - - - - - - - - - - - - -
* B B B B B B B B B B B B B B B B
* E L - - - - - - - - - - - - - -
* E - - - - - - - - - - - - - - -
* B B B B B B B B B B B B B B B B
* L L - - - - - - - - - - - - - - */
/* then we delete the leaf (marked as X) */
/* R
* B B B B B B B B B B B B B B B B
* E X - - - - - - - - - - - - - -
* E - - - - - - - - - - - - - - -
* B B B B B B B B B B B B B B B B
* L L - - - - - - - - - - - - - - */
/* and we end up with an extended extension replacing what was previously a top-level branch*/
/* R
* E
* E
* E - - - - - - - - - - - - - - -
* B B B B B B B B B B B B B B B B
* L L - - - - - - - - - - - - - - */
byte[] key1 = Bytes.FromHexString("000000100000000aa");
byte[] key2 = Bytes.FromHexString("000000100000000bb");
byte[] key3 = Bytes.FromHexString("000000200000000cc");
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(key1, _longLeaf1);
patriciaTree.Set(key2, _longLeaf1);
patriciaTree.Set(key3, _longLeaf1);
patriciaTree.UpdateRootHash();
patriciaTree.Commit(0);
patriciaTree.Set(key3, Array.Empty <byte>());
patriciaTree.UpdateRootHash();
patriciaTree.Commit(1);
memDb.Keys.Should().HaveCount(8);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(key1).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(key2).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(key3).Should().BeNull();
}
public void Delete_missing_resolved_on_leaf()
{
PatriciaTree patriciaTree = new PatriciaTree(_db, Keccak.EmptyTreeHash, false, true);
patriciaTree.Set(Keccak.Compute("1234567").Bytes, new byte[] { 1 });
patriciaTree.Set(Keccak.Compute("1234501").Bytes, new byte[] { 2 });
patriciaTree.UpdateRootHash();
Keccak rootBefore = patriciaTree.RootHash;
patriciaTree.Set(Keccak.Compute("1234502").Bytes, new byte[0]);
patriciaTree.UpdateRootHash();
Assert.AreEqual(rootBefore, patriciaTree.RootHash);
}
public void Delete_missing_resolved_on_extension()
{
PatriciaTree patriciaTree = new PatriciaTree(_db, Keccak.EmptyTreeHash, false, true);
patriciaTree.Set(new Nibble[] { 1, 2, 3, 4 }.ToPackedByteArray(), new byte[] { 1 });
patriciaTree.Set(new Nibble[] { 1, 2, 3, 4, 5 }.ToPackedByteArray(), new byte[] { 2 });
patriciaTree.UpdateRootHash();
Keccak rootBefore = patriciaTree.RootHash;
patriciaTree.Set(new Nibble[] { 1, 2, 3 }.ToPackedByteArray(), new byte[] { });
patriciaTree.UpdateRootHash();
Assert.AreEqual(rootBefore, patriciaTree.RootHash);
}
public void Test_update_many(int i)
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, new MemoryLimit(128.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
for (int j = 0; j < i; j++)
{
Keccak key = TestItem.Keccaks[j];
byte[] value = GenerateIndexedAccountRlp(j);
patriciaTree.Set(key.Bytes, value);
}
for (int j = 0; j < i; j++)
{
Keccak key = TestItem.Keccaks[j];
byte[] value = GenerateIndexedAccountRlp(j + 1);
patriciaTree.Set(key.Bytes, value);
}
patriciaTree.Commit(0);
patriciaTree.UpdateRootHash();
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
for (int j = 0; j < i; j++)
{
Keccak key = TestItem.Keccaks[j];
byte[] value = GenerateIndexedAccountRlp(j + 1);
checkTree.Get(key.Bytes).Should().BeEquivalentTo(value, $@"{i} {j}");
}
}
private void RunTest(TrieTest test, bool secure)
{
string permutationDescription =
string.Join(Environment.NewLine, test.Input.Select(p => $"{p.Key} -> {p.Value}"));
TestContext.WriteLine(Surrounded(permutationDescription));
PatriciaTree patriciaTree = secure ? new SecurePatriciaTree(_db) : new PatriciaTree(_db, Keccak.EmptyTreeHash, false);
foreach (KeyValuePair <string, string> keyValuePair in test.Input)
{
string keyString = keyValuePair.Key;
string valueString = keyValuePair.Value;
Nibble[] key = keyString.StartsWith("0x")
? Hex.ToNibbles(keyString)
: Nibbles.FromBytes(Encoding.ASCII.GetBytes(keyString));
byte[] value = valueString.StartsWith("0x")
? Hex.ToBytes(valueString)
: Encoding.ASCII.GetBytes(valueString);
TestContext.WriteLine();
TestContext.WriteLine($"Setting {keyString} -> {valueString}");
patriciaTree.Set(key, value);
}
patriciaTree.UpdateRootHash();
Assert.AreEqual(test.ExpectedRoot, patriciaTree.RootHash.ToString());
}
public void Single_leaf_and_keep_for_multiple_dispatches_then_delete()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), new ConstantInterval(4), LimboLogs.Instance);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Commit(0);
patriciaTree.Commit(1);
patriciaTree.Commit(2);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Commit(3);
patriciaTree.Commit(4);
patriciaTree.Set(_keyA, Array.Empty <byte>());
patriciaTree.Commit(5);
patriciaTree.Set(_keyB, _longLeaf2);
patriciaTree.Commit(6);
patriciaTree.Commit(7);
patriciaTree.Commit(8);
patriciaTree.Commit(9);
patriciaTree.Commit(10);
patriciaTree.Commit(11);
patriciaTree.Set(_keyB, Array.Empty <byte>());
patriciaTree.Commit(12);
patriciaTree.Commit(13);
patriciaTree.UpdateRootHash();
// leaf (root)
memDb.Keys.Should().HaveCount(2);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(_keyA).Should().BeNull();
checkTree.Get(_keyB).Should().BeNull();
}
public void Extension_branch_extension_and_leaf_then_branch_leaf_leaf()
{
/* R
* E - - - - - - - - - - - - - - -
* B B B B B B B B B B B B B B B B
* E L - - - - - - - - - - - - - -
* E - - - - - - - - - - - - - - -
* B B B B B B B B B B B B B B B B
* L L - - - - - - - - - - - - - - */
byte[] key1 = Bytes.FromHexString("000000100000000aa");
byte[] key2 = Bytes.FromHexString("000000100000000bb");
byte[] key3 = Bytes.FromHexString("000000200000000cc");
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(key1, _longLeaf1);
patriciaTree.Set(key2, _longLeaf1);
patriciaTree.Set(key3, _longLeaf1);
patriciaTree.UpdateRootHash();
patriciaTree.Commit(0);
memDb.Keys.Should().HaveCount(7);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(key1).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(key2).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(key3).Should().BeEquivalentTo(_longLeaf1);
}
public void Test_add_and_delete_many_next_block(int i)
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, new MemoryLimit(128.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
for (int j = 0; j < i; j++)
{
Keccak key = TestItem.Keccaks[j];
byte[] value = GenerateIndexedAccountRlp(j);
patriciaTree.Set(key.Bytes, value);
}
patriciaTree.Commit(0);
for (int j = 0; j < i; j++)
{
Keccak key = TestItem.Keccaks[j];
patriciaTree.Set(key.Bytes, Array.Empty <byte>());
}
patriciaTree.Commit(1);
patriciaTree.UpdateRootHash();
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
for (int j = 0; j < i; j++)
{
Keccak key = TestItem.Keccaks[j];
checkTree.Get(key.Bytes).Should().BeNull($@"{i} {j}");
}
}
public void When_branch_with_two_different_children_change_one_and_change_back_next_block()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Set(_keyB, _longLeaf2);
patriciaTree.UpdateRootHash();
patriciaTree.Commit(0);
patriciaTree.Set(_keyA, _longLeaf3);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.UpdateRootHash();
patriciaTree.Commit(1);
// extension
// branch
// leaf x 2
memDb.Keys.Should().HaveCount(4);
}
private void SetReceiptsRootAndBloom(Block block, TransactionReceipt[] transactionReceipts)
{
PatriciaTree receiptTree = transactionReceipts.Length > 0 ? new PatriciaTree(NullDb.Instance, Keccak.EmptyTreeHash, false) : null;
for (int i = 0; i < transactionReceipts.Length; i++)
{
Rlp receiptRlp = Rlp.Encode(transactionReceipts[i], _specProvider.GetSpec(block.Header.Number).IsEip658Enabled ? RlpBehaviors.Eip658Receipts : RlpBehaviors.None);
receiptTree?.Set(Rlp.Encode(i).Bytes, receiptRlp);
}
receiptTree?.UpdateRootHash();
block.Header.ReceiptsRoot = receiptTree?.RootHash ?? PatriciaTree.EmptyTreeHash;
block.Header.Bloom = transactionReceipts.Length > 0 ? BuildBloom(transactionReceipts) : Bloom.Empty;
}
private void SetReceipts(Block block, TransactionReceipt[] receipts)
{
PatriciaTree receiptTree = receipts.Length > 0 ? new PatriciaTree(NullDb.Instance, Keccak.EmptyTreeHash, false) : null;
for (int i = 0; i < receipts.Length; i++)
{
Rlp receiptRlp = Rlp.Encode(receipts[i], _specProvider.GetSpec(block.Header.Number).IsEip658Enabled ? RlpBehaviors.Eip658Receipts : RlpBehaviors.None);
receiptTree?.Set(Rlp.Encode(i).Bytes, receiptRlp);
}
receiptTree?.UpdateRootHash();
block.Header.ReceiptsRoot = receiptTree?.RootHash ?? PatriciaTree.EmptyTreeHash;
block.Header.Bloom = receipts.Length > 0 ? TransactionProcessor.BuildBloom(receipts.SelectMany(r => r.Logs).ToArray()) : Bloom.Empty; // TODO not tested anywhere at the time of writing
}
public static Keccak CalculateReceiptRoot(this Block block, ISpecProvider specProvider, TxReceipt[] txReceipts)
{
PatriciaTree receiptTree = txReceipts.Length > 0 ? new PatriciaTree(NullDb.Instance, Keccak.EmptyTreeHash, false) : null;
for (int i = 0; i < txReceipts.Length; i++)
{
Rlp receiptRlp = Rlp.Encode(txReceipts[i], specProvider.GetSpec(block.Number).IsEip658Enabled ? RlpBehaviors.Eip658Receipts : RlpBehaviors.None);
receiptTree?.Set(Rlp.Encode(i).Bytes, receiptRlp);
}
receiptTree?.UpdateRootHash();
Keccak receiptRoot = receiptTree?.RootHash ?? PatriciaTree.EmptyTreeHash;
return(receiptRoot);
}
private Keccak GetTransactionsRoot(Transaction[] transactions)
{
if (transactions.Length == 0)
{
return(PatriciaTree.EmptyTreeHash);
}
PatriciaTree txTree = new PatriciaTree();
for (int i = 0; i < transactions.Length; i++)
{
Rlp transactionRlp = Rlp.Encode(transactions[i]);
txTree.Set(Rlp.Encode(i).Bytes, transactionRlp);
}
txTree.UpdateRootHash();
return(txTree.RootHash);
}
public static Keccak CalculateReceiptRoot(this Block block, ISpecProvider specProvider, TxReceipt[] txReceipts)
{
if (txReceipts.Length == 0)
{
return(PatriciaTree.EmptyTreeHash);
}
PatriciaTree receiptTree = new PatriciaTree();
for (int i = 0; i < txReceipts.Length; i++)
{
Rlp receiptRlp = Rlp.Encode(txReceipts[i], specProvider.GetSpec(block.Number).IsEip658Enabled ? RlpBehaviors.Eip658Receipts : RlpBehaviors.None);
receiptTree.Set(Rlp.Encode(i).Bytes, receiptRlp);
}
receiptTree.UpdateRootHash();
return(receiptTree.RootHash);
}
public static Keccak CalculateTxRoot(this Block block)
{
if (block.Transactions.Length == 0)
{
return(PatriciaTree.EmptyTreeHash);
}
PatriciaTree txTree = new PatriciaTree();
for (int i = 0; i < block.Transactions.Length; i++)
{
Rlp transactionRlp = Rlp.Encode(block.Transactions[i]);
txTree.Set(Rlp.Encode(i).Bytes, transactionRlp);
}
txTree.UpdateRootHash();
return(txTree.RootHash);
}
public void Single_leaf_delete_next_block()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Commit(0);
patriciaTree.Set(_keyA, Array.Empty <byte>());
patriciaTree.Commit(1);
patriciaTree.UpdateRootHash();
// leaf (root)
memDb.Keys.Should().HaveCount(1);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(_keyA).Should().BeNull();
}
public void Test_try_delete_and_read_missing_nodes(int i)
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, new MemoryLimit(128.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, Keccak.EmptyTreeHash, true, true, _logManager);
for (int j = 0; j < i; j++)
{
Keccak key = TestItem.Keccaks[j];
byte[] value = GenerateIndexedAccountRlp(j);
patriciaTree.Set(key.Bytes, value);
}
// delete missing
for (int j = 0; j < i; j++)
{
Keccak key = TestItem.Keccaks[j + 100];
patriciaTree.Set(key.Bytes, Array.Empty <byte>());
}
patriciaTree.Commit(0);
patriciaTree.UpdateRootHash();
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
// confirm nothing deleted
for (int j = 0; j < i; j++)
{
Keccak key = TestItem.Keccaks[j];
byte[] value = GenerateIndexedAccountRlp(j);
checkTree.Get(key.Bytes).Should().BeEquivalentTo(value, $@"{i} {j}");
}
// read missing
for (int j = 0; j < i; j++)
{
Keccak key = TestItem.Keccaks[j + 100];
checkTree.Get(key.Bytes).Should().BeNull();
}
}
private void RunTest(TrieTest test, bool secure)
{
if (secure)
{
// removed the implementation of secure trie as it was not used outside of tests
return;
}
string permutationDescription =
string.Join(Environment.NewLine, test.Input.Select(p => $"{p.Key} -> {p.Value}"));
TestContext.WriteLine(Surrounded(permutationDescription));
PatriciaTree patriciaTree = new PatriciaTree(_db, Keccak.EmptyTreeHash, false, true);
foreach (KeyValuePair <string, string> keyValuePair in test.Input)
{
string keyString = keyValuePair.Key;
string valueString = keyValuePair.Value;
Nibble[] key = keyString.StartsWith("0x")
? Nibbles.FromHexString(keyString)
: Nibbles.FromBytes(Encoding.ASCII.GetBytes(keyString));
byte[] value = valueString.StartsWith("0x")
? Bytes.FromHexString(valueString)
: Encoding.ASCII.GetBytes(valueString);
TestContext.WriteLine();
TestContext.WriteLine($"Setting {keyString} -> {valueString}");
patriciaTree.Set(key.ToPackedByteArray(), value);
}
patriciaTree.UpdateRootHash();
Assert.AreEqual(test.ExpectedRoot, patriciaTree.RootHash.ToString());
}
public void Fuzz_accounts_with_reorganizations(
int accountsCount,
int blocksCount,
int uniqueValuesCount,
int lookupLimit,
int?seed)
{
int usedSeed = seed ?? _random.Next(int.MaxValue);
_random = new Random(usedSeed);
_logger.Info($"RANDOM SEED {usedSeed}");
string fileName = Path.GetTempFileName();
//string fileName = "C:\\Temp\\fuzz.txt";
_logger.Info(
$"Fuzzing with accounts: {accountsCount}, " +
$"blocks {blocksCount}, " +
$"values: {uniqueValuesCount}, " +
$"lookup: {lookupLimit} into file {fileName}");
using FileStream fileStream = new FileStream(fileName, FileMode.Create);
using StreamWriter streamWriter = new StreamWriter(fileStream);
Queue <Keccak> rootQueue = new Queue <Keccak>();
Stack <Keccak> rootStack = new Stack <Keccak>();
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.IfBlockOlderThan(lookupLimit), _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
byte[][] accounts = new byte[accountsCount][];
byte[][] randomValues = new byte[uniqueValuesCount][];
for (int i = 0; i < randomValues.Length; i++)
{
bool isEmptyValue = _random.Next(0, 2) == 0;
if (isEmptyValue)
{
randomValues[i] = Array.Empty <byte>();
}
else
{
randomValues[i] = GenerateRandomAccountRlp();
}
}
for (int accountIndex = 0; accountIndex < accounts.Length; accountIndex++)
{
byte[] key = new byte[32];
((UInt256)accountIndex).ToBigEndian(key);
accounts[accountIndex] = key;
}
int blockCount = 0;
for (int blockNumber = 0; blockNumber < blocksCount; blockNumber++)
{
int reorgDepth = _random.Next(Math.Min(5, blockCount));
_logger.Debug($"Reorganizing {reorgDepth}");
for (int i = 0; i < reorgDepth; i++)
{
try
{
// no longer need undo?
// trieStore.UndoOneBlock();
}
catch (InvalidOperationException)
{
// if memory limit hits in
blockCount = 0;
}
rootStack.Pop();
patriciaTree.RootHash = rootStack.Peek();
}
blockCount = Math.Max(0, blockCount - reorgDepth);
_logger.Debug($"Setting block count to {blockCount}");
bool isEmptyBlock = _random.Next(5) == 0;
if (!isEmptyBlock)
{
for (int i = 0; i < Math.Max(1, accountsCount / 8); i++)
{
int randomAccountIndex = _random.Next(accounts.Length);
int randomValueIndex = _random.Next(randomValues.Length);
byte[] account = accounts[randomAccountIndex];
byte[] value = randomValues[randomValueIndex];
streamWriter.WriteLine(
$"Block {blockCount} - setting {account.ToHexString()} = {value.ToHexString()}");
patriciaTree.Set(account, value);
}
}
streamWriter.WriteLine(
$"Commit block {blockCount} | empty: {isEmptyBlock}");
patriciaTree.UpdateRootHash();
patriciaTree.Commit(blockCount);
rootQueue.Enqueue(patriciaTree.RootHash);
rootStack.Push(patriciaTree.RootHash);
blockCount++;
_logger.Debug($"Setting block count to {blockCount}");
}
streamWriter.Flush();
fileStream.Seek(0, SeekOrigin.Begin);
streamWriter.WriteLine($"DB size: {memDb.Keys.Count}");
_logger.Info($"DB size: {memDb.Keys.Count}");
int verifiedBlocks = 0;
rootQueue.Clear();
Stack <Keccak> stackCopy = new Stack <Keccak>();
while (rootStack.Any())
{
stackCopy.Push(rootStack.Pop());
}
rootStack = stackCopy;
while (rootStack.TryPop(out Keccak currentRoot))
{
try
{
patriciaTree.RootHash = currentRoot;
for (int i = 0; i < accounts.Length; i++)
{
patriciaTree.Get(accounts[i]);
}
_logger.Info($"Verified positive {verifiedBlocks}");
}
catch (Exception ex)
{
if (verifiedBlocks % lookupLimit == 0)
{
throw new InvalidDataException(ex.ToString());
}
else
{
_logger.Info($"Verified negative {verifiedBlocks} (which is ok on block {verifiedBlocks})");
}
}
verifiedBlocks++;
}
}
