// RLP encoding/decoding trie nodes
/// <summary>
/// Given a trie node, encodes it into an RLP item such that if it is 32 bytes or less encoded, it is directly included, otherwise it will be a 32 byte reference to the data.
/// </summary>
/// <param name="node">The node to encode into an RLP item for storage in the trie.</param>
/// <returns>Returns the RLP encoded trie node as it should be represented in our trie efficienctly.</returns>
private RLPItem EncodeNode(RLPList node)
{
// If it's a blank node, we return blank.
if (node == null || node.Items.Count == 0)
{
return(BLANK_NODE);
}
// Obtain our node type
TrieNodeType type = GetNodeType(node);
byte[] encoded = RLP.Encode(node);
// If our RLP encoded node is less than 32 bytes, we'll include the node directly as an RLP list.
if (encoded.Length < 32)
{
return(node);
}
// Otherwise if the data is 32 bytes or longer, we encode it as an RLP byte array with the hash to the node in the database.
// Get our hash key
byte[] hashKey = KeccakHash.ComputeHashBytes(encoded);
// Set in our database.
Database.Set(hashKey, encoded);
// Return as an RLP byte array featuring the lookup hash/key for the encoded node.
return(hashKey);
}
/// <summary>
/// Given a trie node type, and a pair of nibbles, packs them into a prefix/shared-nibbles pair (extension) or prefix/key-remainder pair (leaf) respectively.
/// </summary>
/// <param name="type">The trie node type to pack into the first item of our node.</param>
/// <param name="nibbles">The nibble array to pack into the first item of our node.</param>
/// <returns>Returns a packed byte array which represents the new data for the first item of our node.</returns>
private byte[] PackPrefixedNode(TrieNodeType type, byte[] nibbles)
{
// Determine the base prefix for the data (we set our first bit (odd/even) next).
TrieNodePrefix prefix = TrieNodePrefix.ExtensionNodeEven;
if (type == TrieNodeType.Leaf)
{
prefix = TrieNodePrefix.LeafNodeEven;
}
// We set the first bit based off of if it's even or odd length.
bool oddLength = (nibbles.Length % 2) != 0;
if (oddLength)
{
prefix = (TrieNodePrefix)((int)prefix | 1);
}
// If it's odd, just adding our prefix will make it byte-aligned. Otherwise we add a zero after the prefix.
if (oddLength)
{
nibbles = new byte[] { (byte)prefix }.Concat(nibbles);
}
else
{
nibbles = new byte[] { (byte)prefix, 0 }.Concat(nibbles);
}
return(NibbleArrayToByteArray(nibbles));
}
/// <summary>
/// Given an trie node, traverses down all paths to obtain all key-value pairs in the trie.
/// </summary>
/// <param name="node">The trie node to recursively enumerate key-value pairs from.</param>
/// <param name="currentNibbles">The current key nibbles that have been traversed up to this point.</param>
/// <param name="result">The resulting dictionary created by obtaining all pairs from this node recursively downward.</param>
private void NodeGetAllPairs(RLPList node, byte[] currentNibbles, Dictionary <Memory <byte>, byte[]> result)
{
// Obtain our node type
TrieNodeType nodeType = GetNodeType(node);
// Switch on node type
switch (nodeType)
{
// If it's blank, return the blank node
case TrieNodeType.Blank:
return;
case TrieNodeType.Branch:
// Check we have data set
Memory <byte> valueData = ((RLPByteArray)node.Items[16]).Data;
if (valueData.Length > 0)
{
// Set our key-value pair at this node in our results
result[NibbleArrayToByteArray(currentNibbles)] = valueData.ToArray();
}
// Loop for every branch in this branch node (every sub item is a branch except for the last one,
// which is the value if no more nibbles are needed to index it).
for (byte branchIndex = 0; branchIndex < node.Items.Count - 1; branchIndex++)
{
// Obtain our node at the given index and obtain that item
RLPList branchNode = DecodeNode(node.Items[branchIndex]);
// Keep running down the branch recursively.
NodeGetAllPairs(branchNode, currentNibbles.Concat(new byte[] { branchIndex }), result);
}
return;
case TrieNodeType.Extension:
// If it's an extension, we'll unpack the whole nibble set of our first item (prefix + key remainder)
byte[] sharedNibbles = UnpackPrefixedNode(node.Items[0]).nibbles;
// The second item in this node will represent the "next node", so we recursively obtain from that, advancing our key position as well.
RLPList nextNode = DecodeNode(node.Items[1]);
NodeGetAllPairs(nextNode, currentNibbles.Concat(sharedNibbles), result);
return;
case TrieNodeType.Leaf:
// If it's a leaf, we'll unpack the whole nibble set of our first item (prefix + key remainder)
sharedNibbles = UnpackPrefixedNode(node.Items[0]).nibbles;
// Otherwise we passed verification, so we return the "value" that is stored in this leaf.
result[NibbleArrayToByteArray(currentNibbles.Concat(sharedNibbles))] = ((RLPByteArray)node.Items[1]).Data.ToArray();
return;
}
// Throw an exception if we somehow end up here.
throw new ArgumentException("Could not obtain key-value pairs, invalid node type detected!");
}
private byte[] Encode(int depth, bool forceHash)
{
if (!dirty)
{
return(hash != null?RLP.EncodeElement(hash) : rlp);
}
else
{
TrieNodeType type = this.node_type;
byte[] ret;
if (type == TrieNodeType.BranchNode)
{
if (depth == 1 && this.reference.Async)
{
// parallelize encode() on the first trie level only and if there are at least
// MIN_BRANCHES_CONCURRENTLY branches are modified
object[] encoded = new object[17];
int encode_count = 0;
for (int i = 0; i < 16; i++)
{
TrieNode child = BranchNodeGetChild(i);
if (child == null)
{
encoded[i] = RLP.EMPTY_ELEMENT_RLP;
}
else if (!child.dirty)
{
encoded[i] = child.Encode(depth + 1, false);
}
else
{
encode_count++;
}
}
for (int i = 0; i < 16; i++)
{
if (encoded[i] == null)
{
TrieNode child = BranchNodeGetChild(i);
if (child == null)
{
continue;
}
if (encode_count >= MIN_BRANCHES_CONCURRENTLY)
{
encoded[i] = Task.Run <byte[]>(() =>
{
return(child.Encode(depth + 1, false));
});
}
else
{
encoded[i] = child.Encode(depth + 1, false);
}
}
}
byte[] value = BranchNodeGetValue();
encoded[16] = RLP.EncodeElement(value);
try
{
ret = EncodeRlpListTaskResult(encoded);
}
catch (System.Exception e)
{
throw new System.Exception(e.Message);
}
}
else
{
byte[][] encoded = new byte[17][];
for (int i = 0; i < 16; i++)
{
TrieNode child = BranchNodeGetChild(i);
encoded[i] = child == null ? RLP.EMPTY_ELEMENT_RLP : child.Encode(depth + 1, false);
}
byte[] value = BranchNodeGetValue();
encoded[16] = RLP.EncodeElement(value);
ret = RLP.EncodeList(encoded);
}
}
else if (type == TrieNodeType.KVNodeNode)
{
ret = RLP.EncodeList(RLP.EncodeElement(KVNodeGetKey().ToPacked()),
KVNodeGetChildNode().Encode(depth + 1, false));
}
else
{
byte[] value = KVNodeGetValue();
ret = RLP.EncodeList(RLP.EncodeElement(KVNodeGetKey().ToPacked()),
RLP.EncodeElement(value == null ? new byte[0] : value));
}
if (this.hash != null)
{
this.reference.DeleteHash(this.hash);
}
this.dirty = false;
if (ret.Length < 32 && !forceHash)
{
this.rlp = ret;
return(ret);
}
else
{
this.hash = ret.SHA3();
this.reference.AddHash(this.hash, ret);
return(RLP.EncodeElement(hash));
}
}
}
/// <summary>
/// Given a branch node that has had a branch path removed, reviews the node to see if it should be another type, and returns a replacement node for the provided node.
/// </summary>
/// <param name="node">The branch node to clean up and replace with an updated/reformatted node.</param>
/// <returns>Returns an updated/reformatted node which accounts for a deleted branch path, possibly homogenizing the downward path, such that it should be a leaf instead of a branch, etc.</returns>
private RLPList CleanupBranch(RLPList node)
{
// If the node is null or does not have 17 items, then it is not an extension and we do not change it.
if (node == null || node.Items.Count != 17)
{
return(node);
}
// After deletion, we need to clean up adjacent branches which may be unnecessary and could possibly be shortened.
int nonBlankCount = 0;
int lastNonBlankIndex = -1;
for (int i = 0; i < node.Items.Count; i++)
{
if (DecodeNode(node.Items[i]) != null)
{
nonBlankCount++;
lastNonBlankIndex = i;
}
}
// If we have more than one subnode referenced, this branch node is necessary.
if (nonBlankCount > 1)
{
return(node);
}
// If only the value item is non blank, then we can convert this to a leaf with a blank key remainder, and the same value slot.
if (lastNonBlankIndex == 16)
{
return(new RLPList(PackPrefixedNode(TrieNodeType.Leaf, Array.Empty <byte>()), node.Items[16]));
}
// Otherwise it was a branch that was changed, so we obtain the branch subnode
RLPList subNode = DecodeNode(node.Items[lastNonBlankIndex]);
TrieNodeType subNodeType = GetNodeType(subNode);
// If our sub node type is also a branch, we convert this into an extension since its a straight path with shared nibbles. We convert the branch nibble to an extension shared nibble.
if (subNodeType == TrieNodeType.Branch)
{
// Pack our prefix nibble set (using branch nibble as a shared nibble) and return our extension node which links to our branch subnode.
return(new RLPList(PackPrefixedNode(TrieNodeType.Extension, new byte[] { (byte)lastNonBlankIndex }), EncodeNode(subNode)));
}
// If it's a leaf or extension (a node with key and value), the subnode takes the place of this one, and we prefix our subnode's key nibbles with our branch key nibble.
else if (subNodeType == TrieNodeType.Leaf || subNodeType == TrieNodeType.Extension)
{
// Obtain our unpacked prefix/nibble set.
var subNodePrefixUnpacked = UnpackPrefixedNode(subNode.Items[0]);
// Add our branch nibble, and pack it back into the packed prefix node
subNodePrefixUnpacked.nibbles = new byte[] { (byte)lastNonBlankIndex }.Concat(subNodePrefixUnpacked.nibbles);
// Create our modified leaf/extension with it's longer "key" portion and the same "value" and return it
return(new RLPList(PackPrefixedNode(subNodePrefixUnpacked.type, subNodePrefixUnpacked.nibbles), subNode.Items[1]));
}
else
{
// Otherwise if we somehow end up here, throw an error.
throw new ArgumentException("Could not clean up branch node because it has unexpected conditions!");
}
}
/// <summary>
/// Given a trie node, traverses down the appropriate node path, removing the key/value for a given key and the current nibble index for our key in the indexing process.
/// </summary>
/// <param name="node">The current trie node to traverse down to remove our key/value, given our key and current nibble index.</param>
/// <param name="key">The key for the key/value which we wish to remove.</param>
/// <param name="currentNibbleIndex">The index of the current nibble in our key which we are at in our indexing process.</param>
/// <returns>Returns a node with the given removal which is to be used as the replacement for the provided node.</returns>
private RLPList NodeRemoveValue(RLPList node, Memory <byte> key, int currentNibbleIndex = 0)
{
// Obtain the node type
TrieNodeType nodeType = GetNodeType(node);
// Switch on our node type
switch (nodeType)
{
// If it's a blank node, we return the blank node.
case TrieNodeType.Blank:
return(null);
case TrieNodeType.Branch:
// If we reached the end of key, we remove the value from here (since our value for this key resides here), and we cleanup/reformat this branch.
if (currentNibbleIndex == key.Length * 2)
{
node.Items[16] = EncodeNode(null);
return(CleanupBranch(node));
}
// Otherwise we obtain delete down the branch path, so we obtain the branch node.
byte branchNodeIndex = GetNibble(key, currentNibbleIndex);
RLPList branchNode = DecodeNode(node.Items[branchNodeIndex]);
// Delete by going down the branch node, and obtain the resulting branch node.
RLPList newBranchNode = NodeRemoveValue(NodeDuplicate(branchNode), key, currentNibbleIndex + 1);
// If the new node is the same as the old, we can return our node as it is. This node only changes if the path below changed.
if (NodeEquals(branchNode, newBranchNode))
{
return(node);
}
// Otherwise we set our updated now in our branch index.
node.Items[branchNodeIndex] = EncodeNode(newBranchNode);
// If our new node is null, then should check if this node might need to be fixed up, otherwise we can return the node as it is.
if (newBranchNode == null)
{
return(CleanupBranch(node));
}
else
{
return(node);
}
case TrieNodeType.Leaf:
// Obtain our key remainder nibbles from our leaf.
byte[] nibbles = UnpackPrefixedNode(node.Items[0]).nibbles;
// If the remainder of our key doesnt match the key remainder on the leaf, return the node as it is, it isn't the leaf we were searching for.
if ((key.Length * 2) - currentNibbleIndex != nibbles.Length)
{
return(node);
}
// Any key remainder nibbles we find in this nibble set should match the remainder of our key.
for (int i = 0; i < nibbles.Length; i++)
{
if (nibbles[i] != GetNibble(key, currentNibbleIndex + i))
{
return(node);
}
}
// We can now confirm the key has matched, hence this is the leaf we're searching for, and we return a blank node to replace it.
return(null);
case TrieNodeType.Extension:
// Obtain our key shared nibbles from our extension.
nibbles = UnpackPrefixedNode(node.Items[0]).nibbles;
// If the remainder of our key is smaller than the shared nibbles, return the node as it is, the key doesn't exist down this path.
if ((key.Length * 2) - currentNibbleIndex < nibbles.Length)
{
return(node);
}
// Any shared nibbles we find in this nibble set should match the remainder of our key.
for (int i = 0; i < nibbles.Length; i++)
{
if (nibbles[i] != GetNibble(key, currentNibbleIndex + i))
{
return(node);
}
}
// Obtain our decoded next node.
RLPList nextNode = DecodeNode(node.Items[1]);
RLPList newNextNode = NodeRemoveValue(NodeDuplicate(nextNode), key, currentNibbleIndex + nibbles.Length);
// If the next node didn't change at all, we can simply keep this node as it is.
if (NodeEquals(nextNode, newNextNode))
{
return(node);
}
// If our new next node is null, then an extension shouldn't exist here, so we return null, changes should propogate changes upwards.
if (newNextNode == null)
{
return(null);
}
// Get our new next node type
TrieNodeType newNodeType = GetNodeType(newNextNode);
// If the new node type is a branch..
if (newNodeType == TrieNodeType.Branch)
{
// We return a fixed up extension, with our new encoded next node.
return(new RLPList(PackPrefixedNode(TrieNodeType.Extension, nibbles), EncodeNode(newNextNode)));
}
else
{
// Obtain the subnodes prefix/nibbles
var subNodePrefixNibbles = UnpackPrefixedNode(newNextNode.Items[0]);
// If the new node type is an extension or leaf, then we instead join it with this extension.
return(new RLPList(PackPrefixedNode(subNodePrefixNibbles.type, nibbles.Concat(subNodePrefixNibbles.nibbles)), newNextNode.Items[1]));
}
}
// If we somehow end up here, throw an exception.
throw new ArgumentException("Unexpected node type while removing trie nodes.");
}
/// <summary>
/// Given a trie node, traverses down the appropriate node path, updating the value for a given key and the current nibble index for our key in the indexing process.
/// </summary>
/// <param name="node">The current trie node to traverse down to update our value, given our key and current nibble index.</param>
/// <param name="key">The key for the value which we wish to update.</param>
/// <param name="currentNibbleIndex">The index of the current nibble in our key which we are at in our indexing process.</param>
/// <param name="value">The value which we wish to set for the provided key in our trie.</param>
/// <returns>Returns a node with the given update which is to be used as the replacement for the provided node.</returns>
private RLPList NodeUpdateValue(RLPList node, Memory <byte> key, int currentNibbleIndex, byte[] value)
{
// Obtain the node type
TrieNodeType nodeType = GetNodeType(node);
// Switch on our node type
switch (nodeType)
{
case TrieNodeType.Blank:
// Obtain the remainder of our key as a nibble sequence.
byte[] keyNibbles = ByteArrayToNibbleArray(key, currentNibbleIndex);
// Since this node is blank, we turn it into a leaf with the key remainder in it.
return(new RLPList(PackPrefixedNode(TrieNodeType.Leaf, keyNibbles), value));
case TrieNodeType.Branch:
// If we reached the end of key, this node is the destination to set the value in.
if (currentNibbleIndex == key.Length * 2)
{
// The key ended, so we set the value here.
node.Items[16] = value;
return(node);
}
else
{
// Obtain our branch index and node
int branchIndex = GetNibble(key, currentNibbleIndex);
RLPList branchNode = DecodeNode(node.Items[branchIndex]);
// Update the value down this path, and obtain the new branch node.
RLPList newBranchNode = NodeUpdateValue(branchNode, key, currentNibbleIndex + 1, value);
// Set the branch node
node.Items[branchIndex] = EncodeNode(newBranchNode);
return(node);
}
case TrieNodeType.Leaf:
case TrieNodeType.Extension:
// Obtain the key nibbles from the node.
byte[] nodeNibbles = UnpackPrefixedNode(node.Items[0]).nibbles;
keyNibbles = ByteArrayToNibbleArray(key, currentNibbleIndex);
// Next we'll want to count all of the shared nibbles among the node nibbles and our key. We'll count all common nibbles.
int sharedNibbles = 0;
int nibbleCount = Math.Min(nodeNibbles.Length, keyNibbles.Length);
for (int i = 0; i < nibbleCount; i++)
{
// If any nibble doesn't match, we can stop counting.
if (keyNibbles[i] != nodeNibbles[i])
{
break;
}
// Otherwise keep counting all shared nibbles.
sharedNibbles++;
}
// We'll create our node here to handle specific cases.
RLPList newNode = null;
// If our node and key nibbles are all shared
if (sharedNibbles == keyNibbles.Length && nodeNibbles.Length == keyNibbles.Length)
{
// And if our node type was a leaf, then we simply return a leaf with the value we provided (instead of any existing value)
if (nodeType == TrieNodeType.Leaf)
{
return(new RLPList(node.Items[0], value));
}
// Otherwise this is an extension, so we decode our next node and update that.
RLPList nextNode = DecodeNode(node.Items[1]);
newNode = NodeUpdateValue(nextNode, key, currentNibbleIndex + sharedNibbles, value);
}
// If all of our node nibbles are done, yet we still have key nibbles
else if (sharedNibbles == nodeNibbles.Length)
{
if (nodeType == TrieNodeType.Leaf)
{
// If our node type is a leaf and we still have key nibbles, our new node type will be a branch node.
newNode = new RLPList();
for (int i = 0; i < 16; i++)
{
newNode.Items.Add(BLANK_NODE);
}
// We add the leaf's value as the branch's value.
newNode.Items.Add(node.Items[1]);
// We create a new leaf that this branch node will connect to with the remainder of our key, and store our value there.
RLPList newLeafNode = new RLPList(PackPrefixedNode(TrieNodeType.Leaf, keyNibbles.Slice(sharedNibbles + 1)), value);
// Set our branch to our new leaf node
int branchIndex = keyNibbles[sharedNibbles];
newNode.Items[branchIndex] = EncodeNode(newLeafNode);
}
else
{
// This is an extension, we simply update the next node
RLPList nextNode = DecodeNode(node.Items[1]);
newNode = NodeUpdateValue(nextNode, key, currentNibbleIndex + sharedNibbles, value);
}
}
else
{
// Both the node and the key have nibbles they didn't agree on.
newNode = new RLPList();
for (int i = 0; i < 17; i++)
{
newNode.Items.Add(BLANK_NODE);
}
// Obtain our branch index for our node's nibble
int branchNodeIndex = nodeNibbles[sharedNibbles];
// If this is an extension and there's one node nibble left.
if (nodeType == TrieNodeType.Extension && nodeNibbles.Length - sharedNibbles == 1)
{
// We set this branch to the extension's next node.
newNode.Items[branchNodeIndex] = node.Items[1];
}
else
{
// Whatever type of node this was before, we put it into our branch slot and advance it a key/strip a key nibble for the new subnode.
RLPList subNodeReplacement = new RLPList(PackPrefixedNode(nodeType, nodeNibbles.Slice(sharedNibbles + 1)), node.Items[1]);
newNode.Items[branchNodeIndex] = EncodeNode(subNodeReplacement);
}
// If our key nibbles are empty, we can set our value in this branch.
if (keyNibbles.Length - sharedNibbles == 0)
{
newNode.Items[16] = value;
}
else
{
// Otherwise we take our next nibble from our key to decide the branch index for our other path.
int branchKeyIndex = keyNibbles[sharedNibbles];
// Create our new leaf node to store the remainder of our key and value.
RLPList keyLeafNode = new RLPList(PackPrefixedNode(TrieNodeType.Leaf, keyNibbles.Slice(sharedNibbles + 1)), value);
newNode.Items[branchKeyIndex] = EncodeNode(keyLeafNode);
}
}
// If we had any shared nibbles between our node's nibbles and our current key's nibbles, we create an extension for that accordingly and link it our created node below it.
if (sharedNibbles > 0)
{
return(new RLPList(PackPrefixedNode(TrieNodeType.Extension, nodeNibbles.Slice(0, sharedNibbles)), EncodeNode(newNode)));
}
else
{
// Otherwise we just return the created node directly.
return(newNode);
}
}
// If we somehow end up here, throw an exception.
throw new ArgumentException("Unexpected node type while updating trie nodes.");
}
/// <summary>
/// Given an trie node, traverses down the appropriate node path to obtain the value for a given key and the current nibble index in our search.
/// </summary>
/// <param name="node">The current trie node to traverse through for our value, given our key and current nibble index.</param>
/// <param name="key">The key for the value which we wish to obtain.</param>
/// <param name="currentNibbleIndex">The index of the current nibble in our key which we are at in our indexing process.</param>
/// <returns>Returns the value stored in the trie for the provided key, or null if it does not exist.</returns>
private byte[] NodeGetValue(RLPList node, Memory <byte> key, int currentNibbleIndex = 0)
{
// Obtain our node type
TrieNodeType nodeType = GetNodeType(node);
// Switch on node type
switch (nodeType)
{
// If it's blank, return the blank node
case TrieNodeType.Blank:
return(null);
case TrieNodeType.Branch:
// If it's a branch and the key to search for has ended, then we've found our result in this node.
if (key.Length * 2 == currentNibbleIndex)
{
// Return the "value" portion of our node, the 17th item.
return(((RLPByteArray)node.Items[16]).Data.ToArray());
}
// Obtain our next nibble which constitutes branch index.
byte branchIndex = GetNibble(key, currentNibbleIndex);
// Obtain our node at the given index and obtain that item
RLPList branchNode = DecodeNode(node.Items[branchIndex]);
// Keep running down the branch recursively.
return(NodeGetValue(branchNode, key, currentNibbleIndex + 1));
case TrieNodeType.Extension:
// If it's an extension, we'll unpack the whole nibble set of our first item (prefix + key remainder)
byte[] nibbles = UnpackPrefixedNode(node.Items[0]).nibbles;
// If there's less of the key than nibbles in this extension, it's not here.
if ((key.Length * 2) - currentNibbleIndex < nibbles.Length)
{
return(null);
}
// Any "shared nibbles" we find in this nibble set should match the current length of our key.
for (int i = 0; i < nibbles.Length; i++)
{
if (nibbles[i] != GetNibble(key, currentNibbleIndex + i))
{
return(null);
}
}
// The second item in this node will represent the "next node", so we recursively obtain from that, advancing our key position as well.
RLPList nextNode = DecodeNode(node.Items[1]);
return(NodeGetValue(nextNode, key, currentNibbleIndex + nibbles.Length));
case TrieNodeType.Leaf:
// If it's a leaf, we'll unpack the whole nibble set of our first item (prefix + key remainder)
nibbles = UnpackPrefixedNode(node.Items[0]).nibbles;
// And we'll want to verify the remainder of our key, first we verify length.
if ((key.Length * 2) - currentNibbleIndex != nibbles.Length)
{
return(null);
}
// Then we make sure the rest of the key sequence matches
for (int i = 0; i < nibbles.Length; i++)
{
if (nibbles[i] != GetNibble(key, currentNibbleIndex + i))
{
return(null);
}
}
// Otherwise we passed verification, so we return the "value" that is stored in this leaf.
return(node.Items[1]);
}
// Throw an exception if we somehow end up here.
throw new ArgumentException("Could not obtain node, invalid node type!");
}
