protected unsafe Tuple <Slice, Slice> ReadKey(Transaction txh, Tree tree, Slice key)
{
TreeNodeHeader *node;
tree.FindPageFor(key, out node);
if (node == null)
{
return(null);
}
Slice item1;
TreeNodeHeader.ToSlicePtr(txh.Allocator, node, out item1);
if (SliceComparer.CompareInline(item1, key) != 0)
{
return(null);
}
Slice item2;
Slice.External(txh.Allocator, (byte *)node + node->KeySize + Constants.Tree.NodeHeaderSize, (ushort)node->DataSize, ByteStringType.Immutable, out item2);
return(Tuple.Create(item1, item2));
}
public void DebugValidate(Tree tree, long root)
{
if (NumberOfEntries == 0)
{
return;
}
#if VALIDATE
if (Freed)
{
return;
}
#endif
if (IsBranch && NumberOfEntries < 2)
{
throw new InvalidOperationException("The branch page " + PageNumber + " has " + NumberOfEntries + " entry");
}
Slice prev;
var prevScope = GetNodeKey(tree.Llt, 0, out prev);
try
{
var pages = new HashSet <long>();
for (int i = 1; i < NumberOfEntries; i++)
{
var node = GetNode(i);
Slice current;
var currentScope = GetNodeKey(tree.Llt, i, out current);
if (SliceComparer.CompareInline(prev, current) >= 0)
{
DebugStuff.RenderAndShowTree(tree, root);
throw new InvalidOperationException("The page " + PageNumber + " is not sorted");
}
if (node->Flags == (TreeNodeFlags.PageRef))
{
if (pages.Add(node->PageNumber) == false)
{
DebugStuff.RenderAndShowTree(tree, root);
throw new InvalidOperationException("The page " + PageNumber + " references same page multiple times");
}
}
prevScope.Dispose();
prev = current;
prevScope = currentScope;
}
}
finally
{
prevScope.Dispose();
}
}
public unsafe static bool ValidateCurrentKey <T>(this T self, LowLevelTransaction tx, TreeNodeHeader *node) where T : IIterator
{
if (self.RequiredPrefix.HasValue)
{
var currentKey = TreeNodeHeader.ToSlicePtr(tx.Allocator, node);
if (SliceComparer.StartWith(currentKey, self.RequiredPrefix) == false)
{
return(false);
}
}
if (self.MaxKey.HasValue)
{
var currentKey = TreeNodeHeader.ToSlicePtr(tx.Allocator, node);
if (SliceComparer.CompareInline(currentKey, self.MaxKey) >= 0)
{
return(false);
}
}
return(true);
}
public void DebugValidate(LowLevelTransaction tx, long root)
{
if (NumberOfEntries == 0)
{
return;
}
if (IsBranch && NumberOfEntries < 2)
{
throw new InvalidOperationException("The branch page " + PageNumber + " has " + NumberOfEntries + " entry");
}
var prev = GetNodeKey(tx, 0);
var pages = new HashSet <long>();
for (int i = 1; i < NumberOfEntries; i++)
{
var node = GetNode(i);
var current = GetNodeKey(tx, i);
if (SliceComparer.CompareInline(prev, current) >= 0)
{
DebugStuff.RenderAndShowTree(tx, root);
throw new InvalidOperationException("The page " + PageNumber + " is not sorted");
}
if (node->Flags == (TreeNodeFlags.PageRef))
{
if (pages.Add(node->PageNumber) == false)
{
DebugStuff.RenderAndShowTree(tx, root);
throw new InvalidOperationException("The page " + PageNumber + " references same page multiple times");
}
}
prev = current;
}
}
public int CompareTo(BatchOperation other)
{
var r = SliceComparer.CompareInline(Key, other.Key);
if (r != 0)
{
return(r);
}
if (ValueSlice != null)
{
if (other.ValueSlice == null)
{
return(-1);
}
return(SliceComparer.CompareInline(ValueSlice, other.ValueSlice));
}
else if (other.ValueSlice != null)
{
return(1);
}
return(0);
}
private void HandleUncompressedNodes(DecompressedLeafPage decompressedPage, TreePage p, DecompressionUsage usage)
{
int numberOfEntries = p.NumberOfEntries;
for (var i = 0; i < numberOfEntries; i++)
{
var uncompressedNode = p.GetNode(i);
Slice nodeKey;
using (TreeNodeHeader.ToSlicePtr(_tx.Allocator, uncompressedNode, out nodeKey))
{
if (uncompressedNode->Flags == TreeNodeFlags.CompressionTombstone)
{
HandleTombstone(decompressedPage, nodeKey, usage);
continue;
}
if (decompressedPage.HasSpaceFor(_llt, TreeSizeOf.NodeEntry(uncompressedNode)) == false)
{
throw new InvalidOperationException("Could not add uncompressed node to decompressed page");
}
int index;
if (decompressedPage.NumberOfEntries > 0)
{
Slice lastKey;
using (decompressedPage.GetNodeKey(_llt, decompressedPage.NumberOfEntries - 1, out lastKey))
{
// optimization: it's very likely that uncompressed nodes have greater keys than compressed ones
// when we insert sequential keys
var cmp = SliceComparer.CompareInline(nodeKey, lastKey);
if (cmp > 0)
{
index = decompressedPage.NumberOfEntries;
}
else
{
if (cmp == 0)
{
// update of the last entry, just decrement NumberOfEntries in the page and
// put it at the last position
index = decompressedPage.NumberOfEntries - 1;
decompressedPage.Lower -= Constants.Tree.NodeOffsetSize;
}
else
{
index = decompressedPage.NodePositionFor(_llt, nodeKey);
if (decompressedPage.LastMatch == 0) // update
{
decompressedPage.RemoveNode(index);
if (usage == DecompressionUsage.Write)
{
State.NumberOfEntries--;
}
}
}
}
}
}
else
{
// all uncompressed nodes were compresion tombstones which deleted all entries from the decompressed page
index = 0;
}
switch (uncompressedNode->Flags)
{
case TreeNodeFlags.PageRef:
decompressedPage.AddPageRefNode(index, nodeKey, uncompressedNode->PageNumber);
break;
case TreeNodeFlags.Data:
var pos = decompressedPage.AddDataNode(index, nodeKey, uncompressedNode->DataSize);
var nodeValue = TreeNodeHeader.Reader(_llt, uncompressedNode);
Memory.Copy(pos, nodeValue.Base, nodeValue.Length);
break;
case TreeNodeFlags.MultiValuePageRef:
throw new NotSupportedException("Multi trees do not support compression");
default:
throw new NotSupportedException("Invalid node type to copye: " + uncompressedNode->Flags);
}
}
}
}
public TreeNodeHeader *Search(LowLevelTransaction tx, Slice key)
{
int numberOfEntries = NumberOfEntries;
if (numberOfEntries == 0)
{
LastSearchPosition = 0;
LastMatch = 1;
return(null);
}
switch (key.Options)
{
case SliceOptions.Key:
{
if (numberOfEntries == 1)
{
var node = GetNode(0);
Slice pageKey;
using (TreeNodeHeader.ToSlicePtr(tx.Allocator, node, out pageKey))
{
LastMatch = SliceComparer.CompareInline(key, pageKey);
}
LastSearchPosition = LastMatch > 0 ? 1 : 0;
return(LastSearchPosition == 0 ? node : null);
}
int low = IsLeaf ? 0 : 1;
int high = numberOfEntries - 1;
int position = 0;
ByteStringContext allocator = tx.Allocator;
ushort * offsets = KeysOffsets;
while (low <= high)
{
position = (low + high) >> 1;
var node = (TreeNodeHeader *)(Base + offsets[position]);
Slice pageKey;
using (TreeNodeHeader.ToSlicePtr(allocator, node, out pageKey))
{
LastMatch = SliceComparer.CompareInline(key, pageKey);
}
if (LastMatch == 0)
{
break;
}
if (LastMatch > 0)
{
low = position + 1;
}
else
{
high = position - 1;
}
}
if (LastMatch > 0) // found entry less than key
{
position++; // move to the smallest entry larger than the key
}
Debug.Assert(position < ushort.MaxValue);
LastSearchPosition = position;
if (position >= numberOfEntries)
{
return(null);
}
return(GetNode(position));
}
case SliceOptions.BeforeAllKeys:
{
LastSearchPosition = 0;
LastMatch = 1;
return(GetNode(0));
}
case SliceOptions.AfterAllKeys:
{
LastMatch = -1;
LastSearchPosition = numberOfEntries - 1;
return(GetNode(LastSearchPosition));
}
default:
throw new NotSupportedException("This SliceOptions is not supported. Make sure you have updated this code when adding a new one.");
}
}
private NodeHeader *Search(Slice key)
{
int numberOfEntries = NumberOfEntries;
if (numberOfEntries == 0)
{
LastSearchPosition = 0;
LastMatch = 1;
return(null);
}
switch (key.Options)
{
case SliceOptions.Key:
{
var pageKey = new Slice(SliceOptions.Key);
if (numberOfEntries == 1)
{
var node = GetNode(0);
SetNodeKey(node, ref pageKey);
LastMatch = SliceComparer.CompareInline(key, pageKey);
LastSearchPosition = LastMatch > 0 ? 1 : 0;
return(LastSearchPosition == 0 ? node : null);
}
int low = IsLeaf ? 0 : 1;
int high = numberOfEntries - 1;
int position = 0;
while (low <= high)
{
position = (low + high) >> 1;
var node = (NodeHeader *)(_base + KeysOffsets[position]);
SetNodeKey(node, ref pageKey);
LastMatch = SliceComparer.CompareInline(key, pageKey);
if (LastMatch == 0)
{
break;
}
if (LastMatch > 0)
{
low = position + 1;
}
else
{
high = position - 1;
}
}
if (LastMatch > 0) // found entry less than key
{
position++; // move to the smallest entry larger than the key
}
Debug.Assert(position < ushort.MaxValue);
LastSearchPosition = position;
if (position >= numberOfEntries)
{
return(null);
}
return(GetNode(position));
}
case SliceOptions.BeforeAllKeys:
{
LastSearchPosition = 0;
LastMatch = 1;
return(GetNode(0));
}
case SliceOptions.AfterAllKeys:
{
LastMatch = -1;
LastSearchPosition = numberOfEntries - 1;
return(GetNode(LastSearchPosition));
}
default:
throw new NotSupportedException("This SliceOptions is not supported. Make sure you have updated this code when adding a new one.");
}
}
public TreeNodeHeader *Search(LowLevelTransaction tx, Slice key)
{
int numberOfEntries = NumberOfEntries;
if (numberOfEntries == 0)
{
goto NoEntries;
}
int lastMatch = -1;
int lastSearchPosition = 0;
SliceOptions options = key.Options;
if (options == SliceOptions.Key)
{
if (numberOfEntries == 1)
{
goto SingleEntryKey;
}
int low = IsLeaf ? 0 : 1;
int high = numberOfEntries - 1;
int position = 0;
ByteStringContext allocator = tx.Allocator;
ushort * offsets = KeysOffsets;
byte * @base = Base;
while (low <= high)
{
position = (low + high) >> 1;
var node = (TreeNodeHeader *)(@base + offsets[position]);
Slice pageKey;
using (TreeNodeHeader.ToSlicePtr(allocator, node, out pageKey))
{
lastMatch = SliceComparer.CompareInline(key, pageKey);
}
if (lastMatch == 0)
{
break;
}
if (lastMatch > 0)
{
low = position + 1;
}
else
{
high = position - 1;
}
}
if (lastMatch > 0) // found entry less than key
{
position++; // move to the smallest entry larger than the key
}
Debug.Assert(position < ushort.MaxValue);
lastSearchPosition = position;
goto MultipleEntryKey;
}
if (options == SliceOptions.BeforeAllKeys)
{
lastMatch = 1;
goto MultipleEntryKey;
}
if (options == SliceOptions.AfterAllKeys)
{
lastSearchPosition = numberOfEntries - 1;
goto MultipleEntryKey;
}
ThrowNotSupportedException();
NoEntries:
{
LastSearchPosition = 0;
LastMatch = 1;
return(null);
}
SingleEntryKey:
{
var node = GetNode(0);
Slice pageKey;
using (TreeNodeHeader.ToSlicePtr(tx.Allocator, node, out pageKey))
{
LastMatch = SliceComparer.CompareInline(key, pageKey);
}
LastSearchPosition = LastMatch > 0 ? 1 : 0;
return(LastSearchPosition == 0 ? node : null);
}
MultipleEntryKey:
{
LastMatch = lastMatch;
LastSearchPosition = lastSearchPosition;
if (lastSearchPosition >= numberOfEntries)
{
return(null);
}
return(GetNode(lastSearchPosition));
}
}
public TreeNodeHeader *Original_WithPrefetch_Search(TreePage page, ByteStringContext allocator, Slice key)
{
int numberOfEntries = page.NumberOfEntries;
if (numberOfEntries == 0)
{
goto NoEntries;
}
int lastMatch = -1;
int lastSearchPosition = 0;
SliceOptions options = key.Options;
if (options == SliceOptions.Key)
{
if (numberOfEntries == 1)
{
goto SingleEntryKey;
}
int low = page.IsLeaf ? 0 : 1;
int high = numberOfEntries - 1;
int position = 0;
ushort *offsets = page.KeysOffsets;
byte * @base = page.Base;
while (low <= high)
{
position = (low + high) >> 1;
var node = (TreeNodeHeader *)(@base + offsets[position]);
Slice pageKey;
using (TreeNodeHeader.ToSlicePtr(allocator, node, out pageKey))
{
Sse.Prefetch0(pageKey.Content.Ptr);
lastMatch = SliceComparer.CompareInline(key, pageKey);
}
if (lastMatch == 0)
{
break;
}
if (lastMatch > 0)
{
low = position + 1;
}
else
{
high = position - 1;
}
}
if (lastMatch > 0) // found entry less than key
{
position++; // move to the smallest entry larger than the key
}
lastSearchPosition = position;
goto MultipleEntryKey;
}
if (options == SliceOptions.BeforeAllKeys)
{
lastMatch = 1;
goto MultipleEntryKey;
}
if (options == SliceOptions.AfterAllKeys)
{
lastSearchPosition = numberOfEntries - 1;
goto MultipleEntryKey;
}
return(null);
NoEntries:
{
page.LastSearchPosition = 0;
page.LastMatch = 1;
return(null);
}
SingleEntryKey:
{
var node = page.GetNode(0);
Slice pageKey;
using (TreeNodeHeader.ToSlicePtr(allocator, node, out pageKey))
{
page.LastMatch = SliceComparer.CompareInline(key, pageKey);
}
page.LastSearchPosition = page.LastMatch > 0 ? 1 : 0;
return(page.LastSearchPosition == 0 ? node : null);
}
MultipleEntryKey:
{
page.LastMatch = lastMatch;
page.LastSearchPosition = lastSearchPosition;
if (lastSearchPosition >= numberOfEntries)
{
return(null);
}
return(page.GetNode(lastSearchPosition));
}
}
private ActualKeyScope GetActualKey(TreePage page, int pos, out TreeNodeHeader *node, out Slice key)
{
DecompressedLeafPage decompressedLeafPage = null;
node = page.GetNode(pos);
var scope = TreeNodeHeader.ToSlicePtr(_tx.Allocator, node, out key);
while (key.Size == 0)
{
Debug.Assert(page.IsBranch);
page = _tree.GetReadOnlyTreePage(node->PageNumber);
if (page.IsCompressed == false)
{
node = page.GetNode(0);
}
else
{
decompressedLeafPage?.Dispose();
decompressedLeafPage = _tree.DecompressPage(page, DecompressionUsage.Read, skipCache: true);
if (decompressedLeafPage.NumberOfEntries > 0)
{
if (page.NumberOfEntries == 0)
{
node = decompressedLeafPage.GetNode(0);
}
else
{
// we want to find the smallest key in compressed page
// it can be inside compressed part or not compressed one
// in particular, it can be the key of compression tombstone node that we don't see after decompression
// so we need to take first keys from decompressed and compressed page and compare them
var decompressedNode = decompressedLeafPage.GetNode(0);
var compressedNode = page.GetNode(0);
using (TreeNodeHeader.ToSlicePtr(_tx.Allocator, decompressedNode, out var firstDecompressedKey))
using (TreeNodeHeader.ToSlicePtr(_tx.Allocator, compressedNode, out var firstCompressedKey))
{
node = SliceComparer.CompareInline(firstDecompressedKey, firstCompressedKey) > 0 ? compressedNode : decompressedNode;
}
}
}
else
{
// we have empty page after decompression (each compressed entry has a corresponding CompressionTombstone)
// we can safely use the node key of first tombstone (they have proper order)
node = page.GetNode(0);
}
}
scope.Dispose();
scope = TreeNodeHeader.ToSlicePtr(_tx.Allocator, node, out key);
}
return(new ActualKeyScope
{
DecompressedLeafPage = decompressedLeafPage,
ExternalScope = scope
});
}