public bool Seek(Slice key)
{
if (_disposed)
{
throw new ObjectDisposedException("TreeIterator " + _tree.Name);
}
TreeNodeHeader * node;
TreeCursorConstructor constructor;
_currentPage = _tree.FindPageFor(key, node: out node, cursor: out constructor, allowCompressed: _tree.IsLeafCompressionSupported);
if (_currentPage.IsCompressed)
{
DecompressedCurrentPage();
node = _currentPage.Search(_tx, key);
}
_cursor = constructor.Build(key);
_cursor.Pop();
if (node != null)
{
_prevKeyScope.Dispose();
_prevKeyScope = TreeNodeHeader.ToSlicePtr(_tx.Allocator, node, out _currentInternalKey);
_currentKey = _currentInternalKey;
if (DoRequireValidation)
{
return(this.ValidateCurrentKey(_tx, Current));
}
return(true);
}
// The key is not found in the db, but we are Seek()ing for equals or starts with.
// We know that the exact value isn't there, but it is possible that the next page has values
// that is actually greater than the key, so we need to check it as well.
_currentPage.LastSearchPosition = _currentPage.NumberOfEntries; // force next MoveNext to move to the next _page_.
return(MoveNext());
}
public bool MoveNext()
{
if (_disposed)
{
throw new ObjectDisposedException("TreeIterator " + _tree.Name);
}
while (_currentPage != null)
{
_currentPage.LastSearchPosition++;
if (_currentPage.LastSearchPosition < _currentPage.NumberOfEntries)
{
// run out of entries, need to select the next page...
while (_currentPage.IsBranch)
{
_cursor.Push(_currentPage);
var node = _currentPage.GetNode(_currentPage.LastSearchPosition);
_currentPage = _tree.GetReadOnlyTreePage(node->PageNumber);
_currentPage.LastSearchPosition = 0;
}
var current = _currentPage.GetNode(_currentPage.LastSearchPosition);
if (DoRequireValidation && this.ValidateCurrentKey(_tx, current) == false)
{
return(false);
}
_prevKeyScope.Dispose();
_prevKeyScope = TreeNodeHeader.ToSlicePtr(_tx.Allocator, current, out _currentInternalKey);
_currentKey = _currentInternalKey;
return(true);// there is another entry in this page
}
if (_cursor.PageCount == 0)
{
break;
}
_currentPage = _cursor.Pop();
}
_currentPage = null;
return(false);
}
public void Truncate(LowLevelTransaction tx, int i)
{
if (i >= NumberOfEntries)
{
return;
}
// when truncating, we copy the values to a tmp page
// this has the effect of compacting the page data and avoiding
// internal page fragmentation
TemporaryPage tmp;
using (PageSize <= tx.Environment.Options.PageSize ?
tx.Environment.GetTemporaryPage(tx, out tmp) :
tx.Environment.DecompressionBuffers.GetTemporaryPage(tx, PageSize, out tmp))
{
var copy = tmp.GetTempPage();
copy.TreeFlags = TreeFlags;
var slice = default(Slice);
for (int j = 0; j < i; j++)
{
var node = GetNode(j);
using (TreeNodeHeader.ToSlicePtr(tx.Allocator, node, out slice))
copy.CopyNodeDataToEndOfPage(node, slice);
}
Memory.Copy(Base + Constants.Tree.PageHeaderSize,
copy.Base + Constants.Tree.PageHeaderSize,
PageSize - Constants.Tree.PageHeaderSize);
Upper = copy.Upper;
Lower = copy.Lower;
}
if (LastSearchPosition > i)
{
LastSearchPosition = i;
}
}
private void ExpandMultiTreeNestedPageSize(Slice key, Slice value, byte *nestedPagePtr, ushort newSize, int currentSize)
{
Debug.Assert(newSize > currentSize);
TemporaryPage tmp;
using (_llt.Environment.GetTemporaryPage(_llt, out tmp))
{
var tempPagePointer = tmp.TempPagePointer;
Memory.Copy(tempPagePointer, nestedPagePtr, currentSize);
Delete(key); // release our current page
TreePage nestedPage = new TreePage(tempPagePointer, (ushort)currentSize);
byte *ptr;
using (DirectAdd(key, newSize, out ptr))
{
var newNestedPage = new TreePage(ptr, newSize)
{
Lower = (ushort)Constants.Tree.PageHeaderSize,
Upper = newSize,
TreeFlags = TreePageFlags.Leaf,
PageNumber = -1L, // mark as invalid page number
Flags = 0
};
ByteStringContext allocator = _llt.Allocator;
for (int i = 0; i < nestedPage.NumberOfEntries; i++)
{
var nodeHeader = nestedPage.GetNode(i);
Slice nodeKey;
using (TreeNodeHeader.ToSlicePtr(allocator, nodeHeader, out nodeKey))
newNestedPage.AddDataNode(i, nodeKey, 0);
}
newNestedPage.Search(_llt, value);
newNestedPage.AddDataNode(newNestedPage.LastSearchPosition, value, 0);
}
}
}
private void ExpandMultiTreeNestedPageSize(Slice key, Slice value, byte *nestedPagePtr, ushort newSize, int currentSize)
{
Debug.Assert(newSize > currentSize);
TemporaryPage tmp;
using (_llt.Environment.GetTemporaryPage(_llt, out tmp))
{
var tempPagePointer = tmp.TempPagePointer;
Memory.Copy(tempPagePointer, nestedPagePtr, currentSize);
Delete(key); // release our current page
TreePage nestedPage = new TreePage(tempPagePointer, "multi tree", (ushort)currentSize);
var ptr = DirectAdd(key, newSize);
var newNestedPage = new TreePage(ptr, "multi tree", newSize)
{
Lower = (ushort)Constants.TreePageHeaderSize,
Upper = newSize,
TreeFlags = TreePageFlags.Leaf,
PageNumber = -1L // mark as invalid page number
};
ByteStringContext allocator = _llt.Allocator;
for (int i = 0; i < nestedPage.NumberOfEntries; i++)
{
var nodeHeader = nestedPage.GetNode(i);
Slice nodeKey = TreeNodeHeader.ToSlicePtr(allocator, nodeHeader);
newNestedPage.AddDataNode(i, nodeKey, 0, (ushort)(nodeHeader->Version - 1)); // we dec by one because AdddataNode will inc by one, and we don't want to change those values
nodeKey.Release(allocator);
}
newNestedPage.Search(_llt, value);
newNestedPage.AddDataNode(newNestedPage.LastSearchPosition, value, 0, 0);
}
}
private bool TrySetPosition()
{
if (_disposed)
{
throw new ObjectDisposedException("PageIterator");
}
if (_page.LastSearchPosition < 0 || _page.LastSearchPosition >= _page.NumberOfEntries)
{
return(false);
}
var current = _page.GetNode(_page.LastSearchPosition);
if (DoRequireValidation && this.ValidateCurrentKey(_tx, current) == false)
{
return(false);
}
_currentInternalKey = TreeNodeHeader.ToSlicePtr(_tx.Allocator, current);
_currentKey = _currentInternalKey;
return(true);
}
public static unsafe bool ValidateCurrentKey <T>(this T self, LowLevelTransaction tx, TreeNodeHeader *node) where T : IIterator
{
Slice currentKey;
using (TreeNodeHeader.ToSlicePtr(tx.Allocator, node, out currentKey))
{
if (self.RequiredPrefix.HasValue)
{
if (SliceComparer.StartWith(currentKey, self.RequiredPrefix) == false)
{
return(false);
}
}
if (self.MaxKey.HasValue)
{
if (SliceComparer.CompareInline(currentKey, self.MaxKey) >= 0)
{
return(false);
}
}
}
return(true);
}
public bool Seek(Slice key)
{
if (_disposed)
{
throw new ObjectDisposedException("PageIterator");
}
var current = _page.Search(_tx, key);
if (current == null)
{
return(false);
}
_currentInternalKey = TreeNodeHeader.ToSlicePtr(_tx.Allocator, current);
_currentKey = _currentInternalKey;
if (DoRequireValidation)
{
return(this.ValidateCurrentKey(_tx, current));
}
return(true);
}
public bool Seek(Slice key)
{
if (_disposed)
{
throw new ObjectDisposedException("TreeIterator " + _tree.Name);
}
TreeNodeHeader * node;
Func <TreeCursor> constructor;
_currentPage = _tree.FindPageFor(key, out node, out constructor);
_cursor = constructor();
_cursor.Pop();
if (node != null)
{
_currentInternalKey = TreeNodeHeader.ToSlicePtr(_tx.Allocator, node, ByteStringType.Mutable);
_currentKey = _currentInternalKey; // TODO: Check here if aliasing via pointer is the intended use.
if (DoRequireValidation)
{
return(this.ValidateCurrentKey(_tx, Current));
}
else
{
return(true);
}
}
// The key is not found in the db, but we are Seek()ing for equals or starts with.
// We know that the exact value isn't there, but it is possible that the next page has values
// that is actually greater than the key, so we need to check it as well.
_currentPage.LastSearchPosition = _currentPage.NumberOfEntries; // force next MoveNext to move to the next _page_.
return(MoveNext());
}
private ActualKeyScope GetActualKey(TreePage page, int pos, out TreeNodeHeader *node, out Slice key)
{
DecompressedLeafPage decompressedLeafPage = null;
node = page.GetNode(pos);
var scope = TreeNodeHeader.ToSlicePtr(_tx.Allocator, node, out key);
while (key.Size == 0)
{
Debug.Assert(page.IsBranch);
page = _tree.GetReadOnlyTreePage(node->PageNumber);
if (page.IsCompressed == false)
{
node = page.GetNode(0);
}
else
{
decompressedLeafPage?.Dispose();
decompressedLeafPage = _tree.DecompressPage(page, DecompressionUsage.Read, skipCache: true);
if (decompressedLeafPage.NumberOfEntries > 0)
{
if (page.NumberOfEntries == 0)
{
node = decompressedLeafPage.GetNode(0);
}
else
{
// we want to find the smallest key in compressed page
// it can be inside compressed part or not compressed one
// in particular, it can be the key of compression tombstone node that we don't see after decompression
// so we need to take first keys from decompressed and compressed page and compare them
var decompressedNode = decompressedLeafPage.GetNode(0);
var compressedNode = page.GetNode(0);
using (TreeNodeHeader.ToSlicePtr(_tx.Allocator, decompressedNode, out var firstDecompressedKey))
using (TreeNodeHeader.ToSlicePtr(_tx.Allocator, compressedNode, out var firstCompressedKey))
{
node = SliceComparer.CompareInline(firstDecompressedKey, firstCompressedKey) > 0 ? compressedNode : decompressedNode;
}
}
}
else
{
// we have empty page after decompression (each compressed entry has a corresponding CompressionTombstone)
// we can safely use the node key of first tombstone (they have proper order)
node = page.GetNode(0);
}
}
scope.Dispose();
scope = TreeNodeHeader.ToSlicePtr(_tx.Allocator, node, out key);
}
return(new ActualKeyScope
{
DecompressedLeafPage = decompressedLeafPage,
ExternalScope = scope
});
}
public DirectAddScope DirectAdd(Slice key, int len, TreeNodeFlags nodeType, out byte *ptr)
{
if (_llt.Flags == TransactionFlags.ReadWrite)
{
State.IsModified = true;
}
else
{
ThreadCannotAddInReadTx();
}
if (AbstractPager.IsKeySizeValid(key.Size) == false)
{
ThrowInvalidKeySize(key);
}
var foundPage = FindPageFor(key, node: out TreeNodeHeader * node, cursor: out TreeCursorConstructor cursorConstructor, allowCompressed: true);
var page = ModifyPage(foundPage);
bool?shouldGoToOverflowPage = null;
if (page.LastMatch == 0) // this is an update operation
{
if ((nodeType & TreeNodeFlags.NewOnly) == TreeNodeFlags.NewOnly)
{
ThrowConcurrencyException();
}
node = page.GetNode(page.LastSearchPosition);
#if DEBUG
using (TreeNodeHeader.ToSlicePtr(_llt.Allocator, node, out Slice nodeCheck))
{
Debug.Assert(SliceComparer.EqualsInline(nodeCheck, key));
}
#endif
shouldGoToOverflowPage = ShouldGoToOverflowPage(len);
byte *pos;
if (shouldGoToOverflowPage == false)
{
// optimization for Data and MultiValuePageRef - try to overwrite existing node space
if (TryOverwriteDataOrMultiValuePageRefNode(node, len, nodeType, out pos))
{
ptr = pos;
return(new DirectAddScope(this));
}
}
else
{
// optimization for PageRef - try to overwrite existing overflows
if (TryOverwriteOverflowPages(node, len, out pos))
{
ptr = pos;
return(new DirectAddScope(this));
}
}
RemoveLeafNode(page);
}
else // new item should be recorded
{
State.NumberOfEntries++;
}
nodeType &= ~TreeNodeFlags.NewOnly;
Debug.Assert(nodeType == TreeNodeFlags.Data || nodeType == TreeNodeFlags.MultiValuePageRef);
var lastSearchPosition = page.LastSearchPosition; // searching for overflow pages might change this
byte *overFlowPos = null;
var pageNumber = -1L;
if (shouldGoToOverflowPage ?? ShouldGoToOverflowPage(len))
{
pageNumber = WriteToOverflowPages(len, out overFlowPos);
len = -1;
nodeType = TreeNodeFlags.PageRef;
}
byte *dataPos;
if (page.HasSpaceFor(_llt, key, len) == false)
{
if (IsLeafCompressionSupported == false || TryCompressPageNodes(key, len, page) == false)
{
using (var cursor = cursorConstructor.Build(key))
{
cursor.Update(cursor.Pages, page);
var pageSplitter = new TreePageSplitter(_llt, this, key, len, pageNumber, nodeType, cursor);
dataPos = pageSplitter.Execute();
}
DebugValidateTree(State.RootPageNumber);
ptr = overFlowPos == null ? dataPos : overFlowPos;
return(new DirectAddScope(this));
}
// existing values compressed and put at the end of the page, let's insert from Upper position
lastSearchPosition = 0;
}
switch (nodeType)
{
case TreeNodeFlags.PageRef:
dataPos = page.AddPageRefNode(lastSearchPosition, key, pageNumber);
break;
case TreeNodeFlags.Data:
dataPos = page.AddDataNode(lastSearchPosition, key, len);
break;
case TreeNodeFlags.MultiValuePageRef:
dataPos = page.AddMultiValueNode(lastSearchPosition, key, len);
break;
default:
ThrowUnknownNodeTypeAddOperation(nodeType);
dataPos = null; // never executed
break;
}
page.DebugValidate(this, State.RootPageNumber);
ptr = overFlowPos == null ? dataPos : overFlowPos;
return(new DirectAddScope(this));
}
public TreeNodeHeader *Search(LowLevelTransaction tx, Slice key)
{
int numberOfEntries = NumberOfEntries;
if (numberOfEntries == 0)
{
LastSearchPosition = 0;
LastMatch = 1;
return(null);
}
switch (key.Options)
{
case SliceOptions.Key:
{
if (numberOfEntries == 1)
{
var node = GetNode(0);
Slice pageKey;
using (TreeNodeHeader.ToSlicePtr(tx.Allocator, node, out pageKey))
{
LastMatch = SliceComparer.CompareInline(key, pageKey);
}
LastSearchPosition = LastMatch > 0 ? 1 : 0;
return(LastSearchPosition == 0 ? node : null);
}
int low = IsLeaf ? 0 : 1;
int high = numberOfEntries - 1;
int position = 0;
ByteStringContext allocator = tx.Allocator;
ushort * offsets = KeysOffsets;
while (low <= high)
{
position = (low + high) >> 1;
var node = (TreeNodeHeader *)(Base + offsets[position]);
Slice pageKey;
using (TreeNodeHeader.ToSlicePtr(allocator, node, out pageKey))
{
LastMatch = SliceComparer.CompareInline(key, pageKey);
}
if (LastMatch == 0)
{
break;
}
if (LastMatch > 0)
{
low = position + 1;
}
else
{
high = position - 1;
}
}
if (LastMatch > 0) // found entry less than key
{
position++; // move to the smallest entry larger than the key
}
Debug.Assert(position < ushort.MaxValue);
LastSearchPosition = position;
if (position >= numberOfEntries)
{
return(null);
}
return(GetNode(position));
}
case SliceOptions.BeforeAllKeys:
{
LastSearchPosition = 0;
LastMatch = 1;
return(GetNode(0));
}
case SliceOptions.AfterAllKeys:
{
LastMatch = -1;
LastSearchPosition = numberOfEntries - 1;
return(GetNode(LastSearchPosition));
}
default:
throw new NotSupportedException("This SliceOptions is not supported. Make sure you have updated this code when adding a new one.");
}
}
public List <long> AllPages()
{
var results = new List <long>();
var stack = new Stack <TreePage>();
var root = GetReadOnlyTreePage(State.RootPageNumber);
stack.Push(root);
Slice key = default(Slice);
while (stack.Count > 0)
{
var p = stack.Pop();
results.Add(p.PageNumber);
for (int i = 0; i < p.NumberOfEntries; i++)
{
var node = p.GetNode(i);
var pageNumber = node->PageNumber;
if (p.IsBranch)
{
stack.Push(GetReadOnlyTreePage(pageNumber));
}
else if (node->Flags == TreeNodeFlags.PageRef)
{
// This is an overflow page
var overflowPage = GetReadOnlyTreePage(pageNumber);
var numberOfPages = _llt.DataPager.GetNumberOfOverflowPages(overflowPage.OverflowSize);
for (long j = 0; j < numberOfPages; ++j)
{
results.Add(overflowPage.PageNumber + j);
}
}
else if (node->Flags == TreeNodeFlags.MultiValuePageRef)
{
using (TreeNodeHeader.ToSlicePtr(_tx.Allocator, node, out key))
{
var tree = OpenMultiValueTree(key, node);
results.AddRange(tree.AllPages());
}
}
else
{
if ((State.Flags & TreeFlags.FixedSizeTrees) == TreeFlags.FixedSizeTrees)
{
var valueReader = GetValueReaderFromHeader(node);
var valueSize = ((FixedSizeTreeHeader.Embedded *)valueReader.Base)->ValueSize;
Slice fixedSizeTreeName;
using (p.GetNodeKey(_llt, i, out fixedSizeTreeName))
{
var fixedSizeTree = new FixedSizeTree(_llt, this, fixedSizeTreeName, valueSize);
var pages = fixedSizeTree.AllPages();
results.AddRange(pages);
}
}
}
}
}
return(results);
}
public bool MoveNext()
{
if (_disposed)
{
throw new ObjectDisposedException("TreeIterator " + _tree.Name);
}
while (_currentPage != null)
{
_currentPage.LastSearchPosition++;
if (_currentPage.LastSearchPosition < _currentPage.NumberOfEntries)
{
// run out of entries, need to select the next page...
while (_currentPage.IsBranch)
{
// In here we will also have the 'current' page (even if we are traversing a compressed node).
if (_prefetch)
{
MaybePrefetchPagesReferencedBy(_currentPage);
}
_cursor.Push(_currentPage);
var node = _currentPage.GetNode(_currentPage.LastSearchPosition);
_currentPage = _tree.GetReadOnlyTreePage(node->PageNumber);
if (_currentPage.IsCompressed)
{
DecompressedCurrentPage();
}
_currentPage.LastSearchPosition = 0;
}
// We should be prefetching data pages down here.
if (_prefetch)
{
MaybePrefetchPagesReferencedBy(_currentPage);
}
var current = _currentPage.GetNode(_currentPage.LastSearchPosition);
if (DoRequireValidation && this.ValidateCurrentKey(_tx, current) == false)
{
return(false);
}
_prevKeyScope.Dispose();
_prevKeyScope = TreeNodeHeader.ToSlicePtr(_tx.Allocator, current, out _currentInternalKey);
_currentKey = _currentInternalKey;
return(true);// there is another entry in this page
}
if (_cursor.PageCount == 0)
{
break;
}
_currentPage = _cursor.Pop();
}
_currentPage = null;
return(false);
}
public void MultiAdd(Slice key, Slice value)
{
if (!value.HasValue)
{
throw new ArgumentNullException(nameof(value));
}
int maxNodeSize = Llt.DataPager.NodeMaxSize;
if (value.Size > maxNodeSize)
{
throw new ArgumentException("Cannot add a value to child tree that is over " + maxNodeSize + " bytes in size", nameof(value));
}
if (value.Size == 0)
{
throw new ArgumentException("Cannot add empty value to child tree");
}
State.IsModified = true;
State.Flags |= TreeFlags.MultiValueTrees;
TreeNodeHeader *node;
var page = FindPageFor(key, out node);
if (page == null || page.LastMatch != 0)
{
MultiAddOnNewValue(key, value, maxNodeSize);
return;
}
page = ModifyPage(page);
var item = page.GetNode(page.LastSearchPosition);
byte *_;
// already was turned into a multi tree, not much to do here
if (item->Flags == TreeNodeFlags.MultiValuePageRef)
{
var existingTree = OpenMultiValueTree(key, item);
existingTree.DirectAdd(value, 0, out _).Dispose();
return;
}
if (item->Flags == TreeNodeFlags.PageRef)
{
throw new InvalidOperationException("Multi trees don't use overflows");
}
var nestedPagePtr = DirectAccessFromHeader(item);
var nestedPage = new TreePage(nestedPagePtr, (ushort)GetDataSize(item));
var existingItem = nestedPage.Search(_llt, value);
if (nestedPage.LastMatch != 0)
{
existingItem = null;// not an actual match, just greater than
}
if (existingItem != null)
{
// maybe same value added twice?
Slice tmpKey;
using (TreeNodeHeader.ToSlicePtr(_llt.Allocator, item, out tmpKey))
{
if (SliceComparer.Equals(tmpKey, value))
{
return; // already there, turning into a no-op
}
}
nestedPage.RemoveNode(nestedPage.LastSearchPosition);
}
if (nestedPage.HasSpaceFor(_llt, value, 0))
{
// we are now working on top of the modified root page, we can just modify the memory directly
nestedPage.AddDataNode(nestedPage.LastSearchPosition, value, 0);
return;
}
if (page.HasSpaceFor(_llt, value, 0))
{
// page has space for an additional node in nested page ...
var requiredSpace = nestedPage.PageSize + // existing page
nestedPage.GetRequiredSpace(value, 0); // new node
if (requiredSpace + Constants.Tree.NodeHeaderSize <= maxNodeSize)
{
// ... and it won't require to create an overflow, so we can just expand the current value, no need to create a nested tree yet
EnsureNestedPagePointer(page, item, ref nestedPage, ref nestedPagePtr);
var newPageSize = (ushort)Math.Min(Bits.NextPowerOf2(requiredSpace), maxNodeSize - Constants.Tree.NodeHeaderSize);
ExpandMultiTreeNestedPageSize(key, value, nestedPagePtr, newPageSize, nestedPage.PageSize);
return;
}
}
EnsureNestedPagePointer(page, item, ref nestedPage, ref nestedPagePtr);
// we now have to convert this into a tree instance, instead of just a nested page
var tree = Create(_llt, _tx, key, TreeFlags.MultiValue);
for (int i = 0; i < nestedPage.NumberOfEntries; i++)
{
Slice existingValue;
using (nestedPage.GetNodeKey(_llt, i, out existingValue))
{
tree.DirectAdd(existingValue, 0, out _).Dispose();
}
}
tree.DirectAdd(value, 0, out _).Dispose();
_tx.AddMultiValueTree(this, key, tree);
// we need to record that we switched to tree mode here, so the next call wouldn't also try to create the tree again
DirectAdd(key, sizeof(TreeRootHeader), TreeNodeFlags.MultiValuePageRef, out _).Dispose();
}
private byte *SplitPageInHalf(TreePage rightPage)
{
bool toRight;
var currentIndex = _page.LastSearchPosition;
var splitIndex = _page.NumberOfEntries / 2;
if (currentIndex <= splitIndex)
{
toRight = false;
}
else
{
toRight = true;
var leftPageEntryCount = splitIndex;
var rightPageEntryCount = _page.NumberOfEntries - leftPageEntryCount + 1;
if (rightPageEntryCount > leftPageEntryCount)
{
splitIndex++;
Debug.Assert(splitIndex < _page.NumberOfEntries);
}
}
if (_page.IsLeaf)
{
splitIndex = AdjustSplitPosition(currentIndex, splitIndex, ref toRight);
}
var currentKey = _page.GetNodeKey(_tx, splitIndex);
Slice seperatorKey;
if (toRight && splitIndex == currentIndex)
{
seperatorKey = SliceComparer.Compare(currentKey, _newKey) < 0 ? currentKey : _newKey;
}
else
{
seperatorKey = currentKey;
}
TreePage parentOfRight;
AddSeparatorToParentPage(rightPage.PageNumber, seperatorKey, out parentOfRight);
var parentOfPage = _cursor.CurrentPage;
bool addedAsImplicitRef = false;
if (_page.IsBranch && toRight && SliceComparer.EqualsInline(seperatorKey, _newKey))
{
// _newKey needs to be inserted as first key (BeforeAllKeys) to the right page, so we need to add it before we move entries from the current page
AddNodeToPage(rightPage, 0, Slices.BeforeAllKeys);
addedAsImplicitRef = true;
}
// move the actual entries from page to right page
var instance = new Slice();
ushort nKeys = _page.NumberOfEntries;
for (int i = splitIndex; i < nKeys; i++)
{
TreeNodeHeader *node = _page.GetNode(i);
if (_page.IsBranch && rightPage.NumberOfEntries == 0)
{
rightPage.CopyNodeDataToEndOfPage(node, Slices.BeforeAllKeys);
}
else
{
instance = TreeNodeHeader.ToSlicePtr(_tx.Allocator, node);
rightPage.CopyNodeDataToEndOfPage(node, instance);
}
}
_page.Truncate(_tx, splitIndex);
byte *pos;
if (addedAsImplicitRef == false)
{
try
{
if (toRight && _cursor.CurrentPage.PageNumber != parentOfRight.PageNumber)
{
// modify the cursor if we are going to insert to the right page
_cursor.Pop();
_cursor.Push(parentOfRight);
}
// actually insert the new key
pos = toRight ? InsertNewKey(rightPage) : InsertNewKey(_page);
}
catch (InvalidOperationException e)
{
if (e.Message.StartsWith("The page is full and cannot add an entry", StringComparison.Ordinal) == false)
{
throw;
}
throw new InvalidOperationException(GatherDetailedDebugInfo(rightPage, currentKey, seperatorKey, currentIndex, splitIndex, toRight), e);
}
}
else
{
pos = null;
_cursor.Push(rightPage);
}
if (_page.IsBranch) // remove a branch that has only one entry, the page ref needs to be added to the parent of the current page
{
Debug.Assert(_page.NumberOfEntries > 0);
Debug.Assert(rightPage.NumberOfEntries > 0);
if (_page.NumberOfEntries == 1)
{
RemoveBranchWithOneEntry(_page, parentOfPage);
}
if (rightPage.NumberOfEntries == 1)
{
RemoveBranchWithOneEntry(rightPage, parentOfRight);
}
}
return(pos);
}
public TreeNodeHeader *Search(LowLevelTransaction tx, Slice key)
{
int numberOfEntries = NumberOfEntries;
if (numberOfEntries == 0)
{
goto NoEntries;
}
int lastMatch = -1;
int lastSearchPosition = 0;
SliceOptions options = key.Options;
if (options == SliceOptions.Key)
{
if (numberOfEntries == 1)
{
goto SingleEntryKey;
}
int low = IsLeaf ? 0 : 1;
int high = numberOfEntries - 1;
int position = 0;
ByteStringContext allocator = tx.Allocator;
ushort * offsets = KeysOffsets;
byte * @base = Base;
while (low <= high)
{
position = (low + high) >> 1;
var node = (TreeNodeHeader *)(@base + offsets[position]);
Slice pageKey;
using (TreeNodeHeader.ToSlicePtr(allocator, node, out pageKey))
{
lastMatch = SliceComparer.CompareInline(key, pageKey);
}
if (lastMatch == 0)
{
break;
}
if (lastMatch > 0)
{
low = position + 1;
}
else
{
high = position - 1;
}
}
if (lastMatch > 0) // found entry less than key
{
position++; // move to the smallest entry larger than the key
}
Debug.Assert(position < ushort.MaxValue);
lastSearchPosition = position;
goto MultipleEntryKey;
}
if (options == SliceOptions.BeforeAllKeys)
{
lastMatch = 1;
goto MultipleEntryKey;
}
if (options == SliceOptions.AfterAllKeys)
{
lastSearchPosition = numberOfEntries - 1;
goto MultipleEntryKey;
}
ThrowNotSupportedException();
NoEntries:
{
LastSearchPosition = 0;
LastMatch = 1;
return(null);
}
SingleEntryKey:
{
var node = GetNode(0);
Slice pageKey;
using (TreeNodeHeader.ToSlicePtr(tx.Allocator, node, out pageKey))
{
LastMatch = SliceComparer.CompareInline(key, pageKey);
}
LastSearchPosition = LastMatch > 0 ? 1 : 0;
return(LastSearchPosition == 0 ? node : null);
}
MultipleEntryKey:
{
LastMatch = lastMatch;
LastSearchPosition = lastSearchPosition;
if (lastSearchPosition >= numberOfEntries)
{
return(null);
}
return(GetNode(lastSearchPosition));
}
}
private byte *SplitPageInHalf(TreePage rightPage)
{
bool toRight;
var currentIndex = _page.LastSearchPosition;
var splitIndex = _page.NumberOfEntries / 2;
if (currentIndex <= splitIndex)
{
toRight = false;
}
else
{
toRight = true;
var leftPageEntryCount = splitIndex;
var rightPageEntryCount = _page.NumberOfEntries - leftPageEntryCount + 1;
if (rightPageEntryCount > leftPageEntryCount)
{
splitIndex++;
Debug.Assert(splitIndex < _page.NumberOfEntries);
}
}
if (_page.IsLeaf)
{
splitIndex = AdjustSplitPosition(currentIndex, splitIndex, ref toRight);
}
Slice currentKey;
using (_page.GetNodeKey(_tx, splitIndex, out currentKey))
{
Slice seperatorKey;
if (toRight && splitIndex == currentIndex)
{
seperatorKey = SliceComparer.Compare(currentKey, _newKey) < 0 ? currentKey : _newKey;
}
else
{
seperatorKey = currentKey;
}
var addedAsImplicitRef = false;
var parentOfPage = _cursor.CurrentPage;
TreePage parentOfRight;
DecompressedLeafPage rightDecompressed = null;
if (_pageDecompressed != null)
{
// splitting the decompressed page, let's allocate the page of the same size to ensure enough space
rightDecompressed = _tx.Environment.DecompressionBuffers.GetPage(_tx, _pageDecompressed.PageSize, DecompressionUsage.Write, rightPage);
rightPage = rightDecompressed;
}
using (rightDecompressed)
{
AddSeparatorToParentPage(rightPage.PageNumber, seperatorKey, out parentOfRight);
if (_page.IsBranch && toRight && SliceComparer.EqualsInline(seperatorKey, _newKey))
{
// _newKey needs to be inserted as first key (BeforeAllKeys) to the right page, so we need to add it before we move entries from the current page
AddNodeToPage(rightPage, 0, Slices.BeforeAllKeys);
addedAsImplicitRef = true;
}
// move the actual entries from page to right page
ushort nKeys = _page.NumberOfEntries;
for (int i = splitIndex; i < nKeys; i++)
{
TreeNodeHeader *node = _page.GetNode(i);
if (_page.IsBranch && rightPage.NumberOfEntries == 0)
{
rightPage.CopyNodeDataToEndOfPage(node, Slices.BeforeAllKeys);
}
else
{
Slice instance;
using (TreeNodeHeader.ToSlicePtr(_tx.Allocator, node, out instance))
{
rightPage.CopyNodeDataToEndOfPage(node, instance);
}
}
}
if (rightDecompressed != null)
{
rightDecompressed.CopyToOriginal(_tx, defragRequired: false, wasModified: true);
rightPage = rightDecompressed.Original;
}
}
_page.Truncate(_tx, splitIndex);
RecompressPageIfNeeded(wasModified: true);
byte *pos;
if (addedAsImplicitRef == false)
{
try
{
if (toRight && _cursor.CurrentPage.PageNumber != parentOfRight.PageNumber)
{
// modify the cursor if we are going to insert to the right page
_cursor.Pop();
_cursor.Push(parentOfRight);
}
// actually insert the new key
pos = InsertNewKey(toRight ? rightPage : _page);
}
catch (InvalidOperationException e)
{
if (
e.Message.StartsWith("The page is full and cannot add an entry", StringComparison.Ordinal) ==
false)
{
throw;
}
throw new InvalidOperationException(
GatherDetailedDebugInfo(rightPage, currentKey, seperatorKey, currentIndex, splitIndex,
toRight), e);
}
}
else
{
pos = null;
_cursor.Push(rightPage);
}
if (_page.IsBranch)
// remove a branch that has only one entry, the page ref needs to be added to the parent of the current page
{
Debug.Assert(_page.NumberOfEntries > 0);
Debug.Assert(rightPage.NumberOfEntries > 0);
if (_page.NumberOfEntries == 1)
{
RemoveBranchWithOneEntry(_page, parentOfPage);
}
if (rightPage.NumberOfEntries == 1)
{
RemoveBranchWithOneEntry(rightPage, parentOfRight);
}
}
return(pos);
}
}
private void MoveBranchNode(TreePage parentPage, TreePage from, TreePage to)
{
Debug.Assert(from.IsBranch);
Slice originalFromKey;
using (GetActualKey(from, from.LastSearchPositionOrLastEntry, out originalFromKey))
{
to.EnsureHasSpaceFor(_tx, originalFromKey, -1);
var fromNode = from.GetNode(from.LastSearchPosition);
long pageNum = fromNode->PageNumber;
if (to.LastSearchPosition == 0)
{
// cannot add to left implicit side, adjust by moving the left node
// to the right by one, then adding the new one as the left
TreeNodeHeader *actualKeyNode;
Slice implicitLeftKey;
using (GetActualKey(to, 0, out actualKeyNode, out implicitLeftKey))
{
var implicitLeftNode = to.GetNode(0);
var leftPageNumber = implicitLeftNode->PageNumber;
Slice implicitLeftKeyToInsert;
ByteStringContext.ExternalScope?externalScope;
if (implicitLeftNode == actualKeyNode)
{
externalScope = TreeNodeHeader.ToSlicePtr(_tx.Allocator, actualKeyNode, out implicitLeftKeyToInsert);
}
else
{
implicitLeftKeyToInsert = implicitLeftKey;
externalScope = null;
}
to.EnsureHasSpaceFor(_tx, implicitLeftKeyToInsert, -1);
to.AddPageRefNode(1, implicitLeftKeyToInsert, leftPageNumber);
externalScope?.Dispose();
to.ChangeImplicitRefPageNode(pageNum); // setup the new implicit node
}
}
else
{
to.AddPageRefNode(to.LastSearchPosition, originalFromKey, pageNum);
}
}
if (from.LastSearchPositionOrLastEntry == 0)
{
var rightPageNumber = from.GetNode(1)->PageNumber;
from.RemoveNode(0); // remove the original implicit node
from.ChangeImplicitRefPageNode(rightPageNumber); // setup the new implicit node
Debug.Assert(from.NumberOfEntries >= 2);
}
else
{
from.RemoveNode(from.LastSearchPositionOrLastEntry);
}
var pos = parentPage.LastSearchPositionOrLastEntry;
parentPage.RemoveNode(pos);
Slice newSeparatorKey;
var scope = GetActualKey(to, 0, out newSeparatorKey); // get the next smallest key it has now
try
{
var pageNumber = to.PageNumber;
if (parentPage.GetNode(0)->PageNumber == to.PageNumber)
{
pageNumber = from.PageNumber;
scope.Dispose();
scope = GetActualKey(from, 0, out newSeparatorKey);
}
AddSeparatorToParentPage(to, parentPage, pageNumber, newSeparatorKey, pos);
}
finally
{
scope.Dispose();
}
}
public byte *DirectAdd(Slice key, int len, TreeNodeFlags nodeType = TreeNodeFlags.Data, ushort?version = null)
{
Debug.Assert(nodeType == TreeNodeFlags.Data || nodeType == TreeNodeFlags.MultiValuePageRef);
if (State.InWriteTransaction)
{
State.IsModified = true;
}
if (_llt.Flags == (TransactionFlags.ReadWrite) == false)
{
throw new ArgumentException("Cannot add a value in a read only transaction");
}
if (AbstractPager.IsKeySizeValid(key.Size) == false)
{
throw new ArgumentException($"Key size is too big, must be at most {AbstractPager.MaxKeySize} bytes, but was {(key.Size + AbstractPager.RequiredSpaceForNewNode)}", nameof(key));
}
Func <TreeCursor> cursorConstructor;
TreeNodeHeader * node;
var foundPage = FindPageFor(key, out node, out cursorConstructor);
var page = ModifyPage(foundPage);
ushort nodeVersion = 0;
bool? shouldGoToOverflowPage = null;
if (page.LastMatch == 0) // this is an update operation
{
node = page.GetNode(page.LastSearchPosition);
Debug.Assert(SliceComparer.EqualsInline(TreeNodeHeader.ToSlicePtr(_llt.Allocator, node), key));
shouldGoToOverflowPage = ShouldGoToOverflowPage(len);
byte *pos;
if (shouldGoToOverflowPage == false)
{
// optimization for Data and MultiValuePageRef - try to overwrite existing node space
if (TryOverwriteDataOrMultiValuePageRefNode(node, key, len, nodeType, version, out pos))
{
return(pos);
}
}
else
{
// optimization for PageRef - try to overwrite existing overflows
if (TryOverwriteOverflowPages(node, key, len, version, out pos))
{
return(pos);
}
}
RemoveLeafNode(page, out nodeVersion);
}
else // new item should be recorded
{
State.NumberOfEntries++;
}
CheckConcurrency(key, version, nodeVersion, TreeActionType.Add);
var lastSearchPosition = page.LastSearchPosition; // searching for overflow pages might change this
byte *overFlowPos = null;
var pageNumber = -1L;
if (shouldGoToOverflowPage ?? ShouldGoToOverflowPage(len))
{
pageNumber = WriteToOverflowPages(len, out overFlowPos);
len = -1;
nodeType = TreeNodeFlags.PageRef;
}
byte *dataPos;
if (page.HasSpaceFor(_llt, key, len) == false)
{
using (var cursor = cursorConstructor())
{
cursor.Update(cursor.Pages.First, page);
var pageSplitter = new TreePageSplitter(_llt, this, key, len, pageNumber, nodeType, nodeVersion, cursor);
dataPos = pageSplitter.Execute();
}
DebugValidateTree(State.RootPageNumber);
}
else
{
switch (nodeType)
{
case TreeNodeFlags.PageRef:
dataPos = page.AddPageRefNode(lastSearchPosition, key, pageNumber);
break;
case TreeNodeFlags.Data:
dataPos = page.AddDataNode(lastSearchPosition, key, len, nodeVersion);
break;
case TreeNodeFlags.MultiValuePageRef:
dataPos = page.AddMultiValueNode(lastSearchPosition, key, len, nodeVersion);
break;
default:
throw new NotSupportedException("Unknown node type for direct add operation: " + nodeType);
}
page.DebugValidate(_llt, State.RootPageNumber);
}
if (overFlowPos != null)
{
return(overFlowPos);
}
return(dataPos);
}
private void HandleUncompressedNodes(DecompressedLeafPage decompressedPage, TreePage p, DecompressionUsage usage)
{
int numberOfEntries = p.NumberOfEntries;
for (var i = 0; i < numberOfEntries; i++)
{
var uncompressedNode = p.GetNode(i);
Slice nodeKey;
using (TreeNodeHeader.ToSlicePtr(_tx.Allocator, uncompressedNode, out nodeKey))
{
if (uncompressedNode->Flags == TreeNodeFlags.CompressionTombstone)
{
HandleTombstone(decompressedPage, nodeKey, usage);
continue;
}
if (decompressedPage.HasSpaceFor(_llt, TreeSizeOf.NodeEntry(uncompressedNode)) == false)
{
throw new InvalidOperationException("Could not add uncompressed node to decompressed page");
}
int index;
if (decompressedPage.NumberOfEntries > 0)
{
Slice lastKey;
using (decompressedPage.GetNodeKey(_llt, decompressedPage.NumberOfEntries - 1, out lastKey))
{
// optimization: it's very likely that uncompressed nodes have greater keys than compressed ones
// when we insert sequential keys
var cmp = SliceComparer.CompareInline(nodeKey, lastKey);
if (cmp > 0)
{
index = decompressedPage.NumberOfEntries;
}
else
{
if (cmp == 0)
{
// update of the last entry, just decrement NumberOfEntries in the page and
// put it at the last position
index = decompressedPage.NumberOfEntries - 1;
decompressedPage.Lower -= Constants.Tree.NodeOffsetSize;
}
else
{
index = decompressedPage.NodePositionFor(_llt, nodeKey);
if (decompressedPage.LastMatch == 0) // update
{
decompressedPage.RemoveNode(index);
if (usage == DecompressionUsage.Write)
{
State.NumberOfEntries--;
}
}
}
}
}
}
else
{
// all uncompressed nodes were compresion tombstones which deleted all entries from the decompressed page
index = 0;
}
switch (uncompressedNode->Flags)
{
case TreeNodeFlags.PageRef:
decompressedPage.AddPageRefNode(index, nodeKey, uncompressedNode->PageNumber);
break;
case TreeNodeFlags.Data:
var pos = decompressedPage.AddDataNode(index, nodeKey, uncompressedNode->DataSize);
var nodeValue = TreeNodeHeader.Reader(_llt, uncompressedNode);
Memory.Copy(pos, nodeValue.Base, nodeValue.Length);
break;
case TreeNodeFlags.MultiValuePageRef:
throw new NotSupportedException("Multi trees do not support compression");
default:
throw new NotSupportedException("Invalid node type to copye: " + uncompressedNode->Flags);
}
}
}
}
public byte *Execute()
{
using (DisableFreeSpaceUsageIfSplittingRootTree())
{
TreePage rightPage = _tree.NewPage(_page.TreeFlags, 1);
if (_cursor.PageCount == 0) // we need to do a root split
{
TreePage newRootPage = _tree.NewPage(TreePageFlags.Branch, 1);
_cursor.Push(newRootPage);
_tree.State.RootPageNumber = newRootPage.PageNumber;
_tree.State.Depth++;
// now add implicit left page
newRootPage.AddPageRefNode(0, Slices.BeforeAllKeys, _page.PageNumber);
_parentPage = newRootPage;
_parentPage.LastSearchPosition++;
}
else
{
// we already popped the page, so the current one on the stack is the parent of the page
_parentPage = _tree.ModifyPage(_cursor.CurrentPage);
_cursor.Update(_cursor.Pages.First, _parentPage);
}
if (_page.IsLeaf)
{
_tree.ClearPagesCache();
}
if (_page.IsCompressed)
{
_pageDecompressed = _tree.DecompressPage(_page);
_pageDecompressed.Search(_tx, _newKey);
_page = _pageDecompressed;
}
using (_pageDecompressed)
{
if (_page.LastSearchPosition >= _page.NumberOfEntries)
{
// when we get a split at the end of the page, we take that as a hint that the user is doing
// sequential inserts, at that point, we are going to keep the current page as is and create a new
// page, this will allow us to do minimal amount of work to get the best density
TreePage branchOfSeparator;
byte *pos;
if (_page.IsBranch)
{
if (_page.NumberOfEntries > 2)
{
// here we steal the last entry from the current page so we maintain the implicit null left entry
TreeNodeHeader *node = _page.GetNode(_page.NumberOfEntries - 1);
Debug.Assert(node->Flags == TreeNodeFlags.PageRef);
rightPage.AddPageRefNode(0, Slices.BeforeAllKeys, node->PageNumber);
pos = AddNodeToPage(rightPage, 1);
Slice separatorKey;
using (TreeNodeHeader.ToSlicePtr(_tx.Allocator, node, out separatorKey))
{
AddSeparatorToParentPage(rightPage.PageNumber, separatorKey, out branchOfSeparator);
}
_page.RemoveNode(_page.NumberOfEntries - 1);
}
else
{
_tree.FreePage(rightPage); // return the unnecessary right page
pos = AddSeparatorToParentPage(_pageNumber, _newKey, out branchOfSeparator);
if (_cursor.CurrentPage.PageNumber != branchOfSeparator.PageNumber)
{
_cursor.Push(branchOfSeparator);
}
return(pos);
}
}
else
{
AddSeparatorToParentPage(rightPage.PageNumber, _newKey, out branchOfSeparator);
pos = AddNodeToPage(rightPage, 0);
}
_cursor.Push(rightPage);
return(pos);
}
return(SplitPageInHalf(rightPage));
}
}
}