public static void SetInline(Slice slice, NodeHeader* node)
{
slice.Pointer = (byte*)node + Constants.NodeHeaderSize;
slice.Size = node->KeySize;
slice.KeyLength = node->KeySize;
slice.Array = null;
}
public static void CopyTo(Transaction tx, NodeHeader* node, byte* dest)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
Memory.Copy(dest, overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize);
}
Memory.Copy(dest, (byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize);
}
public static byte* DirectAccess(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
return overFlowPage.Base + Constants.PageHeaderSize;
}
return (byte*) node + node->KeySize + Constants.NodeHeaderSize;
}
public static int GetDataSize(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
return overFlowPage.OverflowSize;
}
return node->DataSize;
}
public unsafe static ValueReader Reader(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
return new ValueReader(overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize);
}
return new ValueReader((byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize);
}
public static int NodeEntry(NodeHeader* other)
{
var sz = other->KeySize + Constants.NodeHeaderSize;
if (other->Flags == NodeFlags.Data || other->Flags == NodeFlags.MultiValuePageRef)
sz += other->DataSize;
sz += sz & 1;
return sz;
}
public static int NodeEntryWithAnotherKey(NodeHeader* other, Slice key)
{
var keySize = key == null ? other->KeySize : key.Size;
var sz = keySize + Constants.NodeHeaderSize;
if (other->Flags == NodeFlags.Data || other->Flags == NodeFlags.MultiValuePageRef)
sz += other->DataSize;
sz += sz & 1;
return sz;
}
public static Slice GetData(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
if (overFlowPage.OverflowSize > ushort.MaxValue)
throw new InvalidOperationException("Cannot convert big data to a slice, too big");
return new Slice(overFlowPage.Base + Constants.PageHeaderSize, (ushort)overFlowPage.OverflowSize);
}
return new Slice((byte*)node + node->KeySize + Constants.NodeHeaderSize, (ushort) node->DataSize);
}
public static ValueReader Reader(Transaction tx, NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef))
{
var overFlowPage = tx.GetReadOnlyPage(node->PageNumber);
Debug.Assert(overFlowPage.IsOverflow, "Requested oveflow page but got " + overFlowPage.Flags);
Debug.Assert(overFlowPage.OverflowSize > 0, "Overflow page cannot be size equal 0 bytes");
return new ValueReader(overFlowPage.Base + Constants.PageHeaderSize, overFlowPage.OverflowSize);
}
return new ValueReader((byte*)node + node->KeySize + Constants.NodeHeaderSize, node->DataSize);
}
public PrefixedSlice(NodeHeader* node)
{
if (node->KeySize > 0)
{
var prefixHeaderPtr = (PrefixedSliceHeader*)((byte*)node + Constants.NodeHeaderSize);
Header = *prefixHeaderPtr;
NonPrefixedData = new Slice((byte*)prefixHeaderPtr + Constants.PrefixedSliceHeaderSize, Header.NonPrefixedDataSize);
Size = node->KeySize;
KeyLength = (ushort) (Header.PrefixUsage + Header.NonPrefixedDataSize);
}
else
{
Size = 0;
KeyLength = 0;
}
Options = SliceOptions.Key;
}
/// <summary>
/// Internal method that is used when splitting pages
/// No need to do any work here, we are always adding at the end
/// </summary>
internal void CopyNodeDataToEndOfPage(NodeHeader* other, MemorySlice key)
{
var index = NumberOfEntries;
Debug.Assert(HasSpaceFor(SizeOf.NodeEntryWithAnotherKey(other, key) + Constants.NodeOffsetSize + SizeOf.NewPrefix(key)));
var nodeSize = SizeOf.NodeEntryWithAnotherKey(other, key);
Debug.Assert(IsBranch == false || index != 0 || key.KeyLength == 0);// branch page's first item must be the implicit ref
var nodeVersion = other->Version; // every time new node is allocated the version is increased, but in this case we do not want to increase it
if (nodeVersion > 0)
nodeVersion -= 1;
var prefixedKey = key as PrefixedSlice;
if (prefixedKey != null && prefixedKey.NewPrefix != null)
WritePrefix(prefixedKey.NewPrefix, prefixedKey.Header.PrefixId);
var newNode = AllocateNewNode(index, nodeSize, nodeVersion);
newNode->KeySize = key.Size;
newNode->Flags = other->Flags;
if(key.Options == SliceOptions.Key && key.Size > 0)
key.CopyTo((byte*)newNode + Constants.NodeHeaderSize);
if (IsBranch || other->Flags == (NodeFlags.PageRef))
{
newNode->PageNumber = other->PageNumber;
newNode->Flags = NodeFlags.PageRef;
return;
}
newNode->DataSize = other->DataSize;
Memory.Copy((byte*)newNode + Constants.NodeHeaderSize + key.Size,
(byte*)other + Constants.NodeHeaderSize + other->KeySize,
other->DataSize);
}
private bool TryOverwriteDataOrMultiValuePageRefNode(NodeHeader* updatedNode, Slice key, int len,
NodeFlags requestedNodeType, ushort? version,
out byte* pos)
{
switch (requestedNodeType)
{
case NodeFlags.Data:
case NodeFlags.MultiValuePageRef:
{
if (updatedNode->DataSize == len &&
(updatedNode->Flags == NodeFlags.Data || updatedNode->Flags == NodeFlags.MultiValuePageRef))
{
CheckConcurrency(key, version, updatedNode->Version, TreeActionType.Add);
if (updatedNode->Version == ushort.MaxValue)
updatedNode->Version = 0;
updatedNode->Version++;
updatedNode->Flags = requestedNodeType;
{
pos = (byte*)updatedNode + Constants.NodeHeaderSize + key.Size;
return true;
}
}
break;
}
case NodeFlags.PageRef:
throw new InvalidOperationException("We never add PageRef explicitly");
default:
throw new ArgumentOutOfRangeException();
}
pos = null;
return false;
}
public StructureReader <T> ReadStructForCurrent <T>(StructureSchema <T> schema)
{
var valueReader = NodeHeader.Reader(_tx, Current);
return(new StructureReader <T>(valueReader.Base, schema));
}
public Stream CreateStreamForCurrent()
{
return(NodeHeader.Stream(_tx, Current));
}
public int GetCurrentDataSize()
{
return(NodeHeader.GetDataSize(_tx, Current));
}
public unsafe static bool ValidateCurrentKey(this IIterator self, NodeHeader* node)
{
if (self.RequiredPrefix != null)
{
var currentKey = new Slice(node);
if (currentKey.StartsWith(self.RequiredPrefix) == false)
return false;
}
if (self.MaxKey != null)
{
var currentKey = new Slice(node);
if (currentKey.Compare(self.MaxKey) >= 0)
return false;
}
return true;
}
// REVIEW: Removed forced inlining for now until we can see if we improve without needing it.
// [MethodImpl(MethodImplOptions.AggressiveInlining)]
public void SetNodeKey(NodeHeader* node, ref MemorySlice sliceInstance)
{
if (KeysPrefixed)
{
var slice = (PrefixedSlice)sliceInstance;
SetNodeKey(node, ref slice);
sliceInstance = slice;
}
else
{
Slice slice = (Slice)sliceInstance;
SetNodeKey(node, ref slice);
sliceInstance = slice;
}
}
internal int GetNumberOfFreePages(NodeHeader* node)
{
return GetNodeDataSize(node) / Constants.PageNumberSize;
}
public Slice(NodeHeader* node)
{
Options = SliceOptions.Key;
SetInline(this, node);
}
public override void Set(NodeHeader* node)
{
SetInline(this, node);
}
public void Set(NodeHeader* node)
{
Set((byte*)node + Constants.NodeHeaderSize, node->KeySize);
}
public void MultiAdd(Slice key, Slice value, ushort?version = null)
{
if (value == null)
{
throw new ArgumentNullException("value");
}
int maxNodeSize = _tx.DataPager.MaxNodeSize;
if (value.Size > maxNodeSize)
{
throw new ArgumentException(
"Cannot add a value to child tree that is over " + maxNodeSize + " bytes in size", "value");
}
if (value.Size == 0)
{
throw new ArgumentException("Cannot add empty value to child tree");
}
State.IsModified = true;
Lazy <Cursor> lazy;
NodeHeader * node;
var page = FindPageFor(key, out node, out lazy);
if ((page == null || page.LastMatch != 0))
{
MultiAddOnNewValue(_tx, key, value, version, maxNodeSize);
return;
}
page = _tx.ModifyPage(page.PageNumber, page);
var item = page.GetNode(page.LastSearchPosition);
// already was turned into a multi tree, not much to do here
if (item->Flags == NodeFlags.MultiValuePageRef)
{
var existingTree = OpenMultiValueTree(_tx, key, item);
existingTree.DirectAdd(value, 0, version: version);
return;
}
byte *nestedPagePtr;
if (item->Flags == NodeFlags.PageRef)
{
var overFlowPage = _tx.ModifyPage(item->PageNumber, null);
nestedPagePtr = overFlowPage.Base + Constants.PageHeaderSize;
}
else
{
nestedPagePtr = NodeHeader.DirectAccess(_tx, item);
}
var nestedPage = new Page(nestedPagePtr, "multi tree", (ushort)NodeHeader.GetDataSize(_tx, item));
var existingItem = nestedPage.Search(value);
if (nestedPage.LastMatch != 0)
{
existingItem = null; // not an actual match, just greater than
}
ushort previousNodeRevision = existingItem != null ? existingItem->Version : (ushort)0;
CheckConcurrency(key, value, version, previousNodeRevision, TreeActionType.Add);
if (existingItem != null)
{
// maybe same value added twice?
var tmpKey = page.GetNodeKey(item);
if (tmpKey.Compare(value) == 0)
{
return; // already there, turning into a no-op
}
nestedPage.RemoveNode(nestedPage.LastSearchPosition);
}
var valueToInsert = nestedPage.PrepareKeyToInsert(value, nestedPage.LastSearchPosition);
if (nestedPage.HasSpaceFor(_tx, valueToInsert, 0))
{
// we are now working on top of the modified root page, we can just modify the memory directly
nestedPage.AddDataNode(nestedPage.LastSearchPosition, valueToInsert, 0, previousNodeRevision);
return;
}
int pageSize = nestedPage.CalcSizeUsed() + Constants.PageHeaderSize;
var newRequiredSize = pageSize + nestedPage.GetRequiredSpace(valueToInsert, 0);
if (newRequiredSize <= maxNodeSize)
{
// we can just expand the current value... no need to create a nested tree yet
var actualPageSize = (ushort)Math.Min(Utils.NearestPowerOfTwo(newRequiredSize), maxNodeSize);
ExpandMultiTreeNestedPageSize(_tx, key, value, nestedPagePtr, actualPageSize, item->DataSize);
return;
}
// we now have to convert this into a tree instance, instead of just a nested page
var tree = Create(_tx, KeysPrefixing, TreeFlags.MultiValue);
for (int i = 0; i < nestedPage.NumberOfEntries; i++)
{
var existingValue = nestedPage.GetNodeKey(i);
tree.DirectAdd(existingValue, 0);
}
tree.DirectAdd(value, 0, version: version);
_tx.AddMultiValueTree(this, key, tree);
// we need to record that we switched to tree mode here, so the next call wouldn't also try to create the tree again
DirectAdd(key, sizeof(TreeRootHeader), NodeFlags.MultiValuePageRef);
}
public void SetNodeKey(NodeHeader* node, ref MemorySlice sliceInstance)
{
if (KeysPrefixed == false)
{
sliceInstance.Set(node);
return;
}
if (node->KeySize == 0)
{
sliceInstance = PrefixedSlice.Empty;
return;
}
PrefixedSlice prefixedSlice;
if (sliceInstance != null && sliceInstance != PrefixedSlice.Empty)
{
sliceInstance.Set(node);
prefixedSlice = (PrefixedSlice)sliceInstance;
}
else
sliceInstance = prefixedSlice = new PrefixedSlice(node);
if (prefixedSlice.Header.PrefixId == PrefixedSlice.NonPrefixedId)
{
Debug.Assert(prefixedSlice.Header.PrefixUsage == 0);
return;
}
Debug.Assert(prefixedSlice.Header.PrefixId < PrefixCount);
if (prefixedSlice.Prefix == null)
prefixedSlice.Prefix = new PrefixNode();
AssertPrefixNode(prefixedSlice.Header.PrefixId);
prefixedSlice.Prefix.Set(_base + _prefixSection->PrefixOffsets[prefixedSlice.Header.PrefixId], PageNumber);
}
private bool TryFindSection(Transaction tx, int minSeq, Slice currentKey, Slice start, Slice end, out NodeHeader* current)
{
int minFreeSpace = _minimumFreePagesInSectionSet ?
_minimumFreePagesInSection :
Math.Min(256, _lastTransactionPageUsage);
current = null;
int currentMax = 0;
using (var it = _env.State.FreeSpaceRoot.Iterate(tx))
{
it.RequiredPrefix = _sectionsPrefix;
it.MaxKey = end;
if (it.Seek(start) == false)
return false;
int triesAfterFindingSuitable = 256;
do
{
if (_recordsToSkip.Exists(x => x.Compare(it.CurrentKey, _env.SliceComparer) == 0))
continue; // if it is marked in memory for either update / delete, we don't want it
if (current != null)
triesAfterFindingSuitable--;
if (currentKey != null)
{
if (_currentKey.Compare(it.CurrentKey, _env.SliceComparer) == 0)
continue; // skip current one
}
using (var stream = it.CreateStreamForCurrent())
using (var reader = new BinaryReader(stream))
{
stream.Position = sizeof(long);
var largestSeq = reader.ReadInt32();
if (largestSeq < minSeq)
continue;
var pageCount = reader.ReadInt32();
if (pageCount < minFreeSpace || pageCount < currentMax)
continue;
current = it.Current;
currentMax = pageCount;
}
} while (it.MoveNext() && triesAfterFindingSuitable >= 0);
}
return current != null;
}
private void SetNodeKey(NodeHeader* node, ref Slice slice)
{
Slice.SetInline(slice,node);
}
private void SetNodeKey(NodeHeader* node, ref PrefixedSlice slice)
{
if (node->KeySize == 0)
{
slice = PrefixedSlice.Empty;
return;
}
if (slice != null && slice != PrefixedSlice.Empty)
{
slice.Set(node);
}
else
{
slice = new PrefixedSlice(node);
}
if (slice.Header.PrefixId == PrefixedSlice.NonPrefixedId)
{
Debug.Assert(slice.Header.PrefixUsage == 0);
return;
}
Debug.Assert(slice.Header.PrefixId < PrefixCount);
if (slice.Prefix == null)
slice.Prefix = new PrefixNode();
AssertPrefixNode(slice.Header.PrefixId);
slice.Prefix.Set(_base + _prefixSection->PrefixOffsets[slice.Header.PrefixId], PageNumber);
}
public IIterator MultiRead(Slice key)
{
Lazy <Cursor> lazy;
NodeHeader * node;
var page = FindPageFor(key, out node, out lazy);
if (page == null || page.LastMatch != 0)
{
return(new EmptyIterator());
}
Debug.Assert(node != null);
var fetchedNodeKey = page.GetNodeKey(node);
if (fetchedNodeKey.Compare(key) != 0)
{
throw new InvalidDataException("Was unable to retrieve the correct node. Data corruption possible");
}
if (node->Flags == NodeFlags.MultiValuePageRef)
{
var tree = OpenMultiValueTree(_tx, key, node);
return(tree.Iterate());
}
var nestedPage = new Page(NodeHeader.DirectAccess(_tx, node), "multi tree", (ushort)NodeHeader.GetDataSize(_tx, node));
return(new PageIterator(nestedPage));
}
public MemorySlice GetNodeKey(NodeHeader* node)
{
if (KeysPrefixed == false)
{
var keySize = node->KeySize;
var key = new byte[keySize];
fixed (byte* ptr = key)
Memory.CopyInline(ptr, (byte*)node + Constants.NodeHeaderSize, keySize);
return new Slice(key);
}
if (node->KeySize == 0)
return new PrefixedSlice(Slice.Empty);
var prefixHeader = (PrefixedSliceHeader*)((byte*)node + Constants.NodeHeaderSize);
var nonPrefixedSize = prefixHeader->NonPrefixedDataSize;
var nonPrefixedData = new byte[nonPrefixedSize];
fixed (byte* ptr = nonPrefixedData)
Memory.CopyInline(ptr, (byte*)prefixHeader + Constants.PrefixedSliceHeaderSize, nonPrefixedSize);
var prefixedSlice = new PrefixedSlice(prefixHeader->PrefixId, prefixHeader->PrefixUsage, new Slice(nonPrefixedData));
if (prefixHeader->PrefixId == PrefixedSlice.NonPrefixedId)
return prefixedSlice;
AssertPrefixNode(prefixedSlice.Header.PrefixId);
var prefixNodePtr = (PrefixNodeHeader*) (_base + _prefixSection->PrefixOffsets[prefixedSlice.Header.PrefixId]);
var prefixLength = prefixNodePtr->PrefixLength;
var prefixData = new byte[prefixLength];
fixed (byte* ptr = prefixData)
Memory.CopyInline(ptr, (byte*)prefixNodePtr + Constants.PrefixNodeHeaderSize, prefixLength);
prefixedSlice.Prefix = new PrefixNode(new PrefixNodeHeader{ PrefixLength = prefixLength }, prefixData, PageNumber);
return prefixedSlice;
}
public override void Set(NodeHeader* node)
{
Pointer = (byte*) node + Constants.NodeHeaderSize;
Size = node->KeySize;
KeyLength = node->KeySize;
Array = null;
}
public void MultiDelete(Slice key, Slice value, ushort?version = null)
{
State.IsModified = true;
Lazy <Cursor> lazy;
NodeHeader * node;
var page = FindPageFor(key, out node, out lazy);
if (page == null || page.LastMatch != 0)
{
return; //nothing to delete - key not found
}
page = _tx.ModifyPage(page.PageNumber, page);
var item = page.GetNode(page.LastSearchPosition);
if (item->Flags == NodeFlags.MultiValuePageRef) //multi-value tree exists
{
var tree = OpenMultiValueTree(_tx, key, item);
tree.Delete(value, version);
// previously, we would convert back to a simple model if we dropped to a single entry
// however, it doesn't really make sense, once you got enough values to go to an actual nested
// tree, you are probably going to remain that way, or be removed completely.
if (tree.State.EntriesCount != 0)
{
return;
}
_tx.TryRemoveMultiValueTree(this, key);
_tx.FreePage(tree.State.RootPageNumber);
Delete(key);
}
else // we use a nested page here
{
var nestedPage = new Page(NodeHeader.DirectAccess(_tx, item), "multi tree", (ushort)NodeHeader.GetDataSize(_tx, item));
var nestedItem = nestedPage.Search(value);
if (nestedItem == null) // value not found
{
return;
}
byte *nestedPagePtr;
if (item->Flags == NodeFlags.PageRef)
{
var overFlowPage = _tx.ModifyPage(item->PageNumber, null);
nestedPagePtr = overFlowPage.Base + Constants.PageHeaderSize;
}
else
{
nestedPagePtr = NodeHeader.DirectAccess(_tx, item);
}
nestedPage = new Page(nestedPagePtr, "multi tree", (ushort)NodeHeader.GetDataSize(_tx, item))
{
LastSearchPosition = nestedPage.LastSearchPosition
};
CheckConcurrency(key, value, version, nestedItem->Version, TreeActionType.Delete);
nestedPage.RemoveNode(nestedPage.LastSearchPosition);
if (nestedPage.NumberOfEntries == 0)
{
Delete(key);
}
}
}
public Slice(NodeHeader* node)
{
Options = SliceOptions.Key;
Set(node);
}
public abstract void Set(NodeHeader* node);
private Tree OpenOrCreateMultiValueTree(Transaction tx, Slice key, NodeHeader* item)
{
Tree tree;
if (tx.TryGetMultiValueTree(this, key, out tree))
return tree;
var childTreeHeader =
(TreeRootHeader*)((byte*)item + item->KeySize + Constants.NodeHeaderSize);
Debug.Assert(childTreeHeader->RootPageNumber < tx.State.NextPageNumber);
tree = childTreeHeader != null ?
Open(tx, childTreeHeader) :
Create(tx);
tx.AddMultiValueTree(this, key, tree);
return tree;
}
private bool TryOverwriteOverflowPages(Transaction tx, TreeMutableState treeState, NodeHeader* updatedNode,
Slice key, int len, ushort? version, out byte* pos)
{
if (updatedNode->Flags == NodeFlags.PageRef &&
tx.Id <= tx.Environment.OldestTransaction) // ensure MVCC - do not overwrite if there is some older active transaction that might read those overflows
{
var overflowPage = tx.GetReadOnlyPage(updatedNode->PageNumber);
if (len <= overflowPage.OverflowSize)
{
CheckConcurrency(key, version, updatedNode->Version, TreeActionType.Add);
if (updatedNode->Version == ushort.MaxValue)
updatedNode->Version = 0;
updatedNode->Version++;
var availableOverflows = tx.DataPager.GetNumberOfOverflowPages(overflowPage.OverflowSize);
var requestedOverflows = tx.DataPager.GetNumberOfOverflowPages(len);
var overflowsToFree = availableOverflows - requestedOverflows;
for (int i = 0; i < overflowsToFree; i++)
{
tx.FreePage(overflowPage.PageNumber + requestedOverflows + i);
}
treeState.OverflowPages -= overflowsToFree;
treeState.PageCount -= overflowsToFree;
overflowPage.OverflowSize = len;
pos = overflowPage.Base + Constants.PageHeaderSize;
return true;
}
}
pos = null;
return false;
}
public ValueReader CreateReaderForCurrent()
{
return(NodeHeader.Reader(_tx, Current));
}
private int GetNodeDataSize(NodeHeader* node)
{
if (node->Flags == (NodeFlags.PageRef)) // lots of data, enough to overflow!
{
var overflowPage = GetReadOnlyPage(node->PageNumber);
return overflowPage.OverflowSize;
}
return node->DataSize;
}
private MemorySlice GetActualKey(Page page, int pos, out NodeHeader* node)
{
node = page.GetNode(pos);
var key = page.GetNodeKey(node);
while (key.KeyLength == 0)
{
Debug.Assert(page.IsBranch);
page = _tx.GetReadOnlyPage(node->PageNumber);
node = page.GetNode(0);
key = page.GetNodeKey(node);
}
return key;
}
